svn commit: r361416 - in projects/nfs-over-tls/sys: amd64/amd64 amd64/include amd64/vmm amd64/vmm/amd amd64/vmm/intel amd64/vmm/io arm/arm arm/mv/discovery arm/mv/kirkwood arm/mv/orion arm/ralink a...

Sat May 23 15:49:11 UTC 2020

Author: rmacklem
Date: Sat May 23 15:49:07 2020
New Revision: 361416
URL: https://svnweb.freebsd.org/changeset/base/361416

Log:
  Merge in an up-to-date kernel from head and make the changes
  required by recent modifications to the mbuf structure for ext_pgs mbufs.
  Now I just need to see if this all builds and works.

Added:
  projects/nfs-over-tls/sys/amd64/include/vmm_snapshot.h
     - copied unchanged from r361415, head/sys/amd64/include/vmm_snapshot.h
  projects/nfs-over-tls/sys/amd64/vmm/vmm_snapshot.c
     - copied unchanged from r361415, head/sys/amd64/vmm/vmm_snapshot.c
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/irq_work.h
     - copied unchanged from r361415, head/sys/compat/linuxkpi/common/include/linux/irq_work.h
  projects/nfs-over-tls/sys/dev/hyperv/hvsock/
     - copied from r361415, head/sys/dev/hyperv/hvsock/
  projects/nfs-over-tls/sys/modules/hyperv/hvsock/
     - copied from r361415, head/sys/modules/hyperv/hvsock/
  projects/nfs-over-tls/sys/powerpc/aim/mmu_radix.c
     - copied unchanged from r361415, head/sys/powerpc/aim/mmu_radix.c
Deleted:
  projects/nfs-over-tls/sys/arm/mv/discovery/
  projects/nfs-over-tls/sys/arm/mv/kirkwood/
  projects/nfs-over-tls/sys/arm/mv/orion/
  projects/nfs-over-tls/sys/arm/ralink/
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/common/atomic/aarch64/
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/common/atomic/amd64/
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/common/atomic/sparc64/
  projects/nfs-over-tls/sys/crypto/blowfish/
  projects/nfs-over-tls/sys/dev/auxio/
  projects/nfs-over-tls/sys/dev/bktr/
  projects/nfs-over-tls/sys/dev/sound/sbus/
  projects/nfs-over-tls/sys/dev/ubsec/
  projects/nfs-over-tls/sys/modules/auxio/
  projects/nfs-over-tls/sys/modules/bktr/
  projects/nfs-over-tls/sys/modules/epic/
  projects/nfs-over-tls/sys/modules/libalias/modules/cuseeme/
  projects/nfs-over-tls/sys/modules/sound/driver/audiocs/
  projects/nfs-over-tls/sys/modules/ubsec/
  projects/nfs-over-tls/sys/modules/vpo/
  projects/nfs-over-tls/sys/netinet/libalias/alias_cuseeme.c
  projects/nfs-over-tls/sys/opencrypto/cast.c
  projects/nfs-over-tls/sys/opencrypto/cast.h
  projects/nfs-over-tls/sys/opencrypto/castsb.h
  projects/nfs-over-tls/sys/opencrypto/skipjack.c
  projects/nfs-over-tls/sys/opencrypto/skipjack.h
  projects/nfs-over-tls/sys/opencrypto/xform_blf.c
  projects/nfs-over-tls/sys/opencrypto/xform_cast5.c
  projects/nfs-over-tls/sys/opencrypto/xform_des1.c
  projects/nfs-over-tls/sys/opencrypto/xform_des3.c
  projects/nfs-over-tls/sys/opencrypto/xform_md5.c
  projects/nfs-over-tls/sys/opencrypto/xform_skipjack.c
  projects/nfs-over-tls/sys/opencrypto/xform_userland.h
Modified:
  projects/nfs-over-tls/sys/amd64/amd64/cpu_switch.S
  projects/nfs-over-tls/sys/amd64/amd64/initcpu.c
  projects/nfs-over-tls/sys/amd64/amd64/machdep.c
  projects/nfs-over-tls/sys/amd64/amd64/mp_machdep.c
  projects/nfs-over-tls/sys/amd64/amd64/pmap.c
  projects/nfs-over-tls/sys/amd64/amd64/support.S
  projects/nfs-over-tls/sys/amd64/include/vmm.h
  projects/nfs-over-tls/sys/amd64/include/vmm_dev.h
  projects/nfs-over-tls/sys/amd64/vmm/amd/svm.c
  projects/nfs-over-tls/sys/amd64/vmm/amd/svm.h
  projects/nfs-over-tls/sys/amd64/vmm/amd/svm_msr.c
  projects/nfs-over-tls/sys/amd64/vmm/amd/vmcb.c
  projects/nfs-over-tls/sys/amd64/vmm/amd/vmcb.h
  projects/nfs-over-tls/sys/amd64/vmm/intel/vmcs.c
  projects/nfs-over-tls/sys/amd64/vmm/intel/vmcs.h
  projects/nfs-over-tls/sys/amd64/vmm/intel/vmx.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vatpic.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vatpic.h
  projects/nfs-over-tls/sys/amd64/vmm/io/vatpit.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vatpit.h
  projects/nfs-over-tls/sys/amd64/vmm/io/vhpet.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vhpet.h
  projects/nfs-over-tls/sys/amd64/vmm/io/vioapic.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vioapic.h
  projects/nfs-over-tls/sys/amd64/vmm/io/vlapic.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vlapic.h
  projects/nfs-over-tls/sys/amd64/vmm/io/vpmtmr.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vpmtmr.h
  projects/nfs-over-tls/sys/amd64/vmm/io/vrtc.c
  projects/nfs-over-tls/sys/amd64/vmm/io/vrtc.h
  projects/nfs-over-tls/sys/amd64/vmm/vmm.c
  projects/nfs-over-tls/sys/amd64/vmm/vmm_dev.c
  projects/nfs-over-tls/sys/amd64/vmm/x86.c
  projects/nfs-over-tls/sys/amd64/vmm/x86.h
  projects/nfs-over-tls/sys/arm/arm/mp_machdep.c
  projects/nfs-over-tls/sys/arm/xilinx/zy7_mp.c
  projects/nfs-over-tls/sys/arm/xilinx/zy7_slcr.h
  projects/nfs-over-tls/sys/arm64/arm64/cpufunc_asm.S
  projects/nfs-over-tls/sys/arm64/arm64/gicv3_its.c
  projects/nfs-over-tls/sys/arm64/arm64/identcpu.c
  projects/nfs-over-tls/sys/arm64/arm64/mp_machdep.c
  projects/nfs-over-tls/sys/arm64/arm64/pmap.c
  projects/nfs-over-tls/sys/arm64/include/cpufunc.h
  projects/nfs-over-tls/sys/cam/ata/ata_xpt.c
  projects/nfs-over-tls/sys/cam/cam_ccb.h
  projects/nfs-over-tls/sys/cam/cam_xpt.c
  projects/nfs-over-tls/sys/cam/ctl/ctl.c
  projects/nfs-over-tls/sys/cam/ctl/ctl_backend.c
  projects/nfs-over-tls/sys/cam/ctl/ctl_backend.h
  projects/nfs-over-tls/sys/cam/ctl/ctl_backend_block.c
  projects/nfs-over-tls/sys/cam/ctl/ctl_backend_ramdisk.c
  projects/nfs-over-tls/sys/cam/ctl/ctl_frontend_iscsi.c
  projects/nfs-over-tls/sys/cam/ctl/ctl_private.h
  projects/nfs-over-tls/sys/cam/mmc/mmc_xpt.c
  projects/nfs-over-tls/sys/cam/nvme/nvme_da.c
  projects/nfs-over-tls/sys/cam/nvme/nvme_xpt.c
  projects/nfs-over-tls/sys/cam/scsi/scsi_enc_ses.c
  projects/nfs-over-tls/sys/cam/scsi/scsi_targ_bh.c
  projects/nfs-over-tls/sys/cam/scsi/scsi_xpt.c
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/common/lz4/lz4.c
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_rlock.h
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/bitmap.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/errno.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/kernel.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/lockdep.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/math64.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/mutex.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/pci.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/scatterlist.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/srcu.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/wait.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/include/linux/workqueue.h
  projects/nfs-over-tls/sys/compat/linuxkpi/common/src/linux_compat.c
  projects/nfs-over-tls/sys/compat/linuxkpi/common/src/linux_pci.c
  projects/nfs-over-tls/sys/compat/linuxkpi/common/src/linux_work.c
  projects/nfs-over-tls/sys/conf/Makefile.arm
  projects/nfs-over-tls/sys/conf/Makefile.powerpc
  projects/nfs-over-tls/sys/conf/Makefile.riscv
  projects/nfs-over-tls/sys/conf/NOTES
  projects/nfs-over-tls/sys/conf/config.mk
  projects/nfs-over-tls/sys/conf/files
  projects/nfs-over-tls/sys/conf/files.amd64
  projects/nfs-over-tls/sys/conf/files.arm
  projects/nfs-over-tls/sys/conf/files.arm64
  projects/nfs-over-tls/sys/conf/files.i386
  projects/nfs-over-tls/sys/conf/files.mips
  projects/nfs-over-tls/sys/conf/files.powerpc
  projects/nfs-over-tls/sys/conf/files.riscv
  projects/nfs-over-tls/sys/conf/files.x86
  projects/nfs-over-tls/sys/conf/kern.mk
  projects/nfs-over-tls/sys/conf/kern.opts.mk
  projects/nfs-over-tls/sys/conf/kern.post.mk
  projects/nfs-over-tls/sys/conf/kern.pre.mk
  projects/nfs-over-tls/sys/conf/kmod.mk
  projects/nfs-over-tls/sys/conf/ldscript.powerpc
  projects/nfs-over-tls/sys/conf/ldscript.powerpc64
  projects/nfs-over-tls/sys/conf/ldscript.powerpcspe
  projects/nfs-over-tls/sys/conf/options
  projects/nfs-over-tls/sys/conf/options.amd64
  projects/nfs-over-tls/sys/contrib/dev/acpica/changes.txt
  projects/nfs-over-tls/sys/contrib/dev/acpica/common/acgetline.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/common/dmtbdump2.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/compiler/aslcompiler.l
  projects/nfs-over-tls/sys/contrib/dev/acpica/compiler/aslload.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/compiler/aslmessages.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/compiler/aslmessages.h
  projects/nfs-over-tls/sys/contrib/dev/acpica/compiler/aslpredef.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/compiler/aslxref.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/compiler/dtutils.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/components/debugger/dbhistry.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/components/disassembler/dmbuffer.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/components/dispatcher/dsfield.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/components/executer/exfield.c
  projects/nfs-over-tls/sys/contrib/dev/acpica/include/acglobal.h
  projects/nfs-over-tls/sys/contrib/dev/acpica/include/acpixf.h
  projects/nfs-over-tls/sys/contrib/dev/acpica/include/acpredef.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_eeprom.c
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_phy.c
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300eep.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_ap121.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_aphrodite.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_cus157.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_generic.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_hb112.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_hb116.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_osprey_k31.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_wasp_2.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_xb112.h
  projects/nfs-over-tls/sys/contrib/dev/ath/ath_hal/ar9300/ar9300template_xb113.h
  projects/nfs-over-tls/sys/crypto/ccp/ccp.c
  projects/nfs-over-tls/sys/crypto/chacha20/chacha-sw.c
  projects/nfs-over-tls/sys/crypto/via/padlock_hash.c
  projects/nfs-over-tls/sys/ddb/db_expr.c
  projects/nfs-over-tls/sys/dev/acpica/acpi_lid.c
  projects/nfs-over-tls/sys/dev/acpica/acpi_pxm.c
  projects/nfs-over-tls/sys/dev/acpica/acpi_video.c
  projects/nfs-over-tls/sys/dev/acpica/acpivar.h
  projects/nfs-over-tls/sys/dev/amdtemp/amdtemp.c
  projects/nfs-over-tls/sys/dev/ath/ath_hal/ah.h
  projects/nfs-over-tls/sys/dev/ath/ath_hal/ar5416/ar2133.c
  projects/nfs-over-tls/sys/dev/ath/ath_rate/amrr/amrr.c
  projects/nfs-over-tls/sys/dev/ath/ath_rate/onoe/onoe.c
  projects/nfs-over-tls/sys/dev/ath/ath_rate/sample/sample.c
  projects/nfs-over-tls/sys/dev/ath/ath_rate/sample/sample.h
  projects/nfs-over-tls/sys/dev/ath/if_ath.c
  projects/nfs-over-tls/sys/dev/ath/if_ath_misc.h
  projects/nfs-over-tls/sys/dev/ath/if_ath_rx_edma.c
  projects/nfs-over-tls/sys/dev/ath/if_ath_tx.c
  projects/nfs-over-tls/sys/dev/ath/if_ath_tx_ht.c
  projects/nfs-over-tls/sys/dev/ath/if_athrate.h
  projects/nfs-over-tls/sys/dev/ath/if_athvar.h
  projects/nfs-over-tls/sys/dev/bnxt/bnxt.h
  projects/nfs-over-tls/sys/dev/bnxt/if_bnxt.c
  projects/nfs-over-tls/sys/dev/cesa/cesa.c
  projects/nfs-over-tls/sys/dev/cxgbe/adapter.h
  projects/nfs-over-tls/sys/dev/cxgbe/crypto/t4_crypto.c
  projects/nfs-over-tls/sys/dev/cxgbe/crypto/t4_kern_tls.c
  projects/nfs-over-tls/sys/dev/cxgbe/iw_cxgbe/cm.c
  projects/nfs-over-tls/sys/dev/cxgbe/iw_cxgbe/device.c
  projects/nfs-over-tls/sys/dev/cxgbe/offload.h
  projects/nfs-over-tls/sys/dev/cxgbe/t4_main.c
  projects/nfs-over-tls/sys/dev/cxgbe/t4_sge.c
  projects/nfs-over-tls/sys/dev/cxgbe/tom/t4_cpl_io.c
  projects/nfs-over-tls/sys/dev/cxgbe/tom/t4_tls.c
  projects/nfs-over-tls/sys/dev/e1000/if_em.c
  projects/nfs-over-tls/sys/dev/ena/ena.c
  projects/nfs-over-tls/sys/dev/ena/ena.h
  projects/nfs-over-tls/sys/dev/ena/ena_sysctl.c
  projects/nfs-over-tls/sys/dev/ena/ena_sysctl.h
  projects/nfs-over-tls/sys/dev/etherswitch/etherswitch.h
  projects/nfs-over-tls/sys/dev/evdev/evdev_utils.c
  projects/nfs-over-tls/sys/dev/evdev/input-event-codes.h
  projects/nfs-over-tls/sys/dev/evdev/input.h
  projects/nfs-over-tls/sys/dev/glxsb/glxsb.c
  projects/nfs-over-tls/sys/dev/gpio/gpioiic.c
  projects/nfs-over-tls/sys/dev/hifn/hifn7751.c
  projects/nfs-over-tls/sys/dev/hyperv/include/vmbus.h
  projects/nfs-over-tls/sys/dev/hyperv/vmbus/vmbus.c
  projects/nfs-over-tls/sys/dev/hyperv/vmbus/vmbus_br.c
  projects/nfs-over-tls/sys/dev/hyperv/vmbus/vmbus_brvar.h
  projects/nfs-over-tls/sys/dev/hyperv/vmbus/vmbus_chan.c
  projects/nfs-over-tls/sys/dev/hyperv/vmbus/vmbus_chanvar.h
  projects/nfs-over-tls/sys/dev/hyperv/vmbus/vmbus_reg.h
  projects/nfs-over-tls/sys/dev/ichiic/ig4_acpi.c
  projects/nfs-over-tls/sys/dev/iscsi/icl_soft.c
  projects/nfs-over-tls/sys/dev/iwm/if_iwm.c
  projects/nfs-over-tls/sys/dev/iwm/if_iwmvar.h
  projects/nfs-over-tls/sys/dev/ixgbe/if_ix.c
  projects/nfs-over-tls/sys/dev/ixgbe/if_ixv.c
  projects/nfs-over-tls/sys/dev/ixl/if_iavf.c
  projects/nfs-over-tls/sys/dev/ixl/if_ixl.c
  projects/nfs-over-tls/sys/dev/mlx5/device.h
  projects/nfs-over-tls/sys/dev/mlx5/mlx5_core/mlx5_eq.c
  projects/nfs-over-tls/sys/dev/mlx5/mlx5_core/mlx5_main.c
  projects/nfs-over-tls/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
  projects/nfs-over-tls/sys/dev/nvdimm/nvdimm.c
  projects/nfs-over-tls/sys/dev/nvme/nvme.c
  projects/nfs-over-tls/sys/dev/nvme/nvme_ctrlr.c
  projects/nfs-over-tls/sys/dev/nvme/nvme_ns.c
  projects/nfs-over-tls/sys/dev/nvme/nvme_pci.c
  projects/nfs-over-tls/sys/dev/nvme/nvme_qpair.c
  projects/nfs-over-tls/sys/dev/nvme/nvme_sim.c
  projects/nfs-over-tls/sys/dev/nvme/nvme_sysctl.c
  projects/nfs-over-tls/sys/dev/nvme/nvme_test.c
  projects/nfs-over-tls/sys/dev/ow/ow.c
  projects/nfs-over-tls/sys/dev/qlnx/qlnxe/bcm_osal.h
  projects/nfs-over-tls/sys/dev/random/random_harvestq.c
  projects/nfs-over-tls/sys/dev/rtwn/usb/rtwn_usb_attach.h
  projects/nfs-over-tls/sys/dev/safe/safe.c
  projects/nfs-over-tls/sys/dev/sec/sec.c
  projects/nfs-over-tls/sys/dev/sound/pci/hda/hdac.c
  projects/nfs-over-tls/sys/dev/sound/pcm/dsp.c
  projects/nfs-over-tls/sys/dev/usb/usb_device.c
  projects/nfs-over-tls/sys/dev/usb/usb_device.h
  projects/nfs-over-tls/sys/dev/usb/usb_hub.c
  projects/nfs-over-tls/sys/dev/usb/usbdevs
  projects/nfs-over-tls/sys/dev/usb/usbhid.h
  projects/nfs-over-tls/sys/dev/virtio/balloon/virtio_balloon.c
  projects/nfs-over-tls/sys/dev/virtio/console/virtio_console.c
  projects/nfs-over-tls/sys/dev/virtio/mmio/virtio_mmio.c
  projects/nfs-over-tls/sys/dev/virtio/mmio/virtio_mmio.h
  projects/nfs-over-tls/sys/dev/virtio/random/virtio_random.c
  projects/nfs-over-tls/sys/dev/virtio/scsi/virtio_scsi.c
  projects/nfs-over-tls/sys/dev/xen/evtchn/evtchn_dev.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_alloc.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_balloc.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_bmap.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_csum.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_extattr.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_extents.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_extents.h
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_htree.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_inode.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_inode_cnv.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_lookup.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_subr.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_vfsops.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2_vnops.c
  projects/nfs-over-tls/sys/fs/ext2fs/ext2fs.h
  projects/nfs-over-tls/sys/fs/ext2fs/fs.h
  projects/nfs-over-tls/sys/fs/fuse/fuse_internal.c
  projects/nfs-over-tls/sys/fs/fuse/fuse_vfsops.c
  projects/nfs-over-tls/sys/fs/fuse/fuse_vnops.c
  projects/nfs-over-tls/sys/fs/nfs/nfs_commonacl.c
  projects/nfs-over-tls/sys/fs/nfs/nfs_commonsubs.c
  projects/nfs-over-tls/sys/fs/nfs/nfs_var.h
  projects/nfs-over-tls/sys/fs/nfs/nfsdport.h
  projects/nfs-over-tls/sys/fs/nfs/nfsm_subs.h
  projects/nfs-over-tls/sys/fs/nfs/nfsport.h
  projects/nfs-over-tls/sys/fs/nfsclient/nfs_clcomsubs.c
  projects/nfs-over-tls/sys/fs/nfsclient/nfs_clport.c
  projects/nfs-over-tls/sys/fs/nfsclient/nfs_clrpcops.c
  projects/nfs-over-tls/sys/fs/nfsclient/nfs_clstate.c
  projects/nfs-over-tls/sys/fs/nfsclient/nfs_clvfsops.c
  projects/nfs-over-tls/sys/fs/nfsserver/nfs_nfsdcache.c
  projects/nfs-over-tls/sys/fs/nfsserver/nfs_nfsdport.c
  projects/nfs-over-tls/sys/fs/nfsserver/nfs_nfsdserv.c
  projects/nfs-over-tls/sys/fs/nfsserver/nfs_nfsdsocket.c
  projects/nfs-over-tls/sys/fs/nfsserver/nfs_nfsdstate.c
  projects/nfs-over-tls/sys/fs/nfsserver/nfs_nfsdsubs.c
  projects/nfs-over-tls/sys/geom/eli/g_eli.c
  projects/nfs-over-tls/sys/geom/geom.h
  projects/nfs-over-tls/sys/geom/geom_dev.c
  projects/nfs-over-tls/sys/geom/geom_disk.c
  projects/nfs-over-tls/sys/geom/geom_dump.c
  projects/nfs-over-tls/sys/geom/geom_subr.c
  projects/nfs-over-tls/sys/geom/mirror/g_mirror_ctl.c
  projects/nfs-over-tls/sys/geom/mountver/g_mountver.c
  projects/nfs-over-tls/sys/geom/nop/g_nop.c
  projects/nfs-over-tls/sys/geom/part/g_part.c
  projects/nfs-over-tls/sys/geom/uzip/g_uzip.c
  projects/nfs-over-tls/sys/i386/i386/mp_machdep.c
  projects/nfs-over-tls/sys/i386/i386/support.s
  projects/nfs-over-tls/sys/kern/kern_exec.c
  projects/nfs-over-tls/sys/kern/kern_fork.c
  projects/nfs-over-tls/sys/kern/kern_linker.c
  projects/nfs-over-tls/sys/kern/kern_mbuf.c
  projects/nfs-over-tls/sys/kern/kern_mib.c
  projects/nfs-over-tls/sys/kern/kern_procctl.c
  projects/nfs-over-tls/sys/kern/kern_sendfile.c
  projects/nfs-over-tls/sys/kern/kern_sig.c
  projects/nfs-over-tls/sys/kern/link_elf.c
  projects/nfs-over-tls/sys/kern/subr_bus_dma.c
  projects/nfs-over-tls/sys/kern/subr_prf.c
  projects/nfs-over-tls/sys/kern/subr_sglist.c
  projects/nfs-over-tls/sys/kern/subr_uio.c
  projects/nfs-over-tls/sys/kern/tty.c
  projects/nfs-over-tls/sys/kern/uipc_ktls.c
  projects/nfs-over-tls/sys/kern/uipc_mbuf.c
  projects/nfs-over-tls/sys/kern/uipc_sockbuf.c
  projects/nfs-over-tls/sys/kern/uipc_socket.c
  projects/nfs-over-tls/sys/kern/vfs_export.c
  projects/nfs-over-tls/sys/kern/vfs_extattr.c
  projects/nfs-over-tls/sys/kern/vfs_subr.c
  projects/nfs-over-tls/sys/mips/atheros/ar531x/ar5315_machdep.c
  projects/nfs-over-tls/sys/mips/atheros/ar71xx_machdep.c
  projects/nfs-over-tls/sys/mips/atheros/if_arge.c
  projects/nfs-over-tls/sys/mips/atheros/if_argevar.h
  projects/nfs-over-tls/sys/mips/cavium/cryptocteon/cavium_crypto.c
  projects/nfs-over-tls/sys/mips/cavium/cryptocteon/cryptocteon.c
  projects/nfs-over-tls/sys/mips/cavium/cryptocteon/cryptocteonvar.h
  projects/nfs-over-tls/sys/mips/cavium/octeon_machdep.c
  projects/nfs-over-tls/sys/mips/include/md_var.h
  projects/nfs-over-tls/sys/mips/mediatek/mtk_soc.c
  projects/nfs-over-tls/sys/mips/mips/cpu.c
  projects/nfs-over-tls/sys/mips/mips/freebsd32_machdep.c
  projects/nfs-over-tls/sys/mips/mips/machdep.c
  projects/nfs-over-tls/sys/mips/mips/pm_machdep.c
  projects/nfs-over-tls/sys/mips/mips/swtch.S
  projects/nfs-over-tls/sys/mips/nlm/dev/sec/nlmsec.c
  projects/nfs-over-tls/sys/mips/nlm/dev/sec/nlmseclib.c
  projects/nfs-over-tls/sys/modules/Makefile
  projects/nfs-over-tls/sys/modules/crypto/Makefile
  projects/nfs-over-tls/sys/modules/hyperv/Makefile
  projects/nfs-over-tls/sys/modules/libalias/modules/modules.inc
  projects/nfs-over-tls/sys/modules/vmm/Makefile
  projects/nfs-over-tls/sys/net/if.c
  projects/nfs-over-tls/sys/net/iflib.c
  projects/nfs-over-tls/sys/net/mppcc.c
  projects/nfs-over-tls/sys/net/netisr.c
  projects/nfs-over-tls/sys/net/pfvar.h
  projects/nfs-over-tls/sys/net/route.c
  projects/nfs-over-tls/sys/net/route.h
  projects/nfs-over-tls/sys/net/route/nhop.h
  projects/nfs-over-tls/sys/net/route/nhop_ctl.c
  projects/nfs-over-tls/sys/net/route/nhop_var.h
  projects/nfs-over-tls/sys/net/route/route_ddb.c
  projects/nfs-over-tls/sys/net/route/route_helpers.c
  projects/nfs-over-tls/sys/net/route/route_var.h
  projects/nfs-over-tls/sys/net/rtsock.c
  projects/nfs-over-tls/sys/net80211/ieee80211_freebsd.c
  projects/nfs-over-tls/sys/net80211/ieee80211_output.c
  projects/nfs-over-tls/sys/net80211/ieee80211_proto.c
  projects/nfs-over-tls/sys/net80211/ieee80211_proto.h
  projects/nfs-over-tls/sys/netgraph/bluetooth/include/ng_hci.h
  projects/nfs-over-tls/sys/netinet/accf_data.c
  projects/nfs-over-tls/sys/netinet/accf_dns.c
  projects/nfs-over-tls/sys/netinet/accf_http.c
  projects/nfs-over-tls/sys/netinet/cc/cc_cubic.c
  projects/nfs-over-tls/sys/netinet/cc/cc_dctcp.c
  projects/nfs-over-tls/sys/netinet/in_pcb.c
  projects/nfs-over-tls/sys/netinet/in_pcb.h
  projects/nfs-over-tls/sys/netinet/in_rmx.c
  projects/nfs-over-tls/sys/netinet/ip_output.c
  projects/nfs-over-tls/sys/netinet/libalias/alias.c
  projects/nfs-over-tls/sys/netinet/libalias/alias_ftp.c
  projects/nfs-over-tls/sys/netinet/libalias/libalias.3
  projects/nfs-over-tls/sys/netinet/sctp_asconf.c
  projects/nfs-over-tls/sys/netinet/sctp_auth.c
  projects/nfs-over-tls/sys/netinet/sctp_indata.c
  projects/nfs-over-tls/sys/netinet/sctp_indata.h
  projects/nfs-over-tls/sys/netinet/sctp_input.c
  projects/nfs-over-tls/sys/netinet/sctp_os_bsd.h
  projects/nfs-over-tls/sys/netinet/sctp_output.c
  projects/nfs-over-tls/sys/netinet/sctp_pcb.c
  projects/nfs-over-tls/sys/netinet/sctp_timer.c
  projects/nfs-over-tls/sys/netinet/sctp_usrreq.c
  projects/nfs-over-tls/sys/netinet/sctputil.c
  projects/nfs-over-tls/sys/netinet/tcp_input.c
  projects/nfs-over-tls/sys/netinet/tcp_output.c
  projects/nfs-over-tls/sys/netinet/tcp_pcap.c
  projects/nfs-over-tls/sys/netinet/tcp_stacks/bbr.c
  projects/nfs-over-tls/sys/netinet/tcp_stacks/rack.c
  projects/nfs-over-tls/sys/netinet/tcp_stacks/rack_bbr_common.c
  projects/nfs-over-tls/sys/netinet/tcp_stacks/rack_bbr_common.h
  projects/nfs-over-tls/sys/netinet/tcp_stacks/tcp_bbr.h
  projects/nfs-over-tls/sys/netinet/tcp_stacks/tcp_rack.h
  projects/nfs-over-tls/sys/netinet/tcp_usrreq.c
  projects/nfs-over-tls/sys/netinet/tcp_var.h
  projects/nfs-over-tls/sys/netinet6/in6_pcb.c
  projects/nfs-over-tls/sys/netinet6/in6_pcb.h
  projects/nfs-over-tls/sys/netinet6/in6_rmx.c
  projects/nfs-over-tls/sys/netinet6/ip6_output.c
  projects/nfs-over-tls/sys/netinet6/nd6.c
  projects/nfs-over-tls/sys/netinet6/nd6_rtr.c
  projects/nfs-over-tls/sys/netipsec/ipsec.c
  projects/nfs-over-tls/sys/netipsec/ipsec.h
  projects/nfs-over-tls/sys/netipsec/key.c
  projects/nfs-over-tls/sys/netipsec/xform_ah.c
  projects/nfs-over-tls/sys/netipsec/xform_esp.c
  projects/nfs-over-tls/sys/netpfil/ipfw/ip_fw2.c
  projects/nfs-over-tls/sys/netpfil/pf/pf_ioctl.c
  projects/nfs-over-tls/sys/netpfil/pf/pf_table.c
  projects/nfs-over-tls/sys/nfs/bootp_subr.c
  projects/nfs-over-tls/sys/opencrypto/crypto.c
  projects/nfs-over-tls/sys/opencrypto/cryptodev.c
  projects/nfs-over-tls/sys/opencrypto/cryptodev.h
  projects/nfs-over-tls/sys/opencrypto/cryptosoft.c
  projects/nfs-over-tls/sys/opencrypto/xform.c
  projects/nfs-over-tls/sys/opencrypto/xform.h
  projects/nfs-over-tls/sys/opencrypto/xform_aes_icm.c
  projects/nfs-over-tls/sys/opencrypto/xform_aes_xts.c
  projects/nfs-over-tls/sys/opencrypto/xform_auth.h
  projects/nfs-over-tls/sys/opencrypto/xform_cml.c
  projects/nfs-over-tls/sys/opencrypto/xform_comp.h
  projects/nfs-over-tls/sys/opencrypto/xform_enc.h
  projects/nfs-over-tls/sys/opencrypto/xform_gmac.c
  projects/nfs-over-tls/sys/opencrypto/xform_null.c
  projects/nfs-over-tls/sys/opencrypto/xform_rijndael.c
  projects/nfs-over-tls/sys/opencrypto/xform_sha1.c
  projects/nfs-over-tls/sys/powerpc/aim/aim_machdep.c
  projects/nfs-over-tls/sys/powerpc/aim/locore64.S
  projects/nfs-over-tls/sys/powerpc/aim/mmu_oea.c
  projects/nfs-over-tls/sys/powerpc/aim/mmu_oea64.c
  projects/nfs-over-tls/sys/powerpc/aim/trap_subr64.S
  projects/nfs-over-tls/sys/powerpc/booke/machdep_e500.c
  projects/nfs-over-tls/sys/powerpc/booke/pmap.c
  projects/nfs-over-tls/sys/powerpc/conf/QORIQ64
  projects/nfs-over-tls/sys/powerpc/include/cpu.h
  projects/nfs-over-tls/sys/powerpc/include/cpufunc.h
  projects/nfs-over-tls/sys/powerpc/include/db_machdep.h
  projects/nfs-over-tls/sys/powerpc/include/mmuvar.h
  projects/nfs-over-tls/sys/powerpc/include/param.h
  projects/nfs-over-tls/sys/powerpc/include/pmap.h
  projects/nfs-over-tls/sys/powerpc/include/pte.h
  projects/nfs-over-tls/sys/powerpc/include/spr.h
  projects/nfs-over-tls/sys/powerpc/include/sr.h
  projects/nfs-over-tls/sys/powerpc/include/vmparam.h
  projects/nfs-over-tls/sys/powerpc/ofw/ofw_initrd.c
  projects/nfs-over-tls/sys/powerpc/powerpc/elf32_machdep.c
  projects/nfs-over-tls/sys/powerpc/powerpc/elf64_machdep.c
  projects/nfs-over-tls/sys/powerpc/powerpc/machdep.c
  projects/nfs-over-tls/sys/powerpc/powerpc/mmu_if.m
  projects/nfs-over-tls/sys/powerpc/powerpc/pmap_dispatch.c
  projects/nfs-over-tls/sys/powerpc/powerpc/trap.c
  projects/nfs-over-tls/sys/riscv/include/sbi.h
  projects/nfs-over-tls/sys/riscv/riscv/locore.S
  projects/nfs-over-tls/sys/riscv/riscv/machdep.c
  projects/nfs-over-tls/sys/riscv/riscv/mp_machdep.c
  projects/nfs-over-tls/sys/riscv/riscv/pmap.c
  projects/nfs-over-tls/sys/riscv/riscv/sbi.c
  projects/nfs-over-tls/sys/rpc/rpc_generic.c
  projects/nfs-over-tls/sys/security/audit/audit_bsm.c
  projects/nfs-over-tls/sys/security/mac_bsdextended/mac_bsdextended.c
  projects/nfs-over-tls/sys/security/mac_bsdextended/mac_bsdextended.h
  projects/nfs-over-tls/sys/sys/elf_common.h
  projects/nfs-over-tls/sys/sys/ktls.h
  projects/nfs-over-tls/sys/sys/link_elf.h
  projects/nfs-over-tls/sys/sys/mbuf.h
  projects/nfs-over-tls/sys/sys/mount.h
  projects/nfs-over-tls/sys/sys/param.h
  projects/nfs-over-tls/sys/sys/random.h
  projects/nfs-over-tls/sys/sys/sglist.h
  projects/nfs-over-tls/sys/sys/socket.h
  projects/nfs-over-tls/sys/sys/socketvar.h
  projects/nfs-over-tls/sys/sys/systm.h
  projects/nfs-over-tls/sys/sys/tree.h
  projects/nfs-over-tls/sys/sys/uio.h
  projects/nfs-over-tls/sys/ufs/ffs/ffs_alloc.c
  projects/nfs-over-tls/sys/vm/uma_core.c
  projects/nfs-over-tls/sys/vm/vm_fault.c
  projects/nfs-over-tls/sys/vm/vm_radix.c
  projects/nfs-over-tls/sys/x86/acpica/srat.c
  projects/nfs-over-tls/sys/x86/include/x86_var.h
  projects/nfs-over-tls/sys/x86/x86/cpu_machdep.c
Directory Properties:
  projects/nfs-over-tls/sys/   (props changed)
  projects/nfs-over-tls/sys/cddl/contrib/opensolaris/   (props changed)
  projects/nfs-over-tls/sys/contrib/dev/acpica/   (props changed)

Modified: projects/nfs-over-tls/sys/amd64/amd64/cpu_switch.S
==============================================================================

--- projects/nfs-over-tls/sys/amd64/amd64/cpu_switch.S	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/amd64/cpu_switch.S	Sat May 23 15:49:07 2020	(r361416)
@@ -221,6 +221,8 @@ done_load_dr:
 	movq	%rax,(%rsp)
 	movq	PCPU(CURTHREAD),%rdi
 	call	fpu_activate_sw
+	cmpb	$0,cpu_flush_rsb_ctxsw(%rip)
+	jne	rsb_flush
 	ret
 
 	/*

Modified: projects/nfs-over-tls/sys/amd64/amd64/initcpu.c
==============================================================================
--- projects/nfs-over-tls/sys/amd64/amd64/initcpu.c	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/amd64/initcpu.c	Sat May 23 15:49:07 2020	(r361416)
@@ -238,12 +238,24 @@ initializecpu(void)
 		cr4 |= CR4_PKE;
 
 	/*
+	 * If SMEP is present, we only need to flush RSB (by default)
+	 * on context switches, to prevent cross-process ret2spec
+	 * attacks.  Do it automatically if ibrs_disable is set, to
+	 * complete the mitigation.
+	 *
 	 * Postpone enabling the SMEP on the boot CPU until the page
 	 * tables are switched from the boot loader identity mapping
 	 * to the kernel tables.  The boot loader enables the U bit in
 	 * its tables.
 	 */
-	if (!IS_BSP()) {
+	if (IS_BSP()) {
+		if (cpu_stdext_feature & CPUID_STDEXT_SMEP &&
+		    !TUNABLE_INT_FETCH(
+		    "machdep.mitigations.cpu_flush_rsb_ctxsw",
+		    &cpu_flush_rsb_ctxsw) &&
+		    hw_ibrs_disable)
+			cpu_flush_rsb_ctxsw = 1;
+	} else {
 		if (cpu_stdext_feature & CPUID_STDEXT_SMEP)
 			cr4 |= CR4_SMEP;
 		if (cpu_stdext_feature & CPUID_STDEXT_SMAP)

Modified: projects/nfs-over-tls/sys/amd64/amd64/machdep.c
==============================================================================
--- projects/nfs-over-tls/sys/amd64/amd64/machdep.c	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/amd64/machdep.c	Sat May 23 15:49:07 2020	(r361416)
@@ -1857,7 +1857,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	setidt(IDT_IO_INTS + 15, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
 #endif
 #else
-#error "have you forgotten the isa device?";
+#error "have you forgotten the isa device?"
 #endif
 
 	if (late_console)
@@ -1867,12 +1867,13 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	fpuinit();
 
 	/*
-	 * Set up thread0 pcb save area after fpuinit calculated fpu save
-	 * area size.  Zero out the extended state header in fpu save
-	 * area.
+	 * Reinitialize thread0's stack base now that the xsave area size is
+	 * known.  Set up thread0's pcb save area after fpuinit calculated fpu
+	 * save area size.  Zero out the extended state header in fpu save area.
 	 */
+	set_top_of_stack_td(&thread0);
 	thread0.td_pcb->pcb_save = get_pcb_user_save_td(&thread0);
-	bzero(get_pcb_user_save_td(&thread0), cpu_max_ext_state_size);
+	bzero(thread0.td_pcb->pcb_save, cpu_max_ext_state_size);
 	if (use_xsave) {
 		xhdr = (struct xstate_hdr *)(get_pcb_user_save_td(&thread0) +
 		    1);
@@ -1882,7 +1883,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	rsp0 = thread0.td_md.md_stack_base;
 	/* Ensure the stack is aligned to 16 bytes */
 	rsp0 &= ~0xFul;
-	__pcpu[0].pc_common_tss.tss_rsp0 = rsp0;
+	PCPU_PTR(common_tss)->tss_rsp0 = rsp0;
 	amd64_bsp_pcpu_init2(rsp0);
 
 	/* transfer to user mode */

Modified: projects/nfs-over-tls/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- projects/nfs-over-tls/sys/amd64/amd64/mp_machdep.c	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/amd64/mp_machdep.c	Sat May 23 15:49:07 2020	(r361416)
@@ -29,6 +29,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_acpi.h"
 #include "opt_cpu.h"
 #include "opt_ddb.h"
 #include "opt_kstack_pages.h"
@@ -78,8 +79,10 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpu.h>
 #include <x86/init.h>
 
+#ifdef DEV_ACPI
 #include <contrib/dev/acpica/include/acpi.h>
 #include <dev/acpica/acpivar.h>
+#endif
 
 #define WARMBOOT_TARGET		0
 #define WARMBOOT_OFF		(KERNBASE + 0x0467)
@@ -264,8 +267,11 @@ cpu_mp_start(void)
 	init_ops.start_all_aps();
 
 	set_interrupt_apic_ids();
-}
 
+#if defined(DEV_ACPI) && MAXMEMDOM > 1
+	acpi_pxm_set_cpu_locality();
+#endif
+}
 
 /*
  * AP CPU's call this to initialize themselves.

Modified: projects/nfs-over-tls/sys/amd64/amd64/pmap.c
==============================================================================
--- projects/nfs-over-tls/sys/amd64/amd64/pmap.c	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/amd64/pmap.c	Sat May 23 15:49:07 2020	(r361416)
@@ -323,12 +323,12 @@ pmap_pku_mask_bit(pmap_t pmap)
 #endif
 
 #undef pa_index
+#ifdef NUMA
 #define	pa_index(pa)	({					\
 	KASSERT((pa) <= vm_phys_segs[vm_phys_nsegs - 1].end,	\
 	    ("address %lx beyond the last segment", (pa)));	\
 	(pa) >> PDRSHIFT;					\
 })
-#ifdef NUMA
 #define	pa_to_pmdp(pa)	(&pv_table[pa_index(pa)])
 #define	pa_to_pvh(pa)	(&(pa_to_pmdp(pa)->pv_page))
 #define	PHYS_TO_PV_LIST_LOCK(pa)	({			\
@@ -340,6 +340,7 @@ pmap_pku_mask_bit(pmap_t pmap)
 	_lock;							\
 })
 #else
+#define	pa_index(pa)	((pa) >> PDRSHIFT)
 #define	pa_to_pvh(pa)	(&pv_table[pa_index(pa)])
 
 #define	NPV_LIST_LOCKS	MAXCPU
@@ -1316,6 +1317,8 @@ pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
 {
 	pd_entry_t *pde;
 
+	KASSERT((*pdpe & PG_PS) == 0,
+	    ("%s: pdpe %#lx is a leaf", __func__, *pdpe));
 	pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
 	return (&pde[pmap_pde_index(va)]);
 }
@@ -1340,6 +1343,8 @@ pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
 {
 	pt_entry_t *pte;
 
+	KASSERT((*pde & PG_PS) == 0,
+	    ("%s: pde %#lx is a leaf", __func__, *pde));
 	pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
 	return (&pte[pmap_pte_index(va)]);
 }

Modified: projects/nfs-over-tls/sys/amd64/amd64/support.S
==============================================================================
--- projects/nfs-over-tls/sys/amd64/amd64/support.S	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/amd64/support.S	Sat May 23 15:49:07 2020	(r361416)
@@ -1613,23 +1613,27 @@ ENTRY(pmap_pti_pcid_invlrng)
 	retq
 
 	.altmacro
-	.macro	ibrs_seq_label l
-handle_ibrs_\l:
+	.macro	rsb_seq_label l
+rsb_seq_\l:
 	.endm
-	.macro	ibrs_call_label l
-	call	handle_ibrs_\l
+	.macro	rsb_call_label l
+	call	rsb_seq_\l
 	.endm
-	.macro	ibrs_seq count
+	.macro	rsb_seq count
 	ll=1
 	.rept	\count
-	ibrs_call_label	%(ll)
+	rsb_call_label	%(ll)
 	nop
-	ibrs_seq_label %(ll)
+	rsb_seq_label %(ll)
 	addq	$8,%rsp
 	ll=ll+1
 	.endr
 	.endm
 
+ENTRY(rsb_flush)
+	rsb_seq	32
+	ret
+
 /* all callers already saved %rax, %rdx, and %rcx */
 ENTRY(handle_ibrs_entry)
 	cmpb	$0,hw_ibrs_ibpb_active(%rip)
@@ -1641,8 +1645,7 @@ ENTRY(handle_ibrs_entry)
 	wrmsr
 	movb	$1,PCPU(IBPB_SET)
 	testl	$CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
-	jne	1f
-	ibrs_seq 32
+	je	rsb_flush
 1:	ret
 END(handle_ibrs_entry)
 

Modified: projects/nfs-over-tls/sys/amd64/include/vmm.h
==============================================================================
--- projects/nfs-over-tls/sys/amd64/include/vmm.h	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/include/vmm.h	Sat May 23 15:49:07 2020	(r361416)
@@ -34,6 +34,8 @@
 #include <sys/sdt.h>
 #include <x86/segments.h>
 
+struct vm_snapshot_meta;
+
 #ifdef _KERNEL
 SDT_PROVIDER_DECLARE(vmm);
 #endif
@@ -152,6 +154,7 @@ struct vmspace;
 struct vm_object;
 struct vm_guest_paging;
 struct pmap;
+enum snapshot_req;
 
 struct vm_eventinfo {
 	void	*rptr;		/* rendezvous cookie */
@@ -180,6 +183,10 @@ typedef struct vmspace * (*vmi_vmspace_alloc)(vm_offse
 typedef void	(*vmi_vmspace_free)(struct vmspace *vmspace);
 typedef struct vlapic * (*vmi_vlapic_init)(void *vmi, int vcpu);
 typedef void	(*vmi_vlapic_cleanup)(void *vmi, struct vlapic *vlapic);
+typedef int	(*vmi_snapshot_t)(void *vmi, struct vm_snapshot_meta *meta);
+typedef int	(*vmi_snapshot_vmcx_t)(void *vmi, struct vm_snapshot_meta *meta,
+				       int vcpu);
+typedef int	(*vmi_restore_tsc_t)(void *vmi, int vcpuid, uint64_t now);
 
 struct vmm_ops {
 	vmm_init_func_t		init;		/* module wide initialization */
@@ -199,6 +206,11 @@ struct vmm_ops {
 	vmi_vmspace_free	vmspace_free;
 	vmi_vlapic_init		vlapic_init;
 	vmi_vlapic_cleanup	vlapic_cleanup;
+
+	/* checkpoint operations */
+	vmi_snapshot_t		vmsnapshot;
+	vmi_snapshot_vmcx_t	vmcx_snapshot;
+	vmi_restore_tsc_t	vm_restore_tsc;
 };
 
 extern struct vmm_ops vmm_ops_intel;
@@ -272,7 +284,10 @@ void vm_exit_debug(struct vm *vm, int vcpuid, uint64_t
 void vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip);
 void vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip);
 void vm_exit_reqidle(struct vm *vm, int vcpuid, uint64_t rip);
+int vm_snapshot_req(struct vm *vm, struct vm_snapshot_meta *meta);
+int vm_restore_time(struct vm *vm);
 
+
 #ifdef _SYS__CPUSET_H_
 /*
  * Rendezvous all vcpus specified in 'dest' and execute 'func(arg)'.
@@ -408,6 +423,15 @@ int vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_
 int vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *info);
 
 int vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2);
+
+/*
+ * Function used to keep track of the guest's TSC offset. The
+ * offset is used by the virutalization extensions to provide a consistent
+ * value for the Time Stamp Counter to the guest.
+ *
+ * Return value is 0 on success and non-zero on failure.
+ */
+int vm_set_tsc_offset(struct vm *vm, int vcpu_id, uint64_t offset);
 
 enum vm_reg_name vm_segment_name(int seg_encoding);
 

Modified: projects/nfs-over-tls/sys/amd64/include/vmm_dev.h
==============================================================================
--- projects/nfs-over-tls/sys/amd64/include/vmm_dev.h	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/include/vmm_dev.h	Sat May 23 15:49:07 2020	(r361416)
@@ -31,6 +31,8 @@
 #ifndef	_VMM_DEV_H_
 #define	_VMM_DEV_H_
 
+struct vm_snapshot_meta;
+
 #ifdef _KERNEL
 void	vmmdev_init(void);
 int	vmmdev_cleanup(void);
@@ -54,6 +56,13 @@ struct vm_memseg {
 	char		name[VM_MAX_SUFFIXLEN + 1];
 };
 
+struct vm_memseg_fbsd12 {
+	int		segid;
+	size_t		len;
+	char		name[64];
+};
+_Static_assert(sizeof(struct vm_memseg_fbsd12) == 80, "COMPAT_FREEBSD12 ABI");
+
 struct vm_register {
 	int		cpuid;
 	int		regnum;		/* enum vm_reg_name */
@@ -233,6 +242,15 @@ struct vm_cpu_topology {
 	uint16_t	maxcpus;
 };
 
+struct vm_readwrite_kernemu_device {
+	int		vcpuid;
+	unsigned	access_width : 3;
+	unsigned	_unused : 29;
+	uint64_t	gpa;
+	uint64_t	value;
+};
+_Static_assert(sizeof(struct vm_readwrite_kernemu_device) == 24, "ABI");
+
 enum {
 	/* general routines */
 	IOCNUM_ABIVERS = 0,
@@ -260,6 +278,8 @@ enum {
 	IOCNUM_GET_SEGMENT_DESCRIPTOR = 23,
 	IOCNUM_SET_REGISTER_SET = 24,
 	IOCNUM_GET_REGISTER_SET = 25,
+	IOCNUM_GET_KERNEMU_DEV = 26,
+	IOCNUM_SET_KERNEMU_DEV = 27,
 
 	/* interrupt injection */
 	IOCNUM_GET_INTINFO = 28,
@@ -312,6 +332,11 @@ enum {
 	IOCNUM_RTC_WRITE = 101,
 	IOCNUM_RTC_SETTIME = 102,
 	IOCNUM_RTC_GETTIME = 103,
+
+	/* checkpoint */
+	IOCNUM_SNAPSHOT_REQ = 113,
+
+	IOCNUM_RESTORE_TIME = 115
 };
 
 #define	VM_RUN		\
@@ -320,8 +345,12 @@ enum {
 	_IOW('v', IOCNUM_SUSPEND, struct vm_suspend)
 #define	VM_REINIT	\
 	_IO('v', IOCNUM_REINIT)
+#define	VM_ALLOC_MEMSEG_FBSD12	\
+	_IOW('v', IOCNUM_ALLOC_MEMSEG, struct vm_memseg_fbsd12)
 #define	VM_ALLOC_MEMSEG	\
 	_IOW('v', IOCNUM_ALLOC_MEMSEG, struct vm_memseg)
+#define	VM_GET_MEMSEG_FBSD12	\
+	_IOWR('v', IOCNUM_GET_MEMSEG, struct vm_memseg_fbsd12)
 #define	VM_GET_MEMSEG	\
 	_IOWR('v', IOCNUM_GET_MEMSEG, struct vm_memseg)
 #define	VM_MMAP_MEMSEG	\
@@ -340,6 +369,12 @@ enum {
 	_IOW('v', IOCNUM_SET_REGISTER_SET, struct vm_register_set)
 #define	VM_GET_REGISTER_SET \
 	_IOWR('v', IOCNUM_GET_REGISTER_SET, struct vm_register_set)
+#define	VM_SET_KERNEMU_DEV \
+	_IOW('v', IOCNUM_SET_KERNEMU_DEV, \
+	    struct vm_readwrite_kernemu_device)
+#define	VM_GET_KERNEMU_DEV \
+	_IOWR('v', IOCNUM_GET_KERNEMU_DEV, \
+	    struct vm_readwrite_kernemu_device)
 #define	VM_INJECT_EXCEPTION	\
 	_IOW('v', IOCNUM_INJECT_EXCEPTION, struct vm_exception)
 #define	VM_LAPIC_IRQ 		\
@@ -422,4 +457,8 @@ enum {
 	_IOR('v', IOCNUM_RTC_GETTIME, struct vm_rtc_time)
 #define	VM_RESTART_INSTRUCTION \
 	_IOW('v', IOCNUM_RESTART_INSTRUCTION, int)
+#define VM_SNAPSHOT_REQ \
+	_IOWR('v', IOCNUM_SNAPSHOT_REQ, struct vm_snapshot_meta)
+#define VM_RESTORE_TIME \
+	_IOWR('v', IOCNUM_RESTORE_TIME, int)
 #endif

Copied: projects/nfs-over-tls/sys/amd64/include/vmm_snapshot.h (from r361415, head/sys/amd64/include/vmm_snapshot.h)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ projects/nfs-over-tls/sys/amd64/include/vmm_snapshot.h	Sat May 23 15:49:07 2020	(r361416, copy of r361415, head/sys/amd64/include/vmm_snapshot.h)
@@ -0,0 +1,156 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2016 Flavius Anton
+ * Copyright (c) 2016 Mihai Tiganus
+ * Copyright (c) 2016-2019 Mihai Carabas
+ * Copyright (c) 2017-2019 Darius Mihai
+ * Copyright (c) 2017-2019 Elena Mihailescu
+ * Copyright (c) 2018-2019 Sergiu Weisz
+ * All rights reserved.
+ * The bhyve-snapshot feature was developed under sponsorships
+ * from Matthew Grooms.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_SNAPSHOT_
+#define _VMM_SNAPSHOT_
+
+#include <sys/errno.h>
+#include <sys/types.h>
+#ifndef _KERNEL
+#include <stdbool.h>
+#endif
+
+struct vmctx;
+
+enum snapshot_req {
+	STRUCT_VMX,
+	STRUCT_VIOAPIC,
+	STRUCT_VM,
+	STRUCT_VLAPIC,
+	VM_MEM,
+	STRUCT_VHPET,
+	STRUCT_VMCX,
+	STRUCT_VATPIC,
+	STRUCT_VATPIT,
+	STRUCT_VPMTMR,
+	STRUCT_VRTC,
+};
+
+struct vm_snapshot_buffer {
+	/*
+	 * R/O for device-specific functions;
+	 * written by generic snapshot functions.
+	 */
+	uint8_t *const buf_start;
+	const size_t buf_size;
+
+	/*
+	 * R/W for device-specific functions used to keep track of buffer
+	 * current position and remaining size.
+	 */
+	uint8_t *buf;
+	size_t buf_rem;
+
+	/*
+	 * Length of the snapshot is either determined as (buf_size - buf_rem)
+	 * or (buf - buf_start) -- the second variation returns a signed value
+	 * so it may not be appropriate.
+	 *
+	 * Use vm_get_snapshot_size(meta).
+	 */
+};
+
+enum vm_snapshot_op {
+	VM_SNAPSHOT_SAVE,
+	VM_SNAPSHOT_RESTORE,
+};
+
+struct vm_snapshot_meta {
+	struct vmctx *ctx;
+	void *dev_data;
+	const char *dev_name;      /* identify userspace devices */
+	enum snapshot_req dev_req; /* identify kernel structs */
+
+	struct vm_snapshot_buffer buffer;
+
+	enum vm_snapshot_op op;
+};
+
+
+void vm_snapshot_buf_err(const char *bufname, const enum vm_snapshot_op op);
+int vm_snapshot_buf(volatile void *data, size_t data_size,
+		    struct vm_snapshot_meta *meta);
+size_t vm_get_snapshot_size(struct vm_snapshot_meta *meta);
+int vm_snapshot_guest2host_addr(void **addrp, size_t len, bool restore_null,
+				struct vm_snapshot_meta *meta);
+int vm_snapshot_buf_cmp(volatile void *data, size_t data_size,
+			      struct vm_snapshot_meta *meta);
+
+#define	SNAPSHOT_BUF_OR_LEAVE(DATA, LEN, META, RES, LABEL)			\
+do {										\
+	(RES) = vm_snapshot_buf((DATA), (LEN), (META));				\
+	if ((RES) != 0) {							\
+		vm_snapshot_buf_err(#DATA, (META)->op);				\
+		goto LABEL;							\
+	}									\
+} while (0)
+
+#define	SNAPSHOT_VAR_OR_LEAVE(DATA, META, RES, LABEL)				\
+	SNAPSHOT_BUF_OR_LEAVE(&(DATA), sizeof(DATA), (META), (RES), LABEL)
+
+/*
+ * Address variables are pointers to guest memory.
+ *
+ * When RNULL != 0, do not enforce invalid address checks; instead, make the
+ * pointer NULL at restore time.
+ */
+#define	SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(ADDR, LEN, RNULL, META, RES, LABEL)	\
+do {										\
+	(RES) = vm_snapshot_guest2host_addr((void **)&(ADDR), (LEN), (RNULL),	\
+			(META));					\
+	if ((RES) != 0) {							\
+		if ((RES) == EFAULT)						\
+			fprintf(stderr, "%s: invalid address: %s\r\n",		\
+				__func__, #ADDR);				\
+		goto LABEL;							\
+	}									\
+} while (0)
+
+/* compare the value in the meta buffer with the data */
+#define	SNAPSHOT_BUF_CMP_OR_LEAVE(DATA, LEN, META, RES, LABEL)			\
+do {										\
+	(RES) = vm_snapshot_buf_cmp((DATA), (LEN), (META));			\
+	if ((RES) != 0) {							\
+		vm_snapshot_buf_err(#DATA, (META)->op);				\
+		goto LABEL;							\
+	}									\
+} while (0)
+
+#define	SNAPSHOT_VAR_CMP_OR_LEAVE(DATA, META, RES, LABEL)			\
+	SNAPSHOT_BUF_CMP_OR_LEAVE(&(DATA), sizeof(DATA), (META), (RES), LABEL)
+
+#endif

Modified: projects/nfs-over-tls/sys/amd64/vmm/amd/svm.c
==============================================================================
--- projects/nfs-over-tls/sys/amd64/vmm/amd/svm.c	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/vmm/amd/svm.c	Sat May 23 15:49:07 2020	(r361416)
@@ -29,6 +29,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_bhyve_snapshot.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/smp.h>
@@ -50,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/vmm.h>
 #include <machine/vmm_dev.h>
 #include <machine/vmm_instruction_emul.h>
+#include <machine/vmm_snapshot.h>
 
 #include "vmm_lapic.h"
 #include "vmm_stat.h"
@@ -276,6 +279,25 @@ svm_restore(void)
 	svm_enable(NULL);
 }		
 
+#ifdef BHYVE_SNAPSHOT
+int
+svm_set_tsc_offset(struct svm_softc *sc, int vcpu, uint64_t offset)
+{
+	int error;
+	struct vmcb_ctrl *ctrl;
+
+	ctrl = svm_get_vmcb_ctrl(sc, vcpu);
+	ctrl->tsc_offset = offset;
+
+	svm_set_dirty(sc, vcpu, VMCB_CACHE_I);
+	VCPU_CTR1(sc->vm, vcpu, "tsc offset changed to %#lx", offset);
+
+	error = vm_set_tsc_offset(sc->vm, vcpu, offset);
+
+	return (error);
+}
+#endif
+
 /* Pentium compatible MSRs */
 #define MSR_PENTIUM_START 	0	
 #define MSR_PENTIUM_END 	0x1FFF
@@ -2203,7 +2225,37 @@ svm_setreg(void *arg, int vcpu, int ident, uint64_t va
 	return (EINVAL);
 }
 
+#ifdef BHYVE_SNAPSHOT
 static int
+svm_snapshot_reg(void *arg, int vcpu, int ident,
+		 struct vm_snapshot_meta *meta)
+{
+	int ret;
+	uint64_t val;
+
+	if (meta->op == VM_SNAPSHOT_SAVE) {
+		ret = svm_getreg(arg, vcpu, ident, &val);
+		if (ret != 0)
+			goto done;
+
+		SNAPSHOT_VAR_OR_LEAVE(val, meta, ret, done);
+	} else if (meta->op == VM_SNAPSHOT_RESTORE) {
+		SNAPSHOT_VAR_OR_LEAVE(val, meta, ret, done);
+
+		ret = svm_setreg(arg, vcpu, ident, val);
+		if (ret != 0)
+			goto done;
+	} else {
+		ret = EINVAL;
+		goto done;
+	}
+
+done:
+	return (ret);
+}
+#endif
+
+static int
 svm_setcap(void *arg, int vcpu, int type, int val)
 {
 	struct svm_softc *sc;
@@ -2285,6 +2337,306 @@ svm_vlapic_cleanup(void *arg, struct vlapic *vlapic)
         free(vlapic, M_SVM_VLAPIC);
 }
 
+#ifdef BHYVE_SNAPSHOT
+static int
+svm_snapshot_vmi(void *arg, struct vm_snapshot_meta *meta)
+{
+	/* struct svm_softc is AMD's representation for SVM softc */
+	struct svm_softc *sc;
+	struct svm_vcpu *vcpu;
+	struct vmcb *vmcb;
+	uint64_t val;
+	int i;
+	int ret;
+
+	sc = arg;
+
+	KASSERT(sc != NULL, ("%s: arg was NULL", __func__));
+
+	SNAPSHOT_VAR_OR_LEAVE(sc->nptp, meta, ret, done);
+
+	for (i = 0; i < VM_MAXCPU; i++) {
+		vcpu = &sc->vcpu[i];
+		vmcb = &vcpu->vmcb;
+
+		/* VMCB fields for virtual cpu i */
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->ctrl.v_tpr, meta, ret, done);
+		val = vmcb->ctrl.v_tpr;
+		SNAPSHOT_VAR_OR_LEAVE(val, meta, ret, done);
+		vmcb->ctrl.v_tpr = val;
+
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->ctrl.asid, meta, ret, done);
+		val = vmcb->ctrl.np_enable;
+		SNAPSHOT_VAR_OR_LEAVE(val, meta, ret, done);
+		vmcb->ctrl.np_enable = val;
+
+		val = vmcb->ctrl.intr_shadow;
+		SNAPSHOT_VAR_OR_LEAVE(val, meta, ret, done);
+		vmcb->ctrl.intr_shadow = val;
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->ctrl.tlb_ctrl, meta, ret, done);
+
+		SNAPSHOT_BUF_OR_LEAVE(vmcb->state.pad1,
+				      sizeof(vmcb->state.pad1),
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.cpl, meta, ret, done);
+		SNAPSHOT_BUF_OR_LEAVE(vmcb->state.pad2,
+				      sizeof(vmcb->state.pad2),
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.efer, meta, ret, done);
+		SNAPSHOT_BUF_OR_LEAVE(vmcb->state.pad3,
+				      sizeof(vmcb->state.pad3),
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.cr4, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.cr3, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.cr0, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.dr7, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.dr6, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.rflags, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.rip, meta, ret, done);
+		SNAPSHOT_BUF_OR_LEAVE(vmcb->state.pad4,
+				      sizeof(vmcb->state.pad4),
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.rsp, meta, ret, done);
+		SNAPSHOT_BUF_OR_LEAVE(vmcb->state.pad5,
+				      sizeof(vmcb->state.pad5),
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.rax, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.star, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.lstar, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.cstar, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.sfmask, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.kernelgsbase,
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.sysenter_cs, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.sysenter_esp,
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.sysenter_eip,
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.cr2, meta, ret, done);
+		SNAPSHOT_BUF_OR_LEAVE(vmcb->state.pad6,
+				      sizeof(vmcb->state.pad6),
+				      meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.g_pat, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.dbgctl, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.br_from, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.br_to, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.int_from, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vmcb->state.int_to, meta, ret, done);
+		SNAPSHOT_BUF_OR_LEAVE(vmcb->state.pad7,
+				      sizeof(vmcb->state.pad7),
+				      meta, ret, done);
+
+		/* Snapshot swctx for virtual cpu i */
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_rbp, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_rbx, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_rcx, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_rdx, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_rdi, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_rsi, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r8, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r9, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r10, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r11, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r12, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r13, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r14, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_r15, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_dr0, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_dr1, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_dr2, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.sctx_dr3, meta, ret, done);
+
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.host_dr0, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.host_dr1, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.host_dr2, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.host_dr3, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.host_dr6, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.host_dr7, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->swctx.host_debugctl, meta, ret,
+				      done);
+
+		/* Restore other svm_vcpu struct fields */
+
+		/* Restore NEXTRIP field */
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->nextrip, meta, ret, done);
+
+		/* Restore lastcpu field */
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->lastcpu, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->dirty, meta, ret, done);
+
+		/* Restore EPTGEN field - EPT is Extended Page Tabel */
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->eptgen, meta, ret, done);
+
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->asid.gen, meta, ret, done);
+		SNAPSHOT_VAR_OR_LEAVE(vcpu->asid.num, meta, ret, done);
+
+		/* Set all caches dirty */
+		if (meta->op == VM_SNAPSHOT_RESTORE) {
+			svm_set_dirty(sc, i, VMCB_CACHE_ASID);
+			svm_set_dirty(sc, i, VMCB_CACHE_IOPM);
+			svm_set_dirty(sc, i, VMCB_CACHE_I);
+			svm_set_dirty(sc, i, VMCB_CACHE_TPR);
+			svm_set_dirty(sc, i, VMCB_CACHE_CR2);
+			svm_set_dirty(sc, i, VMCB_CACHE_CR);
+			svm_set_dirty(sc, i, VMCB_CACHE_DT);
+			svm_set_dirty(sc, i, VMCB_CACHE_SEG);
+			svm_set_dirty(sc, i, VMCB_CACHE_NP);
+		}
+	}
+
+	if (meta->op == VM_SNAPSHOT_RESTORE)
+		flush_by_asid();
+
+done:
+	return (ret);
+}
+
+static int
+svm_snapshot_vmcx(void *arg, struct vm_snapshot_meta *meta, int vcpu)
+{
+	struct vmcb *vmcb;
+	struct svm_softc *sc;
+	int err, running, hostcpu;
+
+	sc = (struct svm_softc *)arg;
+	err = 0;
+
+	KASSERT(arg != NULL, ("%s: arg was NULL", __func__));
+	vmcb = svm_get_vmcb(sc, vcpu);
+
+	running = vcpu_is_running(sc->vm, vcpu, &hostcpu);
+	if (running && hostcpu !=curcpu) {
+		printf("%s: %s%d is running", __func__, vm_name(sc->vm), vcpu);
+		return (EINVAL);
+	}
+
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR0, meta);
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR2, meta);
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR3, meta);
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR4, meta);
+
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_DR7, meta);
+
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RAX, meta);
+
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RSP, meta);
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RIP, meta);
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RFLAGS, meta);
+
+	/* Guest segments */
+	/* ES */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_ES, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_ES, meta);
+
+	/* CS */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CS, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_CS, meta);
+
+	/* SS */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_SS, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_SS, meta);
+
+	/* DS */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_DS, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_DS, meta);
+
+	/* FS */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_FS, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_FS, meta);
+
+	/* GS */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_GS, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_GS, meta);
+
+	/* TR */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_TR, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_TR, meta);
+
+	/* LDTR */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_LDTR, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_LDTR, meta);
+
+	/* EFER */
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_EFER, meta);
+
+	/* IDTR and GDTR */
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_IDTR, meta);
+	err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_GDTR, meta);
+
+	/* Specific AMD registers */
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_SYSENTER_CS, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_SYSENTER_ESP, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_SYSENTER_EIP, 8), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_NPT_BASE, 8), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_CR_INTERCEPT, 4), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_DR_INTERCEPT, 4), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_EXC_INTERCEPT, 4), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_INST1_INTERCEPT, 4), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_INST2_INTERCEPT, 4), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_TLB_CTRL, 4), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_EXITINFO1, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_EXITINFO2, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_EXITINTINFO, 8), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_VIRQ, 8), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_GUEST_PAT, 8), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_AVIC_BAR, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_AVIC_PAGE, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_AVIC_LT, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_AVIC_PT, 8), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_IO_PERM, 8), meta);
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_MSR_PERM, 8), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_ASID, 4), meta);
+
+	err += vmcb_snapshot_any(sc, vcpu,
+				VMCB_ACCESS(VMCB_OFF_EXIT_REASON, 8), meta);
+
+	err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_INTR_SHADOW, meta);
+
+	return (err);
+}
+
+static int
+svm_restore_tsc(void *arg, int vcpu, uint64_t offset)
+{
+	int err;
+
+	err = svm_set_tsc_offset(arg, vcpu, offset);
+
+	return (err);
+}
+#endif
+
 struct vmm_ops vmm_ops_amd = {
 	.init		= svm_init,
 	.cleanup	= svm_cleanup,
@@ -2302,4 +2654,9 @@ struct vmm_ops vmm_ops_amd = {
 	.vmspace_free	= svm_npt_free,
 	.vlapic_init	= svm_vlapic_init,
 	.vlapic_cleanup	= svm_vlapic_cleanup,
+#ifdef BHYVE_SNAPSHOT
+	.vmsnapshot	= svm_snapshot_vmi,
+	.vmcx_snapshot	= svm_snapshot_vmcx,
+	.vm_restore_tsc	= svm_restore_tsc,
+#endif
 };

Modified: projects/nfs-over-tls/sys/amd64/vmm/amd/svm.h
==============================================================================
--- projects/nfs-over-tls/sys/amd64/vmm/amd/svm.h	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/vmm/amd/svm.h	Sat May 23 15:49:07 2020	(r361416)
@@ -32,6 +32,7 @@
 #define _SVM_H_
 
 struct pcpu;
+struct svm_softc;
 
 /*
  * Guest register state that is saved outside the VMCB.
@@ -66,5 +67,8 @@ struct svm_regctx {
 };
 
 void svm_launch(uint64_t pa, struct svm_regctx *gctx, struct pcpu *pcpu);
+#ifdef BHYVE_SNAPSHOT
+int  svm_set_tsc_offset(struct svm_softc *sc, int vcpu, uint64_t offset);
+#endif
 
 #endif /* _SVM_H_ */

Modified: projects/nfs-over-tls/sys/amd64/vmm/amd/svm_msr.c
==============================================================================
--- projects/nfs-over-tls/sys/amd64/vmm/amd/svm_msr.c	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/vmm/amd/svm_msr.c	Sat May 23 15:49:07 2020	(r361416)
@@ -29,6 +29,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_bhyve_snapshot.h"
+
 #include <sys/param.h>
 #include <sys/errno.h>
 #include <sys/systm.h>
@@ -162,6 +164,11 @@ svm_wrmsr(struct svm_softc *sc, int vcpu, u_int num, u
 		 * Ignore writes to microcode update register.
 		 */
 		break;
+#ifdef BHYVE_SNAPSHOT
+	case MSR_TSC:
+		error = svm_set_tsc_offset(sc, vcpu, val - rdtsc());
+		break;
+#endif
 	case MSR_EXTFEATURES:
 		break;
 	default:

Modified: projects/nfs-over-tls/sys/amd64/vmm/amd/vmcb.c
==============================================================================
--- projects/nfs-over-tls/sys/amd64/vmm/amd/vmcb.c	Sat May 23 12:15:47 2020	(r361415)
+++ projects/nfs-over-tls/sys/amd64/vmm/amd/vmcb.c	Sat May 23 15:49:07 2020	(r361416)
@@ -29,12 +29,15 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_bhyve_snapshot.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 
 #include <machine/segments.h>
 #include <machine/specialreg.h>
 #include <machine/vmm.h>
+#include <machine/vmm_snapshot.h>
 
 #include "vmm_ktr.h"
 
@@ -452,3 +455,106 @@ vmcb_getdesc(void *arg, int vcpu, int reg, struct seg_
 
 	return (0);
 }
+
+#ifdef BHYVE_SNAPSHOT
+int
+vmcb_getany(struct svm_softc *sc, int vcpu, int ident, uint64_t *val)
+{
+	int error = 0;
+
+	if (vcpu < 0 || vcpu >= VM_MAXCPU) {
+		error = EINVAL;
+		goto err;
+	}

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
