svn commit: r363522 - in head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs: . sys

Mateusz Guzik mjg at FreeBSD.org
Sat Jul 25 10:39:42 UTC 2020


Author: mjg
Date: Sat Jul 25 10:39:41 2020
New Revision: 363522
URL: https://svnweb.freebsd.org/changeset/base/363522

Log:
  zfs: add support for lockless lookup
  
  Tested by:	pho (in a patchset, previous version)
  Differential Revision:	https://reviews.freebsd.org/D25581

Modified:
  head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_znode.h
  head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c
  head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
  head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
  head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c

Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_znode.h
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_znode.h	Sat Jul 25 10:38:44 2020	(r363521)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_znode.h	Sat Jul 25 10:39:41 2020	(r363522)
@@ -246,6 +246,8 @@ VTOZ(vnode_t *vp)
 #define	VTOZ(VP)	((znode_t *)(VP)->v_data)
 #endif
 
+#define	VTOZ_SMR(VP)	((znode_t *)vn_load_v_data_smr(VP))
+
 /* Called on entry to each ZFS vnode and vfs operation  */
 #define	ZFS_ENTER(zfsvfs) \
 	{ \

Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c	Sat Jul 25 10:38:44 2020	(r363521)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c	Sat Jul 25 10:39:41 2020	(r363522)
@@ -1146,6 +1146,7 @@ zfs_acl_chown_setattr(znode_t *zp)
 
 	ASSERT_VOP_ELOCKED(ZTOV(zp), __func__);
 	ASSERT(MUTEX_HELD(&zp->z_acl_lock));
+	ASSERT_VOP_IN_SEQC(ZTOV(zp));
 
 	if ((error = zfs_acl_node_read(zp, &aclp, B_FALSE)) == 0)
 		zp->z_mode = zfs_mode_compute(zp->z_mode, aclp,
@@ -1172,6 +1173,8 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t
 	uint64_t		ctime[2];
 	int			count = 0;
 	zfs_acl_phys_t		acl_phys;
+
+	ASSERT_VOP_IN_SEQC(ZTOV(zp));
 
 	mode = zp->z_mode;
 

Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c	Sat Jul 25 10:38:44 2020	(r363521)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c	Sat Jul 25 10:39:41 2020	(r363522)
@@ -1391,6 +1391,9 @@ zfs_domount(vfs_t *vfsp, char *osname)
 
 	vfsp->vfs_data = zfsvfs;
 	vfsp->mnt_flag |= MNT_LOCAL;
+#if defined(_KERNEL) && !defined(KMEM_DEBUG)
+	vfsp->mnt_kern_flag |= MNTK_FPLOOKUP;
+#endif
 	vfsp->mnt_kern_flag |= MNTK_LOOKUP_SHARED;
 	vfsp->mnt_kern_flag |= MNTK_SHARED_WRITES;
 	vfsp->mnt_kern_flag |= MNTK_EXTENDED_SHARED;

Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c	Sat Jul 25 10:38:44 2020	(r363521)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c	Sat Jul 25 10:39:41 2020	(r363522)
@@ -38,6 +38,7 @@
 #include <sys/vfs.h>
 #include <sys/vm.h>
 #include <sys/vnode.h>
+#include <sys/smr.h>
 #include <sys/file.h>
 #include <sys/stat.h>
 #include <sys/kmem.h>
@@ -78,6 +79,8 @@
 #include <vm/vm_param.h>
 #include <sys/zil.h>
 
+VFS_SMR_DECLARE;
+
 /*
  * Programming rules.
  *
@@ -3698,6 +3701,7 @@ zfs_rename(vnode_t *sdvp, vnode_t **svpp, struct compo
 	char		*snm = scnp->cn_nameptr;
 	char		*tnm = tcnp->cn_nameptr;
 	int		error = 0;
+	bool		want_seqc_end = false;
 
 	/* Reject renames across filesystems. */
 	if ((*svpp)->v_mount != tdvp->v_mount ||
@@ -3828,6 +3832,14 @@ zfs_rename(vnode_t *sdvp, vnode_t **svpp, struct compo
 		}
 	}
 
+	vn_seqc_write_begin(*svpp);
+	vn_seqc_write_begin(sdvp);
+	if (*tvpp != NULL)
+		vn_seqc_write_begin(*tvpp);
+	if (tdvp != *tvpp)
+		vn_seqc_write_begin(tdvp);
+	want_seqc_end = true;
+
 	vnevent_rename_src(*svpp, sdvp, scnp->cn_nameptr, ct);
 	if (tzp)
 		vnevent_rename_dest(*tvpp, tdvp, tnm, ct);
@@ -3914,10 +3926,20 @@ zfs_rename(vnode_t *sdvp, vnode_t **svpp, struct compo
 
 unlockout:			/* all 4 vnodes are locked, ZFS_ENTER called */
 	ZFS_EXIT(zfsvfs);
+	if (want_seqc_end) {
+		vn_seqc_write_end(*svpp);
+		vn_seqc_write_end(sdvp);
+		if (*tvpp != NULL)
+			vn_seqc_write_end(*tvpp);
+		if (tdvp != *tvpp)
+			vn_seqc_write_end(tdvp);
+		want_seqc_end = false;
+	}
 	VOP_UNLOCK(*svpp);
 	VOP_UNLOCK(sdvp);
 
 out:				/* original two vnodes are locked */
+	MPASS(!want_seqc_end);
 	if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
 		zil_commit(zilog, 0);
 
@@ -4861,7 +4883,32 @@ zfs_freebsd_write(ap)
 	    ap->a_cred, NULL));
 }
 
+/*
+ * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see
+ * the comment above cache_fplookup for details.
+ */
 static int
+zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v)
+{
+	vnode_t *vp;
+	znode_t *zp;
+	uint64_t pflags;
+
+	vp = v->a_vp;
+	zp = VTOZ_SMR(vp);
+	if (__predict_false(zp == NULL))
+		return (EAGAIN);
+	pflags = atomic_load_64(&zp->z_pflags);
+	if (pflags & ZFS_AV_QUARANTINED)
+		return (EAGAIN);
+	if (pflags & ZFS_XATTR)
+		return (EAGAIN);
+	if ((pflags & ZFS_NO_EXECS_DENIED) == 0)
+		return (EAGAIN);
+	return (0);
+}
+
+static int
 zfs_freebsd_access(ap)
 	struct vop_access_args /* {
 		struct vnode *a_vp;
@@ -5998,6 +6045,7 @@ struct vop_vector zfs_vnodeops = {
 	.vop_inactive =		zfs_freebsd_inactive,
 	.vop_need_inactive =	zfs_freebsd_need_inactive,
 	.vop_reclaim =		zfs_freebsd_reclaim,
+	.vop_fplookup_vexec =	zfs_freebsd_fplookup_vexec,
 	.vop_access =		zfs_freebsd_access,
 	.vop_allocate =		VOP_EINVAL,
 	.vop_lookup =		zfs_cache_lookup,
@@ -6066,6 +6114,7 @@ VFS_VOP_VECTOR_REGISTER(zfs_fifoops);
  */
 struct vop_vector zfs_shareops = {
 	.vop_default =		&default_vnodeops,
+	.vop_fplookup_vexec =	zfs_freebsd_fplookup_vexec,
 	.vop_access =		zfs_freebsd_access,
 	.vop_inactive =		zfs_freebsd_inactive,
 	.vop_reclaim =		zfs_freebsd_reclaim,

Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c	Sat Jul 25 10:38:44 2020	(r363521)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c	Sat Jul 25 10:39:41 2020	(r363522)
@@ -99,7 +99,12 @@ SYSCTL_INT(_debug_sizeof, OID_AUTO, znode, CTLFLAG_RD,
  */
 krwlock_t zfsvfs_lock;
 
+#if defined(_KERNEL) && !defined(KMEM_DEBUG)
+#define _ZFS_USE_SMR
+static uma_zone_t znode_uma_zone;
+#else
 static kmem_cache_t *znode_cache = NULL;
+#endif
 
 /*ARGSUSED*/
 static void
@@ -366,6 +371,23 @@ zfs_znode_move(void *buf, void *newbuf, size_t size, v
 }
 #endif /* illumos */
 
+#ifdef _ZFS_USE_SMR
+VFS_SMR_DECLARE;
+
+static int
+zfs_znode_cache_constructor_smr(void *mem, int size __unused, void *private, int flags)
+{
+
+	return (zfs_znode_cache_constructor(mem, private, flags));
+}
+
+static void
+zfs_znode_cache_destructor_smr(void *mem, int size __unused, void *private)
+{
+
+	zfs_znode_cache_destructor(mem, private);
+}
+
 void
 zfs_znode_init(void)
 {
@@ -373,6 +395,34 @@ zfs_znode_init(void)
 	 * Initialize zcache
 	 */
 	rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL);
+	ASSERT(znode_uma_zone == NULL);
+	znode_uma_zone = uma_zcreate("zfs_znode_cache",
+	    sizeof (znode_t), zfs_znode_cache_constructor_smr,
+	    zfs_znode_cache_destructor_smr, NULL, NULL, 0, 0);
+	VFS_SMR_ZONE_SET(znode_uma_zone);
+}
+
+static znode_t *
+zfs_znode_alloc_kmem(int flags)
+{
+
+	return (uma_zalloc_smr(znode_uma_zone, flags));
+}
+
+static void
+zfs_znode_free_kmem(znode_t *zp)
+{
+
+	uma_zfree_smr(znode_uma_zone, zp);
+}
+#else
+void
+zfs_znode_init(void)
+{
+	/*
+	 * Initialize zcache
+	 */
+	rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL);
 	ASSERT(znode_cache == NULL);
 	znode_cache = kmem_cache_create("zfs_znode_cache",
 	    sizeof (znode_t), 0, zfs_znode_cache_constructor,
@@ -380,6 +430,21 @@ zfs_znode_init(void)
 	kmem_cache_set_move(znode_cache, zfs_znode_move);
 }
 
+static znode_t *
+zfs_znode_alloc_kmem(int flags)
+{
+
+	return (kmem_cache_alloc(znode_cache, flags));
+}
+
+static void
+zfs_znode_free_kmem(znode_t *zp)
+{
+
+	kmem_cache_free(znode_cache, zp);
+}
+#endif
+
 void
 zfs_znode_fini(void)
 {
@@ -393,9 +458,17 @@ zfs_znode_fini(void)
 	/*
 	 * Cleanup zcache
 	 */
-	if (znode_cache)
+#ifdef _ZFS_USE_SMR
+	if (znode_uma_zone) {
+		uma_zdestroy(znode_uma_zone);
+		znode_uma_zone = NULL;
+	}
+#else
+	if (znode_cache) {
 		kmem_cache_destroy(znode_cache);
-	znode_cache = NULL;
+		znode_cache = NULL;
+	}
+#endif
 	rw_destroy(&zfsvfs_lock);
 }
 
@@ -508,7 +581,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
 	vattr.va_uid = crgetuid(kcred);
 	vattr.va_gid = crgetgid(kcred);
 
-	sharezp = kmem_cache_alloc(znode_cache, KM_SLEEP);
+	sharezp = zfs_znode_alloc_kmem(KM_SLEEP);
 	ASSERT(!POINTER_IS_VALID(sharezp->z_zfsvfs));
 	sharezp->z_moved = 0;
 	sharezp->z_unlinked = 0;
@@ -527,7 +600,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
 
 	zfs_acl_ids_free(&acl_ids);
 	sa_handle_destroy(sharezp->z_sa_hdl);
-	kmem_cache_free(znode_cache, sharezp);
+	zfs_znode_free_kmem(sharezp);
 
 	return (error);
 }
@@ -642,13 +715,18 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int b
 	int count = 0;
 	int error;
 
-	zp = kmem_cache_alloc(znode_cache, KM_SLEEP);
+	zp = zfs_znode_alloc_kmem(KM_SLEEP);
 
+#ifndef _ZFS_USE_SMR
+	KASSERT((zfsvfs->z_parent->z_vfs->mnt_kern_flag & MNTK_FPLOOKUP) == 0,
+	    ("%s: fast path lookup enabled without smr", __func__));
+#endif
+
 	KASSERT(curthread->td_vp_reserved != NULL,
 	    ("zfs_znode_alloc: getnewvnode without preallocated vnode"));
 	error = getnewvnode("zfs", zfsvfs->z_parent->z_vfs, &zfs_vnodeops, &vp);
 	if (error != 0) {
-		kmem_cache_free(znode_cache, zp);
+		zfs_znode_free_kmem(zp);
 		return (NULL);
 	}
 	zp->z_vnode = vp;
@@ -695,7 +773,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int b
 			sa_handle_destroy(zp->z_sa_hdl);
 		zfs_vnode_forget(vp);
 		zp->z_vnode = NULL;
-		kmem_cache_free(znode_cache, zp);
+		zfs_znode_free_kmem(zp);
 		return (NULL);
 	}
 
@@ -1061,6 +1139,8 @@ zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *
 	xoap = xva_getxoptattr(xvap);
 	ASSERT(xoap);
 
+	ASSERT_VOP_IN_SEQC(ZTOV(zp));
+
 	if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) {
 		uint64_t times[2];
 		ZFS_TIME_ENCODE(&xoap->xoa_createtime, times);
@@ -1490,7 +1570,7 @@ zfs_znode_free(znode_t *zp)
 		zp->z_acl_cached = NULL;
 	}
 
-	kmem_cache_free(znode_cache, zp);
+	zfs_znode_free_kmem(zp);
 
 #ifdef illumos
 	VFS_RELE(zfsvfs->z_vfs);
@@ -1950,7 +2030,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplp
 
 	zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP);
 
-	rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP);
+	rootzp = zfs_znode_alloc_kmem(KM_SLEEP);
 	ASSERT(!POINTER_IS_VALID(rootzp->z_zfsvfs));
 	rootzp->z_moved = 0;
 	rootzp->z_unlinked = 0;
@@ -1994,7 +2074,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplp
 	POINTER_INVALIDATE(&rootzp->z_zfsvfs);
 
 	sa_handle_destroy(rootzp->z_sa_hdl);
-	kmem_cache_free(znode_cache, rootzp);
+	zfs_znode_free_kmem(rootzp);
 
 	/*
 	 * Create shares directory


More information about the svn-src-head mailing list