svn commit: r355633 - in head/sys: cddl/contrib/opensolaris/uts/common/fs/zfs fs/tmpfs kern sys

Mateusz Guzik mjg at FreeBSD.org
Wed Dec 11 23:11:23 UTC 2019


Author: mjg
Date: Wed Dec 11 23:11:21 2019
New Revision: 355633
URL: https://svnweb.freebsd.org/changeset/base/355633

Log:
  vfs: locking primitives which elide ->v_vnlock and shared locking disablement
  
  Both of these features are not needed by many consumers and result in avoidable
  reads which in turn puts them on profiles due to cache-line ping ponging.
  
  On top of that the current lockgmr entry point is slower than necessary
  single-threaded. As an attempted clean up preparing for other changes,
  provide new routines which don't support any of the aforementioned features.
  
  With these patches in place vop_stdlock and vop_stdunlock disappear from
  flamegraphs during -j 104 buildkernel.
  
  Reviewed by:	jeff (previous version)
  Tested by:	pho
  Differential Revision:	https://reviews.freebsd.org/D22665

Modified:
  head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
  head/sys/fs/tmpfs/tmpfs_subr.c
  head/sys/fs/tmpfs/tmpfs_vnops.c
  head/sys/kern/kern_lock.c
  head/sys/kern/vfs_default.c
  head/sys/sys/lockmgr.h
  head/sys/sys/vnode.h

Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c	Wed Dec 11 23:09:12 2019	(r355632)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c	Wed Dec 11 23:11:21 2019	(r355633)
@@ -5932,7 +5932,7 @@ zfs_lock(ap)
 	znode_t *zp;
 	int err;
 
-	err = vop_stdlock(ap);
+	err = vop_lock(ap);
 	if (err == 0 && (ap->a_flags & LK_NOWAIT) == 0) {
 		vp = ap->a_vp;
 		zp = vp->v_data;
@@ -5989,7 +5989,11 @@ struct vop_vector zfs_vnodeops = {
 	.vop_vptocnp =		zfs_vptocnp,
 #ifdef DIAGNOSTIC
 	.vop_lock1 =		zfs_lock,
+#else
+	.vop_lock1 =		vop_lock,
 #endif
+	.vop_unlock =		vop_unlock,
+	.vop_islocked =		vop_islocked,
 };
 
 struct vop_vector zfs_fifoops = {

Modified: head/sys/fs/tmpfs/tmpfs_subr.c
==============================================================================
--- head/sys/fs/tmpfs/tmpfs_subr.c	Wed Dec 11 23:09:12 2019	(r355632)
+++ head/sys/fs/tmpfs/tmpfs_subr.c	Wed Dec 11 23:11:21 2019	(r355633)
@@ -671,7 +671,7 @@ loop:
 	MPASS(vp != NULL);
 
 	/* lkflag is ignored, the lock is exclusive */
-	(void) vn_lock(vp, lkflag | LK_RETRY);
+	(void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 
 	vp->v_data = node;
 	vp->v_type = node->tn_type;

Modified: head/sys/fs/tmpfs/tmpfs_vnops.c
==============================================================================
--- head/sys/fs/tmpfs/tmpfs_vnops.c	Wed Dec 11 23:09:12 2019	(r355632)
+++ head/sys/fs/tmpfs/tmpfs_vnops.c	Wed Dec 11 23:11:21 2019	(r355633)
@@ -1632,6 +1632,9 @@ struct vop_vector tmpfs_vnodeop_entries = {
 	.vop_whiteout =			tmpfs_whiteout,
 	.vop_bmap =			VOP_EOPNOTSUPP,
 	.vop_vptocnp =			tmpfs_vptocnp,
+	.vop_lock1 =			vop_lock,
+	.vop_unlock = 			vop_unlock,
+	.vop_islocked = 		vop_islocked,
 };
 
 /*

Modified: head/sys/kern/kern_lock.c
==============================================================================
--- head/sys/kern/kern_lock.c	Wed Dec 11 23:09:12 2019	(r355632)
+++ head/sys/kern/kern_lock.c	Wed Dec 11 23:11:21 2019	(r355633)
@@ -1156,6 +1156,88 @@ lockmgr_unlock_fast_path(struct lock *lk, u_int flags,
 	return (0);
 }
 
+/*
+ * Lightweight entry points for common operations.
+ *
+ * Functionality is similar to sx locks, in that none of the additional lockmgr
+ * features are supported. To be clear, these are NOT supported:
+ * 1. shared locking disablement
+ * 2. returning with an error after sleep
+ * 3. unlocking the interlock
+ *
+ * If in doubt, use lockmgr_*_fast_path.
+ */
+int
+lockmgr_slock(struct lock *lk, u_int flags, const char *file, int line)
+{
+	uintptr_t x;
+
+	MPASS((flags & LK_TYPE_MASK) == LK_SHARED);
+	MPASS((flags & LK_INTERLOCK) == 0);
+	MPASS((lk->lock_object.lo_flags & LK_NOSHARE) == 0);
+
+	if (LK_CAN_WITNESS(flags))
+		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER,
+		    file, line, NULL);
+	if (__predict_true(lockmgr_slock_try(lk, &x, flags, true))) {
+		lockmgr_note_shared_acquire(lk, 0, 0, file, line, flags);
+		return (0);
+	}
+
+	return (lockmgr_slock_hard(lk, flags, NULL, file, line, NULL));
+}
+
+int
+lockmgr_xlock(struct lock *lk, u_int flags, const char *file, int line)
+{
+	uintptr_t tid;
+
+	MPASS((flags & LK_TYPE_MASK) == LK_EXCLUSIVE);
+	MPASS((flags & LK_INTERLOCK) == 0);
+
+	if (LK_CAN_WITNESS(flags))
+		WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER |
+		    LOP_EXCLUSIVE, file, line, NULL);
+	tid = (uintptr_t)curthread;
+	if (atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
+		lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
+		    flags);
+		return (0);
+	}
+
+	return (lockmgr_xlock_hard(lk, flags, NULL, file, line, NULL));
+}
+
+int
+lockmgr_unlock(struct lock *lk)
+{
+	uintptr_t x, tid;
+	const char *file;
+	int line;
+
+	file = __FILE__;
+	line = __LINE__;
+
+	_lockmgr_assert(lk, KA_LOCKED, file, line);
+	x = lk->lk_lock;
+	if (__predict_true(x & LK_SHARE) != 0) {
+		if (lockmgr_sunlock_try(lk, &x)) {
+			lockmgr_note_shared_release(lk, file, line);
+		} else {
+			return (lockmgr_sunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
+		}
+	} else {
+		tid = (uintptr_t)curthread;
+		if (!lockmgr_recursed(lk) &&
+		    atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
+			lockmgr_note_exclusive_release(lk, file, line);
+		} else {
+			return (lockmgr_xunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
+		}
+	}
+	return (0);
+}
+
 int
 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
     const char *wmesg, int pri, int timo, const char *file, int line)

Modified: head/sys/kern/vfs_default.c
==============================================================================
--- head/sys/kern/vfs_default.c	Wed Dec 11 23:09:12 2019	(r355632)
+++ head/sys/kern/vfs_default.c	Wed Dec 11 23:11:21 2019	(r355633)
@@ -545,6 +545,71 @@ vop_stdislocked(ap)
 }
 
 /*
+ * Variants of the above set.
+ *
+ * Differences are:
+ * - shared locking disablement is not supported
+ * - v_vnlock pointer is not honored
+ */
+int
+vop_lock(ap)
+	struct vop_lock1_args /* {
+		struct vnode *a_vp;
+		int a_flags;
+		char *file;
+		int line;
+	} */ *ap;
+{
+	struct vnode *vp = ap->a_vp;
+	int flags = ap->a_flags;
+	struct mtx *ilk;
+
+	MPASS(vp->v_vnlock == &vp->v_lock);
+
+	if (__predict_false((flags & ~(LK_TYPE_MASK | LK_NODDLKTREAT | LK_RETRY)) != 0))
+		goto other;
+
+	switch (flags & LK_TYPE_MASK) {
+	case LK_SHARED:
+		return (lockmgr_slock(&vp->v_lock, flags, ap->a_file, ap->a_line));
+	case LK_EXCLUSIVE:
+		return (lockmgr_xlock(&vp->v_lock, flags, ap->a_file, ap->a_line));
+	}
+other:
+	ilk = VI_MTX(vp);
+	return (lockmgr_lock_fast_path(&vp->v_lock, flags,
+	    &ilk->lock_object, ap->a_file, ap->a_line));
+}
+
+int
+vop_unlock(ap)
+	struct vop_unlock_args /* {
+		struct vnode *a_vp;
+		int a_flags;
+	} */ *ap;
+{
+	struct vnode *vp = ap->a_vp;
+
+	MPASS(vp->v_vnlock == &vp->v_lock);
+	MPASS(ap->a_flags == 0);
+
+	return (lockmgr_unlock(&vp->v_lock));
+}
+
+int
+vop_islocked(ap)
+	struct vop_islocked_args /* {
+		struct vnode *a_vp;
+	} */ *ap;
+{
+	struct vnode *vp = ap->a_vp;
+
+	MPASS(vp->v_vnlock == &vp->v_lock);
+
+	return (lockstatus(&vp->v_lock));
+}
+
+/*
  * Return true for select/poll.
  */
 int

Modified: head/sys/sys/lockmgr.h
==============================================================================
--- head/sys/sys/lockmgr.h	Wed Dec 11 23:09:12 2019	(r355632)
+++ head/sys/sys/lockmgr.h	Wed Dec 11 23:11:21 2019	(r355633)
@@ -74,6 +74,10 @@ int	 lockmgr_lock_fast_path(struct lock *lk, u_int fla
 	    struct lock_object *ilk, const char *file, int line);
 int	 lockmgr_unlock_fast_path(struct lock *lk, u_int flags,
 	    struct lock_object *ilk);
+int	lockmgr_slock(struct lock *lk, u_int flags, const char *file, int line);
+int	lockmgr_xlock(struct lock *lk, u_int flags, const char *file, int line);
+int	lockmgr_unlock(struct lock *lk);
+
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
 void	 _lockmgr_assert(const struct lock *lk, int what, const char *file, int line);
 #endif

Modified: head/sys/sys/vnode.h
==============================================================================
--- head/sys/sys/vnode.h	Wed Dec 11 23:09:12 2019	(r355632)
+++ head/sys/sys/vnode.h	Wed Dec 11 23:11:21 2019	(r355633)
@@ -761,11 +761,14 @@ int	vop_stdgetwritemount(struct vop_getwritemount_args
 int	vop_stdgetpages(struct vop_getpages_args *);
 int	vop_stdinactive(struct vop_inactive_args *);
 int	vop_stdneed_inactive(struct vop_need_inactive_args *);
-int	vop_stdislocked(struct vop_islocked_args *);
 int	vop_stdkqfilter(struct vop_kqfilter_args *);
 int	vop_stdlock(struct vop_lock1_args *);
-int	vop_stdputpages(struct vop_putpages_args *);
 int	vop_stdunlock(struct vop_unlock_args *);
+int	vop_stdislocked(struct vop_islocked_args *);
+int	vop_lock(struct vop_lock1_args *);
+int	vop_unlock(struct vop_unlock_args *);
+int	vop_islocked(struct vop_islocked_args *);
+int	vop_stdputpages(struct vop_putpages_args *);
 int	vop_nopoll(struct vop_poll_args *);
 int	vop_stdaccess(struct vop_access_args *ap);
 int	vop_stdaccessx(struct vop_accessx_args *ap);


More information about the svn-src-head mailing list