From nobody Sat Nov 27 23:19:32 2021 X-Original-To: dev-commits-src-main@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id C8B3118A9106; Sat, 27 Nov 2021 23:19:33 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4J1nd10hLcz3Lwf; Sat, 27 Nov 2021 23:19:33 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id E796E24538; Sat, 27 Nov 2021 23:19:32 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 1ARNJWhK028457; Sat, 27 Nov 2021 23:19:32 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 1ARNJW9c028456; Sat, 27 Nov 2021 23:19:32 GMT (envelope-from git) Date: Sat, 27 Nov 2021 23:19:32 GMT Message-Id: <202111272319.1ARNJW9c028456@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Mateusz Guzik Subject: git: e511bd1406fa - main - vfs: fully lockless v_writecount adjustment List-Id: Commit messages for the main branch of the src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-main List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-main@freebsd.org X-BeenThere: dev-commits-src-main@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: mjg X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: e511bd1406fa35301e770aad05d1bfdc345c8639 Auto-Submitted: auto-generated ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1638055173; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=07tqhZjfvf5KJw/b86tEYtIsV/5xLG7TYcj/MpwLOH0=; b=VDZzjRlB2iKPT9j7wFbgcduMlbYT/xZ5gWcUVk+H89UGAPfRYUAJXCb6+91WN4HpDV0l3V ufuGQHYsr4NR54dwGoOWXAvokI7P98QInlRLSeA+JxbcJkDsRREUHs1lOQX+4YwsCdGiPk YiAHz7FwKXteRMF14TkVG+bMnOJ8utkVZKV1/B3Oo0+2POkWktE1ZCJEDHqkOe1cNV0OnB vkXiAepSs+ufLW8DBKePBj5GVMFL5XF0KJK8aQOvpIHaVigqOEvmJTg7WHG4Tl0cIIeuq1 xQuUouMTCBypOKybiApn8TTqYg6dpC7zWfrODa7CXCkJJEzolAhaCWY90RrqTQ== ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1638055173; a=rsa-sha256; cv=none; b=LuRqfhrSMtVXH/AITpH/Q7rhF6kXhfh5+xAqLamYPD7nK9s0Khtfz90M/cS4I28WB2Uy24 5rTIIBLY/eBYcF7fKvJwq3bJCxSHW0oxQ87WhcJjRGMHbL/i/2eSuMDe5mVlRqdtUGala/ h8aTb2uCtpvvGKI1l+uw6wnsS0xcKZe5jr0a9jDbEarwxo2nkYBg2vhUzvcIFQHdvV0FvC CEI9Qu81833c2mp/Tgd2Pgf8W3nbqg7wLZnpPgfT6N238i08pLJ1LRTpgRuxFsi7jFoouy iFMo6V2Q9LvFzQvUnm6YEdpF/9yORLITjrL7jq37l9VzBgwacQOCtTkHCzTeBA== ARC-Authentication-Results: i=1; mx1.freebsd.org; none X-ThisMailContainsUnwantedMimeParts: N The branch main has been updated by mjg: URL: https://cgit.FreeBSD.org/src/commit/?id=e511bd1406fa35301e770aad05d1bfdc345c8639 commit e511bd1406fa35301e770aad05d1bfdc345c8639 Author: Mateusz Guzik AuthorDate: 2021-11-26 12:33:28 +0000 Commit: Mateusz Guzik CommitDate: 2021-11-27 23:07:26 +0000 vfs: fully lockless v_writecount adjustment Reviewed by: kib Differential Revision: https://reviews.freebsd.org/D33128 --- sys/kern/vfs_default.c | 125 +++++++++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 62 deletions(-) diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c index 1f5869bd8cf3..6f304f0f719d 100644 --- a/sys/kern/vfs_default.c +++ b/sys/kern/vfs_default.c @@ -1293,91 +1293,86 @@ static int vop_stdis_text(struct vop_is_text_args *ap) { - return (ap->a_vp->v_writecount < 0); + return (atomic_load_int(&ap->a_vp->v_writecount) < 0); } int vop_stdset_text(struct vop_set_text_args *ap) { struct vnode *vp; - int error, n; + int n; + bool gotref; vp = ap->a_vp; - /* - * Avoid the interlock if execs are already present. - */ n = atomic_load_int(&vp->v_writecount); for (;;) { - if (n > -1) { - break; - } - if (atomic_fcmpset_int(&vp->v_writecount, &n, n - 1)) { - return (0); + if (__predict_false(n > 0)) { + return (ETXTBSY); } - } - VI_LOCK(vp); - if (vp->v_writecount > 0) { - error = ETXTBSY; - } else { /* - * If requested by fs, keep a use reference to the - * vnode until the last text reference is released. + * Transition point, we may need to grab a reference on the vnode. + * + * Take the ref early As a safety measure against bogus calls + * to vop_stdunset_text. */ - if ((vn_irflag_read(vp) & VIRF_TEXT_REF) != 0) { - if (vp->v_writecount == 0) { - vrefl(vp); + if (n == 0) { + gotref = false; + if ((vn_irflag_read(vp) & VIRF_TEXT_REF) != 0) { + vref(vp); + gotref = true; + } + if (atomic_fcmpset_int(&vp->v_writecount, &n, -1)) { + return (0); } + if (gotref) { + vunref(vp); + } + continue; } - atomic_subtract_int(&vp->v_writecount, 1); - error = 0; + MPASS(n < 0); + if (atomic_fcmpset_int(&vp->v_writecount, &n, n - 1)) { + return (0); + } } - VI_UNLOCK(vp); - return (error); + __assert_unreachable(); } static int vop_stdunset_text(struct vop_unset_text_args *ap) { struct vnode *vp; - int error, n; - bool last; + int n; vp = ap->a_vp; - /* - * Avoid the interlock if this is not the last exec. - */ n = atomic_load_int(&vp->v_writecount); for (;;) { - if (n >= -1) { - break; - } - if (atomic_fcmpset_int(&vp->v_writecount, &n, n + 1)) { - return (0); + if (__predict_false(n >= 0)) { + return (EINVAL); } - } - last = false; - VI_LOCK(vp); - if (vp->v_writecount < 0) { - if ((vn_irflag_read(vp) & VIRF_TEXT_REF) != 0) { - if (vp->v_writecount == -1) { - last = true; + /* + * Transition point, we may need to release a reference on the vnode. + */ + if (n == -1) { + if (atomic_fcmpset_int(&vp->v_writecount, &n, 0)) { + if ((vn_irflag_read(vp) & VIRF_TEXT_REF) != 0) { + vunref(vp); + } + return (0); } + continue; } - atomic_add_int(&vp->v_writecount, 1); - error = 0; - } else { - error = EINVAL; + MPASS(n < -1); + if (atomic_fcmpset_int(&vp->v_writecount, &n, n + 1)) { + return (0); + } } - VI_UNLOCK(vp); - if (last) - vunref(vp); - return (error); + __assert_unreachable(); } static int __always_inline @@ -1385,7 +1380,7 @@ vop_stdadd_writecount_impl(struct vop_add_writecount_args *ap, bool handle_msync { struct vnode *vp; struct mount *mp __diagused; - int error; + int n; vp = ap->a_vp; @@ -1400,20 +1395,26 @@ vop_stdadd_writecount_impl(struct vop_add_writecount_args *ap, bool handle_msync } #endif - VI_LOCK_FLAGS(vp, MTX_DUPOK); - if (__predict_false(vp->v_writecount < 0)) { - error = ETXTBSY; - } else { - VNASSERT(vp->v_writecount + ap->a_inc >= 0, vp, - ("neg writecount increment %d", ap->a_inc)); - if (handle_msync && vp->v_writecount == 0) { - vlazy(vp); + n = atomic_load_int(&vp->v_writecount); + for (;;) { + if (__predict_false(n < 0)) { + return (ETXTBSY); + } + + VNASSERT(n + ap->a_inc >= 0, vp, + ("neg writecount increment %d + %d = %d", n, ap->a_inc, + n + ap->a_inc)); + if (n == 0) { + if (handle_msync) { + vlazy(vp); + } + } + + if (atomic_fcmpset_int(&vp->v_writecount, &n, n + ap->a_inc)) { + return (0); } - vp->v_writecount += ap->a_inc; - error = 0; } - VI_UNLOCK(vp); - return (error); + __assert_unreachable(); } int