svn commit: r364811 - head/sys/kern
Mateusz Guzik
mjg at FreeBSD.org
Wed Aug 26 12:50:10 UTC 2020
Author: mjg
Date: Wed Aug 26 12:50:10 2020
New Revision: 364811
URL: https://svnweb.freebsd.org/changeset/base/364811
Log:
cache: decouple smr and locked lookup in the slowpath
Tested by: pho
Modified:
head/sys/kern/vfs_cache.c
Modified: head/sys/kern/vfs_cache.c
==============================================================================
--- head/sys/kern/vfs_cache.c Wed Aug 26 12:49:39 2020 (r364810)
+++ head/sys/kern/vfs_cache.c Wed Aug 26 12:50:10 2020 (r364811)
@@ -1538,17 +1538,104 @@ out_no_entry:
* .., dvp is unlocked. If we're looking up . an extra ref is taken, but the
* lock is not recursively acquired.
*/
+static int __noinline
+cache_lookup_fallback(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
+ struct timespec *tsp, int *ticksp)
+{
+ struct namecache *ncp;
+ struct rwlock *blp;
+ uint32_t hash;
+ enum vgetstate vs;
+ int error;
+ bool whiteout;
+
+ MPASS((cnp->cn_flags & (MAKEENTRY | ISDOTDOT)) == MAKEENTRY);
+
+retry:
+ hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
+ blp = HASH2BUCKETLOCK(hash);
+ rw_rlock(blp);
+
+ CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
+ if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
+ !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
+ break;
+ }
+
+ /* We failed to find an entry */
+ if (__predict_false(ncp == NULL)) {
+ rw_runlock(blp);
+ SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
+ NULL);
+ counter_u64_add(nummiss, 1);
+ return (0);
+ }
+
+ if (ncp->nc_flag & NCF_NEGATIVE)
+ goto negative_success;
+
+ /* We found a "positive" match, return the vnode */
+ counter_u64_add(numposhits, 1);
+ *vpp = ncp->nc_vp;
+ CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
+ dvp, cnp->cn_nameptr, *vpp, ncp);
+ SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name,
+ *vpp);
+ cache_out_ts(ncp, tsp, ticksp);
+ /*
+ * On success we return a locked and ref'd vnode as per the lookup
+ * protocol.
+ */
+ MPASS(dvp != *vpp);
+ vs = vget_prep(*vpp);
+ cache_lookup_unlock(blp);
+ error = vget_finish(*vpp, cnp->cn_lkflags, vs);
+ if (error) {
+ *vpp = NULL;
+ goto retry;
+ }
+ if ((cnp->cn_flags & ISLASTCN) &&
+ (cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) {
+ ASSERT_VOP_ELOCKED(*vpp, "cache_lookup");
+ }
+ return (-1);
+
+negative_success:
+ /* We found a negative match, and want to create it, so purge */
+ if (__predict_false(cnp->cn_nameiop == CREATE)) {
+ counter_u64_add(numnegzaps, 1);
+ error = cache_zap_rlocked_bucket(ncp, cnp, hash, blp);
+ if (__predict_false(error != 0)) {
+ zap_and_exit_bucket_fail2++;
+ cache_maybe_yield();
+ goto retry;
+ }
+ cache_free(ncp);
+ return (0);
+ }
+
+ SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name);
+ cache_out_ts(ncp, tsp, ticksp);
+ counter_u64_add(numneghits, 1);
+ whiteout = (ncp->nc_flag & NCF_WHITE);
+ cache_negative_hit(ncp);
+ cache_lookup_unlock(blp);
+ if (whiteout)
+ cnp->cn_flags |= ISWHITEOUT;
+ return (ENOENT);
+}
+
int
cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
struct timespec *tsp, int *ticksp)
{
struct namecache *ncp;
struct negstate *negstate;
- struct rwlock *blp;
uint32_t hash;
enum vgetstate vs;
int error;
- bool try_smr, doing_smr, whiteout;
+ bool whiteout;
+ u_short nc_flag;
#ifdef DEBUG_CACHE
if (__predict_false(!doingcache)) {
@@ -1571,24 +1658,15 @@ cache_lookup(struct vnode *dvp, struct vnode **vpp, st
return (0);
}
- try_smr = true;
+ /*
+ * TODO: we only fallback becasue if a negative entry is found it will
+ * need to be purged.
+ */
if (cnp->cn_nameiop == CREATE)
- try_smr = false;
-retry:
- doing_smr = false;
- blp = NULL;
- error = 0;
+ goto out_fallback;
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
-retry_hashed:
- if (try_smr) {
- vfs_smr_enter();
- doing_smr = true;
- try_smr = false;
- } else {
- blp = HASH2BUCKETLOCK(hash);
- rw_rlock(blp);
- }
+ vfs_smr_enter();
CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
@@ -1598,17 +1676,15 @@ retry_hashed:
/* We failed to find an entry */
if (__predict_false(ncp == NULL)) {
- if (doing_smr)
- vfs_smr_exit();
- else
- rw_runlock(blp);
+ vfs_smr_exit();
SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
NULL);
counter_u64_add(nummiss, 1);
return (0);
}
- if (ncp->nc_flag & NCF_NEGATIVE)
+ nc_flag = atomic_load_char(&ncp->nc_flag);
+ if (nc_flag & NCF_NEGATIVE)
goto negative_success;
/* We found a "positive" match, return the vnode */
@@ -1624,26 +1700,21 @@ retry_hashed:
* protocol.
*/
MPASS(dvp != *vpp);
- if (doing_smr) {
- if (!cache_ncp_canuse(ncp)) {
- vfs_smr_exit();
- *vpp = NULL;
- goto retry;
- }
- vs = vget_prep_smr(*vpp);
+ if (!cache_ncp_canuse(ncp)) {
vfs_smr_exit();
- if (__predict_false(vs == VGET_NONE)) {
- *vpp = NULL;
- goto retry;
- }
- } else {
- vs = vget_prep(*vpp);
- cache_lookup_unlock(blp);
+ *vpp = NULL;
+ goto out_fallback;
}
+ vs = vget_prep_smr(*vpp);
+ vfs_smr_exit();
+ if (__predict_false(vs == VGET_NONE)) {
+ *vpp = NULL;
+ goto out_fallback;
+ }
error = vget_finish(*vpp, cnp->cn_lkflags, vs);
if (error) {
*vpp = NULL;
- goto retry;
+ goto out_fallback;
}
if ((cnp->cn_flags & ISLASTCN) &&
(cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) {
@@ -1652,44 +1723,25 @@ retry_hashed:
return (-1);
negative_success:
- /* We found a negative match, and want to create it, so purge */
- if (__predict_false(cnp->cn_nameiop == CREATE)) {
- MPASS(!doing_smr);
- counter_u64_add(numnegzaps, 1);
- error = cache_zap_rlocked_bucket(ncp, cnp, hash, blp);
- if (__predict_false(error != 0)) {
- zap_and_exit_bucket_fail2++;
- cache_maybe_yield();
- goto retry;
- }
- cache_free(ncp);
- return (0);
- }
-
SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name);
cache_out_ts(ncp, tsp, ticksp);
counter_u64_add(numneghits, 1);
whiteout = (ncp->nc_flag & NCF_WHITE);
-
- if (doing_smr) {
- /*
- * We need to take locks to promote an entry.
- */
- negstate = NCP2NEGSTATE(ncp);
- if ((negstate->neg_flag & NEG_HOT) == 0 ||
- !cache_ncp_canuse(ncp)) {
- vfs_smr_exit();
- doing_smr = false;
- goto retry_hashed;
- }
+ /*
+ * We need to take locks to promote an entry.
+ */
+ negstate = NCP2NEGSTATE(ncp);
+ if ((negstate->neg_flag & NEG_HOT) == 0 ||
+ !cache_ncp_canuse(ncp)) {
vfs_smr_exit();
- } else {
- cache_negative_hit(ncp);
- cache_lookup_unlock(blp);
+ goto out_fallback;
}
+ vfs_smr_exit();
if (whiteout)
cnp->cn_flags |= ISWHITEOUT;
return (ENOENT);
+out_fallback:
+ return (cache_lookup_fallback(dvp, vpp, cnp, tsp, ticksp));
}
struct celockstate {
More information about the svn-src-all
mailing list