git: 7806832f23fb - main - libthr: Cache recent pshared_key
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 04 Nov 2026 23:41:39 UTC
The branch main has been updated by aokblast:
URL: https://cgit.FreeBSD.org/src/commit/?id=7806832f23fb31d28386d42692b221776c9f4ee4
commit 7806832f23fb31d28386d42692b221776c9f4ee4
Author: ShengYi Hung <aokblast@FreeBSD.org>
AuthorDate: 2026-10-21 07:03:27 +0000
Commit: ShengYi Hung <aokblast@FreeBSD.org>
CommitDate: 2026-11-04 23:41:05 +0000
libthr: Cache recent pshared_key
For a pshare key, we can actually cache the last one as a fast path
instead of trapping into kernel directly. It makes sense as a lock and
unlock should be a pair on the same key and at the same time. For
example, lock(A) -> unlock(A) -> lock(B) -> unlock(B). Each unlock is
a hit and refresh later (50% hit).
For a single pshare lock, it is evne a 100% hit as the key never
changed.
For nested lock lock(A) -> lock(B) -> lock(C) -> unlock(C) ->
unlock(B) -> unlock(A). It only cache the last one. However, we should
really think about rewritting this logic as it is a pshare lock shared
by many processes and nested should be uncommon.
Reviewed by: kib
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D60957
---
lib/libthr/thread/thr_private.h | 4 ++++
lib/libthr/thread/thr_pshared.c | 45 +++++++++++++++++++++++++++++++++++++++--
2 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index ffc4ff7233db..4469533bc40e 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -522,6 +522,10 @@ struct pthread {
/* Number rwlocks rdlocks held. */
int rdlock_count;
+ void *pshared_key;
+ void *pshared_val;
+ unsigned long pshared_gen;
+
/*
* Current locks bitmap for rtld. */
int rtld_bits;
diff --git a/lib/libthr/thread/thr_pshared.c b/lib/libthr/thread/thr_pshared.c
index 698cc5e6adc7..bebae21601a6 100644
--- a/lib/libthr/thread/thr_pshared.c
+++ b/lib/libthr/thread/thr_pshared.c
@@ -47,9 +47,14 @@ LIST_HEAD(pshared_hash_head, psh);
static struct pshared_hash_head pshared_hash[HASH_SIZE];
#define PSHARED_KEY_HASH(key) (((unsigned long)(key) >> 8) % HASH_SIZE)
/* XXXKIB: lock could be split to per-hash chain, if appears contested */
-static struct urwlock pshared_lock = DEFAULT_URWLOCK;
+static struct urwlock pshared_lock __aligned(CACHE_LINE_SIZE) = DEFAULT_URWLOCK;
static int page_size;
+static struct {
+ unsigned long val;
+ char pad[CACHE_LINE_SIZE - sizeof(unsigned long)];
+} pshared_gen __aligned(CACHE_LINE_SIZE);
+
#define GC_INTERVAL_USEC 25000
static struct timeval pshared_gc_time = {0, 0};
bool __thr_pshared_destroy_imm_gc = false;
@@ -93,6 +98,30 @@ pshared_unlock(struct pthread *curthread)
_thr_ast(curthread);
}
+static void
+pshared_gen_bump(void)
+{
+ atomic_add_long(&pshared_gen.val, 1);
+}
+
+static void *
+pshared_cache_lookup(struct pthread *curthread, void *key)
+{
+ if (curthread->pshared_key == key &&
+ curthread->pshared_gen == atomic_load_long(&pshared_gen.val))
+ return (curthread->pshared_val);
+ return (NULL);
+}
+
+static void
+pshared_cache_insert(struct pthread *curthread, void *key, void *val,
+ unsigned long gen)
+{
+ curthread->pshared_key = key;
+ curthread->pshared_val = val;
+ curthread->pshared_gen = gen;
+}
+
/*
* Among all processes sharing a lock only one executes
* pthread_lock_destroy(). Other processes still have the hash and
@@ -128,6 +157,7 @@ pshared_gc(struct pthread *curthread)
if (error == 0)
continue;
LIST_REMOVE(h, link);
+ pshared_gen_bump();
munmap(h->val, page_size);
free(h);
}
@@ -212,6 +242,7 @@ pshared_remove(void *key)
LIST_FOREACH(h, hd, link) {
if (h->key == key) {
LIST_REMOVE(h, link);
+ pshared_gen_bump();
val = h->val;
free(h);
return (val);
@@ -245,6 +276,7 @@ __thr_pshared_offpage(void *key, int doalloc)
{
struct pthread *curthread;
void *res;
+ unsigned long gen;
int fd, ins_done;
curthread = _get_curthread();
@@ -252,11 +284,17 @@ __thr_pshared_offpage(void *key, int doalloc)
pshared_destroy(curthread, key);
res = NULL;
} else {
+ res = pshared_cache_lookup(curthread, key);
+ if (res != NULL)
+ return (res);
pshared_rlock(curthread);
+ gen = pshared_gen.val;
res = pshared_lookup(key);
pshared_unlock(curthread);
- if (res != NULL)
+ if (res != NULL) {
+ pshared_cache_insert(curthread, key, res, gen);
return (res);
+ }
}
fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT :
UMTX_SHM_LOOKUP, key, NULL);
@@ -267,11 +305,14 @@ __thr_pshared_offpage(void *key, int doalloc)
if (res == MAP_FAILED)
return (NULL);
pshared_wlock(curthread);
+ gen = pshared_gen.val;
ins_done = pshared_insert(key, &res);
pshared_unlock(curthread);
if (!ins_done) {
pshared_clean(key, res);
res = NULL;
+ } else {
+ pshared_cache_insert(curthread, key, res, gen);
}
return (res);
}