git: ff1ae3b3639d - main - rangelocks: restore caching of the single rl entry in the struct thread
Date: Tue, 06 Aug 2024 04:06:39 UTC
The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=ff1ae3b3639d39a6485cfc655bf565cd04b9caa6 commit ff1ae3b3639d39a6485cfc655bf565cd04b9caa6 Author: Konstantin Belousov <kib@FreeBSD.org> AuthorDate: 2023-08-23 23:29:50 +0000 Commit: Konstantin Belousov <kib@FreeBSD.org> CommitDate: 2024-08-06 04:05:58 +0000 rangelocks: restore caching of the single rl entry in the struct thread Reviewed by: markj, Olivier Certner <olce.freebsd@certner.fr> Tested by: pho Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D41787 --- sys/kern/kern_rangelock.c | 33 +++++++++++++++++++++++++++------ sys/kern/kern_thread.c | 1 + sys/sys/rangelock.h | 1 + 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/sys/kern/kern_rangelock.c b/sys/kern/kern_rangelock.c index 355b2125dd9b..2f16569b19aa 100644 --- a/sys/kern/kern_rangelock.c +++ b/sys/kern/kern_rangelock.c @@ -82,8 +82,15 @@ static struct rl_q_entry * rlqentry_alloc(vm_ooffset_t start, vm_ooffset_t end, int flags) { struct rl_q_entry *e; - - e = uma_zalloc_smr(rl_entry_zone, M_WAITOK); + struct thread *td; + + td = curthread; + if (td->td_rlqe != NULL) { + e = td->td_rlqe; + td->td_rlqe = NULL; + } else { + e = uma_zalloc_smr(rl_entry_zone, M_WAITOK); + } e->rl_q_next = NULL; e->rl_q_free = NULL; e->rl_q_start = start; @@ -95,6 +102,12 @@ rlqentry_alloc(vm_ooffset_t start, vm_ooffset_t end, int flags) return (e); } +void +rangelock_entry_free(struct rl_q_entry *e) +{ + uma_zfree_smr(rl_entry_zone, e); +} + void rangelock_init(struct rangelock *lock) { @@ -106,6 +119,7 @@ void rangelock_destroy(struct rangelock *lock) { struct rl_q_entry *e, *ep; + struct thread *td; MPASS(!lock->sleepers); for (e = (struct rl_q_entry *)atomic_load_ptr(&lock->head); @@ -386,8 +400,10 @@ rangelock_lock_int(struct rangelock *lock, bool trylock, vm_ooffset_t start, vm_ooffset_t end, int locktype) { struct rl_q_entry *e, *free, *x, *xp; + struct thread *td; enum RL_INSERT_RES res; + td = curthread; for (res = RL_LOCK_RETRY; res == RL_LOCK_RETRY;) { free = NULL; e = rlqentry_alloc(start, end, locktype); @@ -401,10 +417,15 @@ rangelock_lock_int(struct rangelock *lock, bool trylock, vm_ooffset_t start, e = NULL; } for (x = free; x != NULL; x = xp) { - MPASS(!rl_e_is_marked(x)); - xp = x->rl_q_free; - MPASS(!rl_e_is_marked(xp)); - uma_zfree_smr(rl_entry_zone, x); + MPASS(!rl_e_is_marked(x)); + xp = x->rl_q_free; + MPASS(!rl_e_is_marked(xp)); + if (td->td_rlqe == NULL) { + smr_synchronize(rl_smr); + td->td_rlqe = x; + } else { + uma_zfree_smr(rl_entry_zone, x); + } } } return (e); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index c951e7297c89..9c3694feb945 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -480,6 +480,7 @@ thread_fini(void *mem, int size) EVENTHANDLER_DIRECT_INVOKE(thread_fini, td); turnstile_free(td->td_turnstile); sleepq_free(td->td_sleepqueue); + rangelock_entry_free(td->td_rlqe); umtx_thread_fini(td); MPASS(td->td_sel == NULL); } diff --git a/sys/sys/rangelock.h b/sys/sys/rangelock.h index 310371bef879..60f394b67677 100644 --- a/sys/sys/rangelock.h +++ b/sys/sys/rangelock.h @@ -65,6 +65,7 @@ void *rangelock_wlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end); void *rangelock_trywlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end); +void rangelock_entry_free(struct rl_q_entry *e); #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) void _rangelock_cookie_assert(void *cookie, int what, const char *file, int line);