git: 3b498d515faf - main - buf_ring: support different lock classes
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 18 Sep 2026 21:31:54 UTC
The branch main has been updated by glebius:
URL: https://cgit.FreeBSD.org/src/commit/?id=3b498d515faf2221798a64dc15ec0ff1f371de5d
commit 3b498d515faf2221798a64dc15ec0ff1f371de5d
Author: Gleb Smirnoff <glebius@FreeBSD.org>
AuthorDate: 2026-09-18 21:18:16 +0000
Commit: Gleb Smirnoff <glebius@FreeBSD.org>
CommitDate: 2026-09-18 21:20:25 +0000
buf_ring: support different lock classes
And enable locking assertions for INVARIANTS kernel.
Reviewed by: gallatin, kib, markj
Differential Revision: https://reviews.freebsd.org/D59457
---
share/man/man9/buf_ring.9 | 11 ++++++++---
sys/kern/subr_bufring.c | 7 +++----
sys/net/if_ovpn.c | 2 +-
sys/netinet/ip_mroute.c | 3 ++-
sys/sys/buf_ring.h | 38 +++++++++++++++++++-------------------
5 files changed, 33 insertions(+), 28 deletions(-)
diff --git a/share/man/man9/buf_ring.9 b/share/man/man9/buf_ring.9
index ae3b8505ce8c..29a5e2580800 100644
--- a/share/man/man9/buf_ring.9
+++ b/share/man/man9/buf_ring.9
@@ -23,7 +23,7 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.Dd September 27, 2012
+.Dd September 18, 2026
.Dt BUF_RING 9
.Os
.Sh NAME
@@ -42,7 +42,7 @@
.In sys/param.h
.In sys/buf_ring.h
.Ft struct buf_ring *
-.Fn buf_ring_alloc "int count" "struct malloc_type *type" "int flags" "struct mtx *sc_lock"
+.Fn buf_ring_alloc "int count" "struct malloc_type *type" "int flags" "lock"
.Ft void
.Fn buf_ring_free "struct buf_ring *br" "struct malloc_type *type"
.Ft int
@@ -74,7 +74,12 @@ slots using malloc_type
and memory flags
.Fa flags .
The single consumer interface is protected by
-.Fa sc_lock .
+.Fa lock .
+Supported lock types are
+.Xr mutex 9 ,
+.Xr rwlock 9 ,
+and
+.Xr rmlock 9 .
.Pp
The
.Fn buf_ring_free
diff --git a/sys/kern/subr_bufring.c b/sys/kern/subr_bufring.c
index 8ae061c559fb..25691be69be2 100644
--- a/sys/kern/subr_bufring.c
+++ b/sys/kern/subr_bufring.c
@@ -33,7 +33,8 @@
#include <sys/buf_ring.h>
struct buf_ring *
-buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock)
+_buf_ring_alloc(int count, struct malloc_type *type, int flags,
+ struct lock_object *lo)
{
struct buf_ring *br;
@@ -43,9 +44,7 @@ buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock)
type, flags | M_ZERO);
if (br == NULL)
return (NULL);
-#ifdef DEBUG_BUFRING
- br->br_lock = lock;
-#endif
+ br->br_lock = lo;
br->br_prod_size = br->br_cons_size = count;
br->br_prod_mask = br->br_cons_mask = count-1;
br->br_prod_head = br->br_cons_head = 0;
diff --git a/sys/net/if_ovpn.c b/sys/net/if_ovpn.c
index 28e60efaeaa5..385037268230 100644
--- a/sys/net/if_ovpn.c
+++ b/sys/net/if_ovpn.c
@@ -2744,7 +2744,7 @@ ovpn_clone_create(struct if_clone *ifc, char *name, size_t len,
rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE);
sc->refcount = 0;
- sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
+ sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, &sc->lock);
COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK);
diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c
index d18a9865f9bc..1e0043b4e4fe 100644
--- a/sys/netinet/ip_mroute.c
+++ b/sys/netinet/ip_mroute.c
@@ -403,7 +403,8 @@ mfc_alloc(void)
if (rt == NULL)
return rt;
- rt->mfc_stall_ring = buf_ring_alloc(MAX_UPQ, M_MRTABLE, M_NOWAIT, NULL);
+ rt->mfc_stall_ring = buf_ring_alloc(MAX_UPQ, M_MRTABLE, M_NOWAIT,
+ NOLOCK);
if (rt->mfc_stall_ring == NULL) {
free(rt, M_MRTABLE);
return NULL;
diff --git a/sys/sys/buf_ring.h b/sys/sys/buf_ring.h
index 07a4fa52891e..db934e083e1a 100644
--- a/sys/sys/buf_ring.h
+++ b/sys/sys/buf_ring.h
@@ -36,9 +36,12 @@
#include <machine/atomic.h>
#include <machine/cpu.h>
-#if defined(DEBUG_BUFRING) && defined(_KERNEL)
+#if defined(_KERNEL)
#include <sys/lock.h>
-#include <sys/mutex.h>
+#define BR_LOCK_ASSERT(br) \
+ LOCK_CLASS((br)->br_lock)->lc_assert((br)->br_lock, LA_XLOCKED)
+#else
+#define BR_LOCK_ASSERT(br) do {} while (0)
#endif
/*
@@ -60,9 +63,7 @@ struct buf_ring {
uint32_t br_cons_tail;
int br_cons_size;
int br_cons_mask;
-#if defined(DEBUG_BUFRING) && defined(_KERNEL)
- struct mtx *br_lock;
-#endif
+ struct lock_object *br_lock;
void *br_ring[0] __aligned(CACHE_LINE_SIZE);
};
@@ -195,6 +196,8 @@ buf_ring_dequeue_sc(struct buf_ring *br)
uint32_t prod_tail, mask;
void *buf;
+ BR_LOCK_ASSERT(br);
+
mask = br->br_cons_mask;
cons_head = atomic_load_32(&br->br_cons_head);
prod_tail = atomic_load_acq_32(&br->br_prod_tail);
@@ -210,10 +213,6 @@ buf_ring_dequeue_sc(struct buf_ring *br)
#ifdef DEBUG_BUFRING
br->br_ring[cons_idx] = NULL;
-#ifdef _KERNEL
- if (!mtx_owned(br->br_lock))
- panic("lock not held on single consumer dequeue");
-#endif
if (atomic_load_32(&br->br_cons_tail) != cons_head)
panic("inconsistent list cons_tail=%d cons_head=%d",
atomic_load_32(&br->br_cons_tail), cons_head);
@@ -287,10 +286,9 @@ buf_ring_peek(struct buf_ring *br)
{
uint32_t cons_head, prod_tail, mask;
-#if defined(DEBUG_BUFRING) && defined(_KERNEL)
- if ((br->br_lock != NULL) && !mtx_owned(br->br_lock))
- panic("lock not held on single consumer dequeue");
-#endif
+ if (br->br_lock != NULL)
+ BR_LOCK_ASSERT(br);
+
mask = br->br_cons_mask;
prod_tail = atomic_load_acq_32(&br->br_prod_tail);
cons_head = atomic_load_32(&br->br_cons_head);
@@ -307,10 +305,7 @@ buf_ring_peek_clear_sc(struct buf_ring *br)
uint32_t cons_head, prod_tail, mask;
void *buf;
-#if defined(DEBUG_BUFRING) && defined(_KERNEL)
- if (!mtx_owned(br->br_lock))
- panic("lock not held on single consumer dequeue");
-#endif
+ BR_LOCK_ASSERT(br);
mask = br->br_cons_mask;
prod_tail = atomic_load_acq_32(&br->br_prod_tail);
@@ -355,8 +350,13 @@ buf_ring_count(struct buf_ring *br)
}
#ifdef _KERNEL
-struct buf_ring *buf_ring_alloc(int count, struct malloc_type *type, int flags,
- struct mtx *);
+struct buf_ring *_buf_ring_alloc(int count, struct malloc_type *type,
+ int flags, struct lock_object *);
+#define buf_ring_alloc(c, mt, f, lk) _Generic((lk), \
+ struct mtx *: _buf_ring_alloc((c), (mt), (f), lk2lo(lk)), \
+ struct mtx_padalign *: _buf_ring_alloc((c), (mt), (f), lk2lo(lk)),\
+ struct rwlock *: _buf_ring_alloc((c), (mt), (f), lk2lo(lk)), \
+ struct rmlock *: _buf_ring_alloc((c), (mt), (f), lk2lo(lk)))
void buf_ring_free(struct buf_ring *br, struct malloc_type *type);
#else