svn commit: r185126 - in user/kmacy/HEAD_fast_multi_xmit/sys: kern
sys
Kip Macy
kmacy at FreeBSD.org
Thu Nov 20 02:47:21 PST 2008
Author: kmacy
Date: Thu Nov 20 10:47:21 2008
New Revision: 185126
URL: http://svn.freebsd.org/changeset/base/185126
Log:
Add lock-less ring buffer as primitive for replacing ifq
Added:
user/kmacy/HEAD_fast_multi_xmit/sys/kern/subr_bufr.c (contents, props changed)
user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h (contents, props changed)
Added: user/kmacy/HEAD_fast_multi_xmit/sys/kern/subr_bufr.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/kmacy/HEAD_fast_multi_xmit/sys/kern/subr_bufr.c Thu Nov 20 10:47:21 2008 (r185126)
@@ -0,0 +1,66 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2007,2008 Kip Macy kmacy at freebsd.org
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. The name of Kip Macy nor the names of other
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ *
+ ***************************************************************************/
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/ktr.h>
+#include <sys/buf_ring.h>
+
+
+struct buf_ring *
+buf_ring_alloc(int count, struct malloc_type *type, int flags)
+{
+ struct buf_ring *br;
+
+ KASSERT(powerof2(count), ("buf ring must be size power of 2"));
+
+ br = malloc(sizeof(struct buf_ring) + count*sizeof(caddr_t),
+ type, flags|M_ZERO);
+ if (br == NULL)
+ return (NULL);
+
+ br->br_prod_size = br->br_cons_size = count;
+ br->br_prod_mask = br->br_cons_mask = count-1;
+ br->br_prod_head = br->br_cons_head = 0;
+ br->br_prod_tail = br->br_cons_tail = 0;
+
+ return (br);
+}
+
+void
+buf_ring_free(struct buf_ring *br, struct malloc_type *type)
+{
+ free(br, type);
+}
Added: user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h Thu Nov 20 10:47:21 2008 (r185126)
@@ -0,0 +1,195 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2007,2008 Kip Macy kmacy at freebsd.org
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. The name of Kip Macy nor the names of other
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ ***************************************************************************/
+
+#ifndef _SYS_BUF_RING_H_
+#define _SYS_BUF_RING_H_
+
+#include <machine/cpu.h>
+
+#if defined (__GNUC__)
+ #if #cpu(i386) || defined __i386 || defined i386 || defined __i386__ || #cpu(x86_64) || defined __x86_64__
+ #define mb() __asm__ __volatile__ ("mfence;": : :"memory")
+ #define wmb() __asm__ __volatile__ ("sfence;": : :"memory")
+ #define rmb() __asm__ __volatile__ ("lfence;": : :"memory")
+ #elif #cpu(sparc64) || defined sparc64 || defined __sparcv9
+ #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory")
+ #define wmb() mb()
+ #define rmb() mb()
+ #elif #cpu(sparc) || defined sparc || defined __sparc__
+ #define mb() __asm__ __volatile__ ("stbar;": : :"memory")
+ #define wmb() mb()
+ #define rmb() mb()
+#else
+ #define wmb() mb()
+ #define rmb() mb()
+ #define mb() /* XXX just to make this compile */
+ #endif
+#else
+ #error "unknown compiler"
+#endif
+
+struct buf_ring {
+ volatile uint32_t br_prod_head;
+ volatile uint32_t br_prod_tail;
+ int br_prod_size;
+ int br_prod_mask;
+ /*
+ * Pad out to next L2 cache line
+ */
+ uint64_t _pad0[14];
+
+ volatile uint32_t br_cons_head;
+ volatile uint32_t br_cons_tail;
+ int br_cons_size;
+ int br_cons_mask;
+
+ /*
+ * Pad out to next L2 cache line
+ */
+ uint64_t _pad1[14];
+ void *br_ring[0];
+};
+
+
+static __inline int
+buf_ring_enqueue(struct buf_ring *br, void *buf)
+{
+ uint32_t prod_head, prod_next;
+ uint32_t cons_tail;
+ int success;
+
+ do {
+ prod_head = br->br_prod_head;
+ cons_tail = br->br_cons_tail;
+
+ prod_next = (prod_head + 1) & br->br_prod_mask;
+
+ if (prod_next == cons_tail)
+ return (ENOSPC);
+
+ success = atomic_cmpset_int(&br->br_prod_head, prod_head,
+ prod_next);
+ } while (success == 0);
+
+ KASSERT(br->br_ring[prod_head] == NULL, ("dangling value in enqueue"));
+
+ br->br_ring[prod_head] = buf;
+ wmb();
+
+ /*
+ * If there are other enqueues in progress
+ * that preceeded us, we need to wait for them
+ * to complete - this is only a problem if they
+ * have been preempted
+ */
+ while (br->br_prod_tail != prod_head)
+ cpu_spinwait();
+ br->br_prod_tail = prod_next;
+
+ return (0);
+}
+
+static __inline void *
+buf_ring_dequeue(struct buf_ring *br)
+{
+ uint32_t cons_head, cons_next;
+ uint32_t prod_tail;
+ void *buf;
+ int success;
+
+ do {
+ cons_head = br->br_cons_head;
+ prod_tail = br->br_prod_tail;
+
+ cons_next = (cons_head + 1) & br->br_cons_mask;
+
+ if (cons_head == prod_tail)
+ return (NULL);
+
+ success = atomic_cmpset_int(&br->br_cons_head, cons_head,
+ cons_next);
+ } while (success == 0);
+
+ buf = br->br_ring[cons_head];
+#ifdef INVARIANTS
+ br->br_ring[cons_head] = NULL;
+#endif
+ mb();
+
+ /*
+ * If there are other dequeues in progress
+ * that preceeded us, we need to wait for them
+ * to complete - this is only a problem if they
+ * have been preempted
+ */
+ while (br->br_cons_tail != cons_head)
+ cpu_spinwait();
+
+ br->br_cons_tail = cons_next;
+
+ return (buf);
+}
+
+static __inline void *
+buf_ring_peek(struct buf_ring *br)
+{
+
+ return (br->br_ring[br->br_cons_tail]);
+}
+
+static __inline int
+buf_ring_full(struct buf_ring *br)
+{
+
+ return (((br->br_prod_head + 1) & br->br_prod_mask) == br->br_cons_tail);
+}
+
+static __inline int
+buf_ring_empty(struct buf_ring *br)
+{
+
+ return (br->br_cons_head == br->br_prod_tail);
+}
+
+static __inline int
+buf_ring_count(struct buf_ring *br)
+{
+
+ return ((br->br_prod_size + br->br_prod_tail - br->br_cons_tail)
+ & br->br_prod_mask);
+}
+
+struct buf_ring *buf_ring_alloc(int count, struct malloc_type *type, int flags);
+void buf_ring_free(struct buf_ring *br, struct malloc_type *type);
+
+
+
+#endif
More information about the svn-src-user
mailing list