svn commit: r230052 - head/sys/dev/netmap

[Luigi Rizzo]

Author: luigi
Date: Fri Jan 13 10:21:15 2012
New Revision: 230052
URL: http://svn.freebsd.org/changeset/base/230052

Log:
  Two performance-related fixes:
  1. as reported by Alexander Fiveg, the allocator was reporting
     half of the allocated memory. Fix this by exiting from the
     loop earlier (not too critical because this code is going
     away soon).
  
  2. following a discussion on freebsd-current
      http://lists.freebsd.org/pipermail/freebsd-current/2012-January/031144.html
     turns out that (re)loading the dmamap was expensive and not optimized.
     This operation is in the critical path when doing zero-copy forwarding
     between interfaces.
     At least on netmap and i386/amd64, the bus_dmamap_load can be
     completely bypassed if the map is NULL, so we do it.
  
  The latter change gives an almost 3x improvement in forwarding
  performance, from the previous 9.5Mpps at 2.9GHz to the current
  line rate (14.2Mpps) at 1.733GHz. (this is for 64+4 byte packets,
  in other configurations the PCIe bus is a bottleneck).

Modified:
  head/sys/dev/netmap/netmap.c
  head/sys/dev/netmap/netmap_kern.h

Modified: head/sys/dev/netmap/netmap.c
==============================================================================
--- head/sys/dev/netmap/netmap.c	Fri Jan 13 09:08:41 2012	(r230051)
+++ head/sys/dev/netmap/netmap.c	Fri Jan 13 10:21:15 2012	(r230052)
@@ -426,7 +426,6 @@ netmap_dtor(void *data)
 }
 
 
-
 /*
  * Create and return a new ``netmap_if`` object, and possibly also
  * rings and packet buffors.
@@ -1393,30 +1392,6 @@ netmap_reset(struct netmap_adapter *na, 
 	return kring->ring->slot;
 }
 
-static void
-ns_dmamap_cb(__unused void *arg, __unused bus_dma_segment_t * segs,
-	__unused int nseg, __unused int error)
-{
-}
-
-/* unload a bus_dmamap and create a new one. Used when the
- * buffer in the slot is changed.
- * XXX buflen is probably not needed, buffers have constant size.
- */
-void
-netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
-{
-	bus_dmamap_unload(tag, map);
-	bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, ns_dmamap_cb,
-		NULL, BUS_DMA_NOWAIT);
-}
-
-void
-netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
-{
-	bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, ns_dmamap_cb,
-		NULL, BUS_DMA_NOWAIT);
-}
 
 /*------ netmap memory allocator -------*/
 /*
@@ -1541,7 +1516,7 @@ netmap_memory_init(void)
 	int i, n, sz = NETMAP_MEMORY_SIZE;
 	int extra_sz = 0; // space for rings and two spare buffers
 
-	for (; !buf && sz >= 1<<20; sz >>=1) {
+	for (; sz >= 1<<20; sz >>=1) {
 		extra_sz = sz/200;
 		extra_sz = (extra_sz + 2*PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
 	        buf = contigmalloc(sz + extra_sz,
@@ -1552,6 +1527,8 @@ netmap_memory_init(void)
 			     PAGE_SIZE, /* alignment */
 			     0 /* boundary */
 			    );
+		if (buf)
+			break;
 	} 
 	if (buf == NULL)
 		return (ENOMEM);

Modified: head/sys/dev/netmap/netmap_kern.h
==============================================================================
--- head/sys/dev/netmap/netmap_kern.h	Fri Jan 13 09:08:41 2012	(r230051)
+++ head/sys/dev/netmap/netmap_kern.h	Fri Jan 13 10:21:15 2012	(r230052)
@@ -169,16 +169,8 @@ int netmap_start(struct ifnet *, struct 
 enum txrx { NR_RX = 0, NR_TX = 1 };
 struct netmap_slot *netmap_reset(struct netmap_adapter *na,
 	enum txrx tx, int n, u_int new_cur);
-void netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf);
-void netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf);
 int netmap_ring_reinit(struct netmap_kring *);
 
-/*
- * XXX eventually, get rid of netmap_total_buffers and netmap_buffer_base
- * in favour of the structure
- */
-// struct netmap_buf_pool;
-// extern struct netmap_buf_pool nm_buf_pool;
 extern u_int netmap_total_buffers;
 extern char *netmap_buffer_base;
 extern int netmap_verbose;	// XXX debugging
@@ -203,6 +195,35 @@ enum {                                  
 #define	NA(_ifp)	((struct netmap_adapter *)WNA(_ifp))
 
 
+/* Callback invoked by the dma machinery after a successfull dmamap_load */
+static void netmap_dmamap_cb(__unused void *arg,
+	__unused bus_dma_segment_t * segs, __unused int nseg, __unused int error)
+{
+}
+
+/* bus_dmamap_load wrapper: call aforementioned function if map != NULL.
+ * XXX can we do it without a callback ?
+ */
+static inline void
+netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
+{
+	if (map)
+		bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
+			netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
+}
+
+/* update the map when a buffer changes. */
+static inline void
+netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf)
+{
+	if (map) {
+		bus_dmamap_unload(tag, map);
+		bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE,
+			netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT);
+	}
+}
+
+
 /*
  * NMB return the virtual address of a buffer (buffer 0 on bad index)
  * PNMB also fills the physical address