PERFORCE change 117121 for review

[Kip Macy]

http://perforce.freebsd.org/chv.cgi?CH=117121

Change 117121 by kmacy at kmacy_vt-x:opentoe_init on 2007/04/01 20:11:47

	fix mb_ctor_clust to actually reference the correct zone for a cluster
	eliminate duplicate calls to uma_find_refcnt
	cleanup the INVARIANT checks in mb_dtor_clust
	
	add m_cljset to attach a cluster to a deferred mbuf
	add m_getzone to eliminate duplicated zone lookup code for clusters
	add mbuf_iovec for using pktdat for cache-friendly scatter-gather of clusters

Affected files ...

.. //depot/projects/opentoe/sys/kern/kern_mbuf.c#3 edit
.. //depot/projects/opentoe/sys/sys/mbuf.h#3 edit

Differences ...

==== //depot/projects/opentoe/sys/kern/kern_mbuf.c#3 (text+ko) ====

@@ -423,7 +423,11 @@
 	default:
 		panic("unknown cluster size");
 		break;
-	}	
+	}
+	
+	refcnt =  uma_find_refcnt(zone, mem);
+	*refcnt = 1;
+	
 	m = (struct mbuf *)arg;
 	if (m != NULL) {
 		m->m_ext.ext_buf = (caddr_t)mem;
@@ -433,12 +437,9 @@
 		m->m_ext.ext_args = NULL;
 		m->m_ext.ext_size = size;
 		m->m_ext.ext_type = type;
-		m->m_ext.ref_cnt = uma_find_refcnt(zone, mem);
-		*m->m_ext.ref_cnt = 1;
-	} else {
-		refcnt =  uma_find_refcnt(zone, mem);
-		*refcnt = 1;
+		m->m_ext.ref_cnt = refcnt;
 	}
+
 	return (0);
 }
 
@@ -448,11 +449,13 @@
 static void
 mb_dtor_clust(void *mem, int size, void *arg)
 {
+#ifdef INVARIANTS
+	uma_zone_t zone = m_getzone(size);
 
-	KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1,
+	KASSERT(*(uma_find_refcnt(zone, mem)) <= 1,
 		("%s: refcnt incorrect %u", __func__,
-		 *(uma_find_refcnt(zone_clust, mem))) );
-#ifdef INVARIANTS
+		 *(uma_find_refcnt(zone, mem))) );
+
 	trash_dtor(mem, size, arg);
 #endif
 }

==== //depot/projects/opentoe/sys/sys/mbuf.h#3 (text+ko) ====

@@ -185,6 +185,8 @@
 #define	M_LASTFRAG	0x2000	/* packet is last fragment */
 #define	M_VLANTAG	0x10000	/* ether_vtag is valid */
 #define	M_PROMISC	0x20000	/* packet was not for us */
+#define M_IOVEC         0x40000 /* mbuf immediate data area is used to reference clusters */
+#define M_LRO           0x80000 /* large receive offload in use for packet */
 
 /*
  * External buffer types: identify ext_buf type.
@@ -275,6 +277,21 @@
 };
 
 /*
+ * m_pktdat == 200 bytes on 64-bit arches, need to stay below that
+ *
+ */
+#define MAX_MBUF_IOV 12
+struct mbuf_iovec {
+	uint16_t mi_first;                /* first valid cluster        */
+	uint16_t mi_count;                /* number of valid clusters   */
+	uint16_t mi_flags[MAX_MBUF_IOV];  /* per-cluster flags          */
+	uint16_t mi_offsets[MAX_MBUF_IOV];/* data offsets of clusters   */
+	uint16_t mi_lens[MAX_MBUF_IOV];   /* length of clusters         */
+	uint32_t pad;                     /* 8-byte align mi_bases      */
+	caddr_t  mi_bases[MAX_MBUF_IOV];  /* pointers to clusters       */
+};
+
+/*
  * Flags specifying how an allocation should be made.
  *
  * The flag to use is as follows:
@@ -392,6 +409,33 @@
 	return ((struct mbuf *)(uma_zalloc_arg(zone_pack, &args, how)));
 }
 
+static __inline uma_zone_t
+m_getzone(int size)
+{
+	uma_zone_t zone;
+	
+	switch (size) {
+	case MCLBYTES:
+		zone = zone_clust;
+		break;
+#if MJUMPAGESIZE != MCLBYTES
+	case MJUMPAGESIZE:
+		zone = zone_jumbop;
+		break;
+#endif
+	case MJUM9BYTES:
+		zone = zone_jumbo9;
+		break;
+	case MJUM16BYTES:
+		zone = zone_jumbo16;
+		break;
+	default:
+		panic("%s: m_getjcl: invalid cluster type", __func__);
+	}
+
+	return (zone);
+}
+
 /*
  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
@@ -412,24 +456,7 @@
 	if (m == NULL)
 		return (NULL);
 
-	switch (size) {
-	case MCLBYTES:
-		zone = zone_clust;
-		break;
-#if MJUMPAGESIZE != MCLBYTES
-	case MJUMPAGESIZE:
-		zone = zone_jumbop;
-		break;
-#endif
-	case MJUM9BYTES:
-		zone = zone_jumbo9;
-		break;
-	case MJUM16BYTES:
-		zone = zone_jumbo16;
-		break;
-	default:
-		panic("%s: m_getjcl: invalid cluster type", __func__);
-	}
+	zone = m_getzone(size);
 	n = uma_zalloc_arg(zone, m, how);
 	if (n == NULL) {
 		uma_zfree(zone_mbuf, m);
@@ -468,6 +495,7 @@
 	}
 }
 
+
 /*
  * m_cljget() is different from m_clget() as it can allocate clusters without
  * attaching them to an mbuf.  In that case the return value is the pointer
@@ -478,35 +506,55 @@
 static __inline void *
 m_cljget(struct mbuf *m, int how, int size)
 {
-	uma_zone_t zone;
-
+	uma_zone_t zone = m_getzone(size);
+	
 	if (m && m->m_flags & M_EXT)
 		printf("%s: %p mbuf already has cluster\n", __func__, m);
 	if (m != NULL)
 		m->m_ext.ext_buf = NULL;
 
+	return (uma_zalloc_arg(zone, m, how));
+}
+
+static __inline void
+m_cljset(struct mbuf *m, void *cl, int size)
+{
+	uma_zone_t zone;
+	int type;
+	
 	switch (size) {
 	case MCLBYTES:
+		type = EXT_CLUSTER;
 		zone = zone_clust;
 		break;
 #if MJUMPAGESIZE != MCLBYTES
 	case MJUMPAGESIZE:
+		type = EXT_JUMBOP;
 		zone = zone_jumbop;
 		break;
 #endif
 	case MJUM9BYTES:
+		type = EXT_JUMBO9;
 		zone = zone_jumbo9;
 		break;
 	case MJUM16BYTES:
+		type = EXT_JUMBO16;
 		zone = zone_jumbo16;
 		break;
 	default:
-		panic("%s: m_getjcl: invalid cluster type", __func__);
+		panic("unknown cluster size");
+		break;
 	}
 
-	return (uma_zalloc_arg(zone, m, how));
+	m->m_data = m->m_ext.ext_buf = cl;
+	m->m_ext.ext_free = m->m_ext.ext_args = NULL;
+	m->m_ext.ext_size = size;
+	m->m_ext.ext_type = type;
+	m->m_ext.ref_cnt = uma_find_refcnt(zone, cl);
+	m->m_flags |= M_EXT;
+
 }
-
+       
 static __inline void
 m_chtype(struct mbuf *m, short new_type)
 {