svn commit: r256287 - in user/andre/mbuf_staging: kern sys
Andre Oppermann
andre at FreeBSD.org
Thu Oct 10 19:03:36 UTC 2013
Author: andre
Date: Thu Oct 10 19:03:35 2013
New Revision: 256287
URL: http://svnweb.freebsd.org/changeset/base/256287
Log:
De-inline the mbuf and cluster allocation functions and mechanically
move them to kern/kern_mbuf.c with exception of m_extaddref() which
going to kern/uipc_mbuf.c.
Having all mbuf and cluster allocation functions as real functions
again gives us significant future freedom in optimizing the backend
allocator without breaking the ABI for drivers.
As a side-note the function call overhead on modern CPUs is very low
and in other cases micro-benchmarks have shown that de-inlining actually
slightly improved performance by reducing the number of instructions and
improving I-cache efficiency. This change hasn't been measured yet
though.
Discussed with: glebius
Modified:
user/andre/mbuf_staging/kern/kern_mbuf.c
user/andre/mbuf_staging/kern/uipc_mbuf.c
user/andre/mbuf_staging/sys/mbuf.h
Modified: user/andre/mbuf_staging/kern/kern_mbuf.c
==============================================================================
--- user/andre/mbuf_staging/kern/kern_mbuf.c Thu Oct 10 18:34:46 2013 (r256286)
+++ user/andre/mbuf_staging/kern/kern_mbuf.c Thu Oct 10 19:03:35 2013 (r256287)
@@ -640,6 +640,60 @@ mb_ctor_pack(void *mem, int size, void *
return (error);
}
+uma_zone_t
+m_getzone(int size)
+{
+ uma_zone_t zone;
+
+ switch (size) {
+ case MCLBYTES:
+ zone = zone_clust;
+ break;
+#if MJUMPAGESIZE != MCLBYTES
+ case MJUMPAGESIZE:
+ zone = zone_jumbop;
+ break;
+#endif
+ case MJUM9BYTES:
+ zone = zone_jumbo9;
+ break;
+ case MJUM16BYTES:
+ zone = zone_jumbo16;
+ break;
+ default:
+ panic("%s: invalid cluster size", __func__);
+ }
+
+ return (zone);
+}
+
+/*
+ * Initialize an mbuf with linear storage.
+ *
+ * Inline because the consumer text overhead will be roughly the same to
+ * initialize or call a function with this many parameters and M_PKTHDR
+ * should go away with constant propagation for !MGETHDR.
+ */
+int
+m_init(struct mbuf *m, uma_zone_t zone, int size, int how, short type,
+ int flags)
+{
+ int error;
+
+ m->m_next = NULL;
+ m->m_nextpkt = NULL;
+ m->m_data = m->m_dat;
+ m->m_len = 0;
+ m->m_flags = flags;
+ m->m_type = type;
+ if (flags & M_PKTHDR) {
+ if ((error = m_pkthdr_init(m, how)) != 0)
+ return (error);
+ }
+
+ return (0);
+}
+
int
m_pkthdr_init(struct mbuf *m, int how)
{
@@ -671,6 +725,132 @@ m_pkthdr_init(struct mbuf *m, int how)
return (0);
}
+struct mbuf *
+m_get(int how, short type)
+{
+ struct mb_args args;
+
+ args.flags = 0;
+ args.type = type;
+ return (uma_zalloc_arg(zone_mbuf, &args, how));
+}
+
+/*
+ * XXX This should be deprecated, very little use.
+ */
+struct mbuf *
+m_getclr(int how, short type)
+{
+ struct mbuf *m;
+ struct mb_args args;
+
+ args.flags = 0;
+ args.type = type;
+ m = uma_zalloc_arg(zone_mbuf, &args, how);
+ if (m != NULL)
+ bzero(m->m_data, MLEN);
+ return (m);
+}
+
+struct mbuf *
+m_gethdr(int how, short type)
+{
+ struct mb_args args;
+
+ args.flags = M_PKTHDR;
+ args.type = type;
+ return (uma_zalloc_arg(zone_mbuf, &args, how));
+}
+
+struct mbuf *
+m_getcl(int how, short type, int flags)
+{
+ struct mb_args args;
+
+ args.flags = flags;
+ args.type = type;
+ return (uma_zalloc_arg(zone_pack, &args, how));
+}
+
+void
+m_clget(struct mbuf *m, int how)
+{
+
+ if (m->m_flags & M_EXT)
+ printf("%s: %p mbuf already has cluster\n", __func__, m);
+ m->m_ext.ext_buf = (char *)NULL;
+ uma_zalloc_arg(zone_clust, m, how);
+ /*
+ * On a cluster allocation failure, drain the packet zone and retry,
+ * we might be able to loosen a few clusters up on the drain.
+ */
+ if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
+ zone_drain(zone_pack);
+ uma_zalloc_arg(zone_clust, m, how);
+ }
+}
+
+/*
+ * m_cljget() is different from m_clget() as it can allocate clusters without
+ * attaching them to an mbuf. In that case the return value is the pointer
+ * to the cluster of the requested size. If an mbuf was specified, it gets
+ * the cluster attached to it and the return value can be safely ignored.
+ * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
+ */
+void *
+m_cljget(struct mbuf *m, int how, int size)
+{
+ uma_zone_t zone;
+
+ if (m && m->m_flags & M_EXT)
+ printf("%s: %p mbuf already has cluster\n", __func__, m);
+ if (m != NULL)
+ m->m_ext.ext_buf = NULL;
+
+ zone = m_getzone(size);
+ return (uma_zalloc_arg(zone, m, how));
+}
+
+void
+m_cljset(struct mbuf *m, void *cl, int type)
+{
+ uma_zone_t zone;
+ int size;
+
+ switch (type) {
+ case EXT_CLUSTER:
+ size = MCLBYTES;
+ zone = zone_clust;
+ break;
+#if MJUMPAGESIZE != MCLBYTES
+ case EXT_JUMBOP:
+ size = MJUMPAGESIZE;
+ zone = zone_jumbop;
+ break;
+#endif
+ case EXT_JUMBO9:
+ size = MJUM9BYTES;
+ zone = zone_jumbo9;
+ break;
+ case EXT_JUMBO16:
+ size = MJUM16BYTES;
+ zone = zone_jumbo16;
+ break;
+ default:
+ panic("%s: unknown cluster type", __func__);
+ break;
+ }
+
+ m->m_data = m->m_ext.ext_buf = cl;
+ m->m_ext.ext_free = m->m_ext.ext_arg1 = m->m_ext.ext_arg2 = NULL;
+ m->m_ext.ext_size = size;
+ m->m_ext.ext_type = type;
+ m->m_ext.ext_flags = 0;
+ m->m_ext.ref_cnt = uma_find_refcnt(zone, cl);
+ m->m_flags |= M_EXT;
+
+}
+
/*
* This is the protocol drain routine.
*
Modified: user/andre/mbuf_staging/kern/uipc_mbuf.c
==============================================================================
--- user/andre/mbuf_staging/kern/uipc_mbuf.c Thu Oct 10 18:34:46 2013 (r256286)
+++ user/andre/mbuf_staging/kern/uipc_mbuf.c Thu Oct 10 19:03:35 2013 (r256287)
@@ -281,6 +281,28 @@ m_extadd(struct mbuf *mb, caddr_t buf, u
}
/*
+ * Associated an external reference counted buffer with an mbuf.
+ */
+void
+m_extaddref(struct mbuf *m, caddr_t buf, u_int size, u_int *ref_cnt,
+ int (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2)
+{
+
+ KASSERT(ref_cnt != NULL, ("%s: ref_cnt not provided", __func__));
+
+ atomic_add_int(ref_cnt, 1);
+ m->m_flags |= M_EXT;
+ m->m_ext.ext_buf = buf;
+ m->m_ext.ref_cnt = ref_cnt;
+ m->m_data = m->m_ext.ext_buf;
+ m->m_ext.ext_size = size;
+ m->m_ext.ext_free = freef;
+ m->m_ext.ext_arg1 = arg1;
+ m->m_ext.ext_arg2 = arg2;
+ m->m_ext.ext_type = EXT_EXTREF;
+}
+
+/*
* Non-directly-exported function to clean up after mbufs with M_EXT
* storage attached to them if the reference count hits 1.
*/
Modified: user/andre/mbuf_staging/sys/mbuf.h
==============================================================================
--- user/andre/mbuf_staging/sys/mbuf.h Thu Oct 10 18:34:46 2013 (r256286)
+++ user/andre/mbuf_staging/sys/mbuf.h Thu Oct 10 19:03:35 2013 (r256287)
@@ -516,238 +516,19 @@ extern uma_zone_t zone_ext_refcnt;
void mb_free_ext(struct mbuf *);
int m_pkthdr_init(struct mbuf *, int);
-
-static __inline int
-m_gettype(int size)
-{
- int type;
-
- switch (size) {
- case MSIZE:
- type = EXT_MBUF;
- break;
- case MCLBYTES:
- type = EXT_CLUSTER;
- break;
-#if MJUMPAGESIZE != MCLBYTES
- case MJUMPAGESIZE:
- type = EXT_JUMBOP;
- break;
-#endif
- case MJUM9BYTES:
- type = EXT_JUMBO9;
- break;
- case MJUM16BYTES:
- type = EXT_JUMBO16;
- break;
- default:
- panic("%s: invalid cluster size", __func__);
- }
-
- return (type);
-}
-
-/*
- * Associated an external reference counted buffer with an mbuf.
- */
-static __inline void
-m_extaddref(struct mbuf *m, caddr_t buf, u_int size, u_int *ref_cnt,
- int (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2)
-{
-
- KASSERT(ref_cnt != NULL, ("%s: ref_cnt not provided", __func__));
-
- atomic_add_int(ref_cnt, 1);
- m->m_flags |= M_EXT;
- m->m_ext.ext_buf = buf;
- m->m_ext.ref_cnt = ref_cnt;
- m->m_data = m->m_ext.ext_buf;
- m->m_ext.ext_size = size;
- m->m_ext.ext_free = freef;
- m->m_ext.ext_arg1 = arg1;
- m->m_ext.ext_arg2 = arg2;
- m->m_ext.ext_type = EXT_EXTREF;
-}
-
-static __inline uma_zone_t
-m_getzone(int size)
-{
- uma_zone_t zone;
-
- switch (size) {
- case MCLBYTES:
- zone = zone_clust;
- break;
-#if MJUMPAGESIZE != MCLBYTES
- case MJUMPAGESIZE:
- zone = zone_jumbop;
- break;
-#endif
- case MJUM9BYTES:
- zone = zone_jumbo9;
- break;
- case MJUM16BYTES:
- zone = zone_jumbo16;
- break;
- default:
- panic("%s: invalid cluster size", __func__);
- }
-
- return (zone);
-}
-
-/*
- * Initialize an mbuf with linear storage.
- *
- * Inline because the consumer text overhead will be roughly the same to
- * initialize or call a function with this many parameters and M_PKTHDR
- * should go away with constant propagation for !MGETHDR.
- */
-static __inline int
-m_init(struct mbuf *m, uma_zone_t zone, int size, int how, short type,
- int flags)
-{
- int error;
-
- m->m_next = NULL;
- m->m_nextpkt = NULL;
- m->m_data = m->m_dat;
- m->m_len = 0;
- m->m_flags = flags;
- m->m_type = type;
- if (flags & M_PKTHDR) {
- if ((error = m_pkthdr_init(m, how)) != 0)
- return (error);
- }
-
- return (0);
-}
-
-static __inline struct mbuf *
-m_get(int how, short type)
-{
- struct mb_args args;
-
- args.flags = 0;
- args.type = type;
- return (uma_zalloc_arg(zone_mbuf, &args, how));
-}
-
-/*
- * XXX This should be deprecated, very little use.
- */
-static __inline struct mbuf *
-m_getclr(int how, short type)
-{
- struct mbuf *m;
- struct mb_args args;
-
- args.flags = 0;
- args.type = type;
- m = uma_zalloc_arg(zone_mbuf, &args, how);
- if (m != NULL)
- bzero(m->m_data, MLEN);
- return (m);
-}
-
-static __inline struct mbuf *
-m_gethdr(int how, short type)
-{
- struct mb_args args;
-
- args.flags = M_PKTHDR;
- args.type = type;
- return (uma_zalloc_arg(zone_mbuf, &args, how));
-}
-
-static __inline struct mbuf *
-m_getcl(int how, short type, int flags)
-{
- struct mb_args args;
-
- args.flags = flags;
- args.type = type;
- return (uma_zalloc_arg(zone_pack, &args, how));
-}
-
-static __inline void
-m_clget(struct mbuf *m, int how)
-{
-
- if (m->m_flags & M_EXT)
- printf("%s: %p mbuf already has cluster\n", __func__, m);
- m->m_ext.ext_buf = (char *)NULL;
- uma_zalloc_arg(zone_clust, m, how);
- /*
- * On a cluster allocation failure, drain the packet zone and retry,
- * we might be able to loosen a few clusters up on the drain.
- */
- if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
- zone_drain(zone_pack);
- uma_zalloc_arg(zone_clust, m, how);
- }
-}
-
-/*
- * m_cljget() is different from m_clget() as it can allocate clusters without
- * attaching them to an mbuf. In that case the return value is the pointer
- * to the cluster of the requested size. If an mbuf was specified, it gets
- * the cluster attached to it and the return value can be safely ignored.
- * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
- */
-static __inline void *
-m_cljget(struct mbuf *m, int how, int size)
-{
- uma_zone_t zone;
-
- if (m && m->m_flags & M_EXT)
- printf("%s: %p mbuf already has cluster\n", __func__, m);
- if (m != NULL)
- m->m_ext.ext_buf = NULL;
-
- zone = m_getzone(size);
- return (uma_zalloc_arg(zone, m, how));
-}
-
-static __inline void
-m_cljset(struct mbuf *m, void *cl, int type)
-{
- uma_zone_t zone;
- int size;
-
- switch (type) {
- case EXT_CLUSTER:
- size = MCLBYTES;
- zone = zone_clust;
- break;
-#if MJUMPAGESIZE != MCLBYTES
- case EXT_JUMBOP:
- size = MJUMPAGESIZE;
- zone = zone_jumbop;
- break;
-#endif
- case EXT_JUMBO9:
- size = MJUM9BYTES;
- zone = zone_jumbo9;
- break;
- case EXT_JUMBO16:
- size = MJUM16BYTES;
- zone = zone_jumbo16;
- break;
- default:
- panic("%s: unknown cluster type", __func__);
- break;
- }
-
- m->m_data = m->m_ext.ext_buf = cl;
- m->m_ext.ext_free = m->m_ext.ext_arg1 = m->m_ext.ext_arg2 = NULL;
- m->m_ext.ext_size = size;
- m->m_ext.ext_type = type;
- m->m_ext.ext_flags = 0;
- m->m_ext.ref_cnt = uma_find_refcnt(zone, cl);
- m->m_flags |= M_EXT;
-
-}
+uma_zone_t m_getzone(int size);
+int m_gettype(int);
+void m_extaddref(struct mbuf *, caddr_t, u_int, u_int *,
+ int (*)(struct mbuf *, void *, void *), void *, void *);
+uma_zone_t m_getzone(int);
+int m_init(struct mbuf *, uma_zone_t, int, int, short, int);
+struct mbuf *m_get(int, short);
+struct mbuf *m_getclr(int, short);
+struct mbuf *m_gethdr(int, short);
+struct mbuf *m_getcl(int, short, int);
+void m_clget(struct mbuf *, int);
+void *m_cljget(struct mbuf *, int, int);
+void m_cljset(struct mbuf *, void *, int);
static __inline void
m_chtype(struct mbuf *m, short new_type)
More information about the svn-src-user
mailing list