PERFORCE change 124430 for review
Marko Zec
zec at FreeBSD.org
Tue Jul 31 16:45:45 UTC 2007
http://perforce.freebsd.org/chv.cgi?CH=124430
Change 124430 by zec at zec_tpx32 on 2007/07/31 16:45:43
Probably a futile optimization experiment: implement a simple
private memory allocator aimed at reducing TLB trashing by
placing all vnet data on 4M superpage(s). Unlike on FBSD 4.11,
I couldn't observe any performance gains by using such tricks,
so either all malloc()ed data in the kernel already now
automagically ends up on superpages, or the overhead of TLB
trashing is so low that it is lost in other noise...
Nevertheless, in order not to loose this code, I'm submitting
it now for possible later experimentation. For now, the code
is commented out by default.
Affected files ...
.. //depot/projects/vimage/src/sys/kern/kern_vimage.c#28 edit
Differences ...
==== //depot/projects/vimage/src/sys/kern/kern_vimage.c#28 (text+ko) ====
@@ -72,6 +72,44 @@
static int vnet_mod_constructor(struct vnet_modlink *);
static int vnet_mod_destructor(struct vnet_modlink *);
+#ifdef VI_PREALLOC_SIZE
+/*
+ * A private memory allocator can be enabled by setting VI_PREALLOC_SIZE
+ * to amount of memory (in bytes) to be reserved for the allocator at
+ * boot time. This pool is guaranteed to reside on a 4M superpage(s) on
+ * i386 and amd64, thus potentially reducing TLB trashing.
+ *
+ * So far I couldn't observe any significant performance impact of using
+ * this allocator vs. the standard malloc(), whereas in FreeBSD 4.11
+ * days I recall using "uninitialized data" storage vs. malloc() would
+ * be an instant win... Is it possible that these days all malloc'ed
+ * kernel storage is automagically placed on 4M superpages, so that this
+ * effort is redundant? Who knows... Therefore this code is disabled by
+ * default, so vi_alloc() and vi_free() simply resolve to standard
+ * malloc() and free().
+ */
+
+static void *vi_malloc(unsigned long, struct malloc_type *, int);
+static void vi_free(void *, struct malloc_type *);
+
+struct vi_mtrack {
+ LIST_ENTRY(vi_mtrack) vmt_le;
+ char *vmt_addr;
+ size_t vmt_size;
+ int vmt_flags;
+};
+
+static char vi_mpool[VI_PREALLOC_SIZE];
+static struct uma_zone *vi_mtrack_zone;
+static LIST_HEAD(, vi_mtrack) vi_mem_free_head;
+static LIST_HEAD(, vi_mtrack) vi_mem_alloc_head;
+static int vi_mpool_fail_cnt = 0;
+#else
+#define vi_malloc(addr, type, flags) malloc((addr), (type), (flags))
+#define vi_free(addr, type) free((addr), (type))
+#endif /* VI_PREALLOC_SIZE */
+
+/* XXX those must vanish */
struct vimage vimage_0;
struct vprocg vprocg_0;
struct vcpu vcpu_0;
@@ -82,7 +120,7 @@
struct vprocg_list_head vprocg_head;
struct vcpu_list_head vcpu_head;
-int last_vi_id = 0;
+static int last_vi_id = 0;
static TAILQ_HEAD(vnet_modlink_head, vnet_modlink) vnet_modlink_head;
@@ -105,7 +143,7 @@
break;
if (vml != NULL)
panic("attempt to register an already registered vnet module");
- vml = malloc(sizeof(struct vnet_modlink), M_VIMAGE, M_NOWAIT);
+ vml = vi_malloc(sizeof(struct vnet_modlink), M_VIMAGE, M_NOWAIT);
/*
* XXX we support only statically assigned module IDs at the time.
@@ -149,7 +187,7 @@
VNET_ITERLOOP_END();
TAILQ_REMOVE(&vnet_modlink_head, vml, vml_mod_le);
- free(vml, M_VIMAGE);
+ vi_free(vml, M_VIMAGE);
}
struct vimage *vnet2vimage(vnet)
@@ -451,27 +489,28 @@
*/
/* A brute force check whether there's enough mem for a new vimage */
- vip = malloc(127*1024, M_VIMAGE, M_NOWAIT); /* XXX aaaargh... */
+ vip = malloc(512*1024, M_VIMAGE, M_NOWAIT); /* XXX aaaargh... */
if (vip == NULL)
goto vi_alloc_done;
+ free(vip, M_VIMAGE);
- vip = realloc(vip, sizeof(struct vimage), M_VIMAGE, M_NOWAIT | M_ZERO);
+ vip = vi_malloc(sizeof(struct vimage), M_VIMAGE, M_NOWAIT | M_ZERO);
if (vip == NULL)
panic("vi_alloc: malloc failed for vimage \"%s\"\n", name);
vip->vi_id = last_vi_id++;
- vnet = malloc(sizeof(struct vnet), M_VNET, M_NOWAIT | M_ZERO);
+ vnet = vi_malloc(sizeof(struct vnet), M_VNET, M_NOWAIT | M_ZERO);
if (vnet == NULL)
panic("vi_alloc: malloc failed for vnet \"%s\"\n", name);
vip->v_vnet = vnet;
vnet->vnet_magic_n = VNET_MAGIC_N;
- vprocg = malloc(sizeof(struct vprocg), M_VPROCG, M_NOWAIT | M_ZERO);
+ vprocg = vi_malloc(sizeof(struct vprocg), M_VPROCG, M_NOWAIT | M_ZERO);
if (vprocg == NULL)
panic("vi_alloc: malloc failed for vprocg \"%s\"\n", name);
vip->v_procg = vprocg;
- vcpu = malloc(sizeof(struct vcpu), M_VCPU, M_NOWAIT | M_ZERO);
+ vcpu = vi_malloc(sizeof(struct vcpu), M_VCPU, M_NOWAIT | M_ZERO);
if (vcpu == NULL)
panic ("vi_alloc: malloc failed for vcpu \"%s\"\n", name);
vip->v_cpu = vcpu;
@@ -556,16 +595,16 @@
/* hopefully, we are finally OK to free the vnet container itself! */
vnet->vnet_magic_n = -1;
- free(vnet, M_VNET);
+ vi_free(vnet, M_VNET);
LIST_REMOVE(vprocg, vprocg_le);
- free(vprocg, M_VPROCG);
+ vi_free(vprocg, M_VPROCG);
LIST_REMOVE(vcpu, vcpu_le);
- free(vcpu, M_VCPU);
+ vi_free(vcpu, M_VCPU);
LIST_REMOVE(vip, vi_le);
- free(vip, M_VIMAGE);
+ vi_free(vip, M_VIMAGE);
}
static int vnet_mod_constructor(vml)
@@ -574,10 +613,10 @@
const struct vnet_modinfo *vmi = vml->vml_modinfo;
if (vml->vml_modinfo->vmi_struct_size) {
- void *mem = malloc(vmi->vmi_struct_size, M_VNET, M_NOWAIT);
+ void *mem = vi_malloc(vmi->vmi_struct_size, M_VNET,
+ M_NOWAIT | M_ZERO);
if (mem == NULL) /* XXX should return error, not panic */
panic("vi_alloc: malloc for %s\n", vmi->vmi_name);
- bzero(mem, vmi->vmi_struct_size);
curvnet->mod_data[vmi->vmi_id] = mem;
}
@@ -596,7 +635,7 @@
if (curvnet->mod_data[vml->vml_modinfo->vmi_id] == NULL)
panic("vi_destroy: %s\n",
vml->vml_modinfo->vmi_name);
- free(curvnet->mod_data[vml->vml_modinfo->vmi_id],
+ vi_free(curvnet->mod_data[vml->vml_modinfo->vmi_id],
M_VNET);
curvnet->mod_data[vml->vml_modinfo->vmi_id] = NULL;
}
@@ -607,6 +646,23 @@
static void
vi_init(void *unused)
{
+#ifdef VI_PREALLOC_SIZE
+ struct vi_mtrack *vmt;
+
+ /* Initialize our private memory allocator */
+ LIST_INIT(&vi_mem_free_head);
+ LIST_INIT(&vi_mem_alloc_head);
+ vi_mtrack_zone = uma_zcreate("vi_mtrack", sizeof(struct vi_mtrack),
+ NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
+ vmt = uma_zalloc(vi_mtrack_zone, M_NOWAIT);
+ vmt->vmt_addr = vi_mpool;
+ vmt->vmt_size = VI_PREALLOC_SIZE;
+ LIST_INSERT_HEAD(&vi_mem_free_head, vmt, vmt_le);
+#endif /* VI_PREALLOC_SIZE */
+
+ /* vnet module list is both forward and reverse traversable */
+ TAILQ_INIT(&vnet_modlink_head);
+
LIST_INIT(&vimage_head);
LIST_INIT(&vnet_head);
LIST_INIT(&vprocg_head);
@@ -624,8 +680,6 @@
vnet_0.vnet_magic_n = VNET_MAGIC_N;
- TAILQ_INIT(&vnet_modlink_head);
-
/* We MUST clear curvnet in vi_init_done before going SMP. */
curvnet = &vnet_0;
}
@@ -639,6 +693,77 @@
SYSINIT(vimage, SI_SUB_VIMAGE, SI_ORDER_FIRST, vi_init, NULL)
SYSINIT(vimage_done, SI_SUB_VIMAGE_DONE, SI_ORDER_FIRST, vi_init_done, NULL)
+#ifdef VI_PREALLOC_SIZE
+
+void *
+vi_malloc(unsigned long size, struct malloc_type *type, int flags)
+{
+ void *addr;
+ struct vi_mtrack *vmt = NULL;
+ struct vi_mtrack *vmt_iter;
+
+ /* Attempt to find a free chunk in our private pool */
+ LIST_FOREACH(vmt_iter, &vi_mem_free_head, vmt_le)
+ if (vmt_iter->vmt_size >= size &&
+ (vmt == NULL || vmt_iter->vmt_size < vmt->vmt_size)) {
+ vmt = vmt_iter;
+ /* Exact fit is an optimal choice, we are done. */
+ if (vmt_iter->vmt_size == size)
+ break;
+ }
+
+ /* Not (enough) free space in our pool, resort to malloc() */
+ if (vmt == NULL) {
+ if (vi_mpool_fail_cnt == 0)
+ printf("vi_mpool exhausted, consider increasing VI_PREALLOC_SIZE\n");
+ vi_mpool_fail_cnt++;
+ addr = malloc(size, type, flags);
+ return addr;
+ }
+
+ addr = vmt->vmt_addr;
+ if (vmt->vmt_size == size) {
+ /* Move the descriptor from free to allocated list */
+ LIST_REMOVE(vmt, vmt_le);
+ LIST_INSERT_HEAD(&vi_mem_alloc_head, vmt, vmt_le);
+ } else {
+ /* Shrink the existing free space block */
+ vmt->vmt_addr += size;
+ vmt->vmt_size -= size;
+
+ /* Create a new descriptor and place it on allocated list */
+ vmt = uma_zalloc(vi_mtrack_zone, M_NOWAIT);
+ vmt->vmt_addr = addr;
+ vmt->vmt_size = size;
+ LIST_INSERT_HEAD(&vi_mem_alloc_head, vmt, vmt_le);
+ }
+
+ bzero(addr, size);
+ return addr;
+}
+
+void
+vi_free(void *addr, struct malloc_type *type)
+{
+ struct vi_mtrack *vmt;
+
+ /* Attempt to find the chunk in our allocated pool */
+ LIST_FOREACH(vmt, &vi_mem_alloc_head, vmt_le)
+ if (vmt->vmt_addr == addr)
+ break;
+
+ /* Not found in our private pool, resort to free() */
+ if (vmt == NULL) {
+ free(addr, type);
+ return;
+ }
+
+ /* Move the descriptor from allocated to free list */
+ LIST_REMOVE(vmt, vmt_le);
+ LIST_INSERT_HEAD(&vi_mem_free_head, vmt, vmt_le);
+}
+#endif /* VI_PREALLOC_SIZE */
+
#ifdef DDB
static void
db_vnet_ptr(void *arg)
More information about the p4-projects
mailing list