svn commit: r314905 - in head/sys: compat/linuxkpi/common/include/linux compat/linuxkpi/common/src conf modules/linuxkpi
Hans Petter Selasky
hselasky at FreeBSD.org
Wed Mar 8 11:09:28 UTC 2017
Author: hselasky
Date: Wed Mar 8 11:09:27 2017
New Revision: 314905
URL: https://svnweb.freebsd.org/changeset/base/314905
Log:
Cleanup the LinuxKPI slab implementation.
Put large functions into linux_slab.c instead of declaring them static
inline.
Add support for more memory allocation wrappers like kmalloc_array()
and __vmalloc().
Make sure either the M_WAITOK or the M_NOWAIT flag is set and mask
away unused memory allocation flags before calling FreeBSD's malloc()
routine.
Move kmalloc_node() definition to slab.h where it belongs.
Implement support for the SLAB_DESTROY_BY_RCU feature when creating a
kmem_cache which basically means kmem_cache memory is freed using
call_rcu().
MFC after: 1 week
Sponsored by: Mellanox Technologies
Added:
head/sys/compat/linuxkpi/common/src/linux_slab.c (contents, props changed)
Modified:
head/sys/compat/linuxkpi/common/include/linux/gfp.h
head/sys/compat/linuxkpi/common/include/linux/slab.h
head/sys/conf/files
head/sys/modules/linuxkpi/Makefile
Modified: head/sys/compat/linuxkpi/common/include/linux/gfp.h
==============================================================================
--- head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Mar 8 09:53:20 2017 (r314904)
+++ head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Mar 8 11:09:27 2017 (r314905)
@@ -168,8 +168,6 @@ gfpflags_allow_blocking(const gfp_t gfp_
return ((gfp_flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK);
}
-#define kmalloc_node(chunk, mask, node) kmalloc(chunk, mask)
-
#define SetPageReserved(page) do { } while (0) /* NOP */
#define ClearPageReserved(page) do { } while (0) /* NOP */
Modified: head/sys/compat/linuxkpi/common/include/linux/slab.h
==============================================================================
--- head/sys/compat/linuxkpi/common/include/linux/slab.h Wed Mar 8 09:53:20 2017 (r314904)
+++ head/sys/compat/linuxkpi/common/include/linux/slab.h Wed Mar 8 11:09:27 2017 (r314905)
@@ -2,7 +2,7 @@
* Copyright (c) 2010 Isilon Systems, Inc.
* Copyright (c) 2010 iX Systems, Inc.
* Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -41,20 +41,19 @@
MALLOC_DECLARE(M_KMALLOC);
-#define kmalloc(size, flags) malloc((size), M_KMALLOC, (flags))
-#define kvmalloc(size) kmalloc((size), 0)
-#define kzalloc(size, flags) kmalloc((size), M_ZERO | ((flags) ? (flags) : M_NOWAIT))
-#define kzalloc_node(size, flags, node) kzalloc(size, flags)
-#define kfree(ptr) free(__DECONST(void *, (ptr)), M_KMALLOC)
+#define kvmalloc(size) kmalloc(size, 0)
+#define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
+#define kzalloc_node(size, flags, node) kmalloc(size, (flags) | __GFP_ZERO)
#define kfree_const(ptr) kfree(ptr)
-#define krealloc(ptr, size, flags) realloc((ptr), (size), M_KMALLOC, (flags))
-#define kcalloc(n, size, flags) kmalloc((n) * (size), flags | M_ZERO)
-#define vzalloc(size) kzalloc(size, GFP_KERNEL | __GFP_NOWARN)
+#define kcalloc(n, size, flags) kmalloc((n) * (size), (flags) | __GFP_ZERO)
+#define vzalloc(size) __vmalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0)
#define vfree(arg) kfree(arg)
#define kvfree(arg) kfree(arg)
-#define vmalloc(size) kmalloc(size, GFP_KERNEL)
-#define vmalloc_node(size, node) kmalloc(size, GFP_KERNEL)
-
+#define vmalloc_node(size, node) __vmalloc(size, GFP_KERNEL, 0)
+#define vmalloc_user(size) __vmalloc(size, GFP_KERNEL | __GFP_ZERO, 0)
+#define vmalloc(size) __vmalloc(size, GFP_KERNEL, 0)
+#define __kmalloc(...) kmalloc(__VA_ARGS__)
+#define kmalloc_node(chunk, flags, n) kmalloc(chunk, flags)
/*
* Prefix some functions with linux_ to avoid namespace conflict
@@ -66,59 +65,94 @@ MALLOC_DECLARE(M_KMALLOC);
#define kmem_cache_free(...) linux_kmem_cache_free(__VA_ARGS__)
#define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__)
+typedef void linux_kmem_ctor_t (void *);
+
struct linux_kmem_cache {
- uma_zone_t cache_zone;
- void (*cache_ctor)(void *);
+ uma_zone_t cache_zone;
+ linux_kmem_ctor_t *cache_ctor;
+ unsigned cache_flags;
+ unsigned cache_size;
};
-#define SLAB_HWCACHE_ALIGN 0x0001
+#define SLAB_HWCACHE_ALIGN (1 << 0)
+#define SLAB_DESTROY_BY_RCU (1 << 1)
+#define SLAB_RECLAIM_ACCOUNT (1 << 2)
-static inline int
-linux_kmem_ctor(void *mem, int size, void *arg, int flags)
+static inline gfp_t
+linux_check_m_flags(gfp_t flags)
{
- void (*ctor)(void *);
+ const gfp_t m = M_NOWAIT | M_WAITOK;
- ctor = arg;
- ctor(mem);
+ /* make sure either M_NOWAIT or M_WAITOK is set */
+ if ((flags & m) == 0)
+ flags |= M_NOWAIT;
+ else if ((flags & m) == m)
+ flags &= ~M_WAITOK;
- return (0);
+ /* mask away LinuxKPI specific flags */
+ return (flags & GFP_NATIVE_MASK);
}
-static inline struct kmem_cache *
-linux_kmem_cache_create(char *name, size_t size, size_t align, u_long flags,
- void (*ctor)(void *))
+static inline void *
+kmalloc(size_t size, gfp_t flags)
{
- struct kmem_cache *c;
+ return (malloc(size, M_KMALLOC, linux_check_m_flags(flags)));
+}
- c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK);
- if (align)
- align--;
- if (flags & SLAB_HWCACHE_ALIGN)
- align = UMA_ALIGN_CACHE;
- c->cache_zone = uma_zcreate(name, size, ctor ? linux_kmem_ctor : NULL,
- NULL, NULL, NULL, align, 0);
- c->cache_ctor = ctor;
+static inline void *
+__vmalloc(size_t size, gfp_t flags, int other)
+{
+ return (malloc(size, M_KMALLOC, linux_check_m_flags(flags)));
+}
- return c;
+static inline void *
+kmalloc_array(size_t n, size_t size, gfp_t flags)
+{
+ if (size != 0 && n > (SIZE_MAX / size))
+ return (NULL);
+ return (malloc(n * size, M_KMALLOC, linux_check_m_flags(flags)));
}
static inline void *
-linux_kmem_cache_alloc(struct kmem_cache *c, int flags)
+krealloc(void *ptr, size_t size, gfp_t flags)
{
- return uma_zalloc_arg(c->cache_zone, c->cache_ctor, flags);
+ return (realloc(ptr, size, M_KMALLOC, linux_check_m_flags(flags)));
}
static inline void
-linux_kmem_cache_free(struct kmem_cache *c, void *m)
+kfree(const void *ptr)
+{
+ free(__DECONST(void *, ptr), M_KMALLOC);
+}
+
+extern struct linux_kmem_cache *linux_kmem_cache_create(const char *name,
+ size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor);
+
+static inline void *
+linux_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags)
+{
+ return (uma_zalloc_arg(c->cache_zone, c,
+ linux_check_m_flags(flags)));
+}
+
+static inline void *
+kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags)
{
- uma_zfree(c->cache_zone, m);
+ return (uma_zalloc_arg(c->cache_zone, c,
+ linux_check_m_flags(flags | M_ZERO)));
}
+extern void linux_kmem_cache_free_rcu(struct linux_kmem_cache *, void *);
+
static inline void
-linux_kmem_cache_destroy(struct kmem_cache *c)
+linux_kmem_cache_free(struct linux_kmem_cache *c, void *m)
{
- uma_zdestroy(c->cache_zone);
- free(c, M_KMALLOC);
+ if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU))
+ linux_kmem_cache_free_rcu(c, m);
+ else
+ uma_zfree(c->cache_zone, m);
}
-#endif /* _LINUX_SLAB_H_ */
+extern void linux_kmem_cache_destroy(struct linux_kmem_cache *);
+
+#endif /* _LINUX_SLAB_H_ */
Added: head/sys/compat/linuxkpi/common/src/linux_slab.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/compat/linuxkpi/common/src/linux_slab.c Wed Mar 8 11:09:27 2017 (r314905)
@@ -0,0 +1,128 @@
+/*-
+ * Copyright (c) 2017 Mellanox Technologies, Ltd.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <linux/slab.h>
+#include <linux/rcupdate.h>
+#include <linux/kernel.h>
+
+struct linux_kmem_rcu {
+ struct rcu_head rcu_head;
+ struct linux_kmem_cache *cache;
+};
+
+#define LINUX_KMEM_TO_RCU(c, m) \
+ ((struct linux_kmem_rcu *)((char *)(m) + \
+ (c)->cache_size - sizeof(struct linux_kmem_rcu)))
+
+#define LINUX_RCU_TO_KMEM(r) \
+ ((void *)((char *)(r) + sizeof(struct linux_kmem_rcu) - \
+ (r)->cache->cache_size))
+
+static int
+linux_kmem_ctor(void *mem, int size, void *arg, int flags)
+{
+ struct linux_kmem_cache *c = arg;
+
+ if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU)) {
+ struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, mem);
+
+ /* duplicate cache pointer */
+ rcu->cache = c;
+ }
+
+ /* check for constructor */
+ if (likely(c->cache_ctor != NULL))
+ c->cache_ctor(mem);
+
+ return (0);
+}
+
+static void
+linux_kmem_cache_free_rcu_callback(struct rcu_head *head)
+{
+ struct linux_kmem_rcu *rcu =
+ container_of(head, struct linux_kmem_rcu, rcu_head);
+
+ uma_zfree(rcu->cache->cache_zone, LINUX_RCU_TO_KMEM(rcu));
+}
+
+struct linux_kmem_cache *
+linux_kmem_cache_create(const char *name, size_t size, size_t align,
+ unsigned flags, linux_kmem_ctor_t *ctor)
+{
+ struct linux_kmem_cache *c;
+
+ c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK);
+
+ if (flags & SLAB_HWCACHE_ALIGN)
+ align = UMA_ALIGN_CACHE;
+ else if (align != 0)
+ align--;
+
+ if (flags & SLAB_DESTROY_BY_RCU) {
+ /* make room for RCU structure */
+ size = ALIGN(size, sizeof(void *));
+ size += sizeof(struct linux_kmem_rcu);
+
+ /* create cache_zone */
+ c->cache_zone = uma_zcreate(name, size,
+ linux_kmem_ctor, NULL, NULL, NULL,
+ align, UMA_ZONE_ZINIT);
+ } else {
+ /* create cache_zone */
+ c->cache_zone = uma_zcreate(name, size,
+ ctor ? linux_kmem_ctor : NULL, NULL,
+ NULL, NULL, align, 0);
+ }
+
+ c->cache_flags = flags;
+ c->cache_ctor = ctor;
+ c->cache_size = size;
+ return (c);
+}
+
+void
+linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m)
+{
+ struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m);
+
+ call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback);
+}
+
+void
+linux_kmem_cache_destroy(struct linux_kmem_cache *c)
+{
+ if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU)) {
+ /* make sure all free callbacks have been called */
+ rcu_barrier();
+ }
+
+ uma_zdestroy(c->cache_zone);
+ free(c, M_KMALLOC);
+}
Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Wed Mar 8 09:53:20 2017 (r314904)
+++ head/sys/conf/files Wed Mar 8 11:09:27 2017 (r314905)
@@ -4296,6 +4296,8 @@ compat/linuxkpi/common/src/linux_radix.c
compile-with "${LINUXKPI_C}"
compat/linuxkpi/common/src/linux_rcu.c optional compat_linuxkpi \
compile-with "${LINUXKPI_C} -I$S/contrib/ck/include"
+compat/linuxkpi/common/src/linux_slab.c optional compat_linuxkpi \
+ compile-with "${LINUXKPI_C}"
compat/linuxkpi/common/src/linux_usb.c optional compat_linuxkpi usb \
compile-with "${LINUXKPI_C}"
compat/linuxkpi/common/src/linux_work.c optional compat_linuxkpi \
Modified: head/sys/modules/linuxkpi/Makefile
==============================================================================
--- head/sys/modules/linuxkpi/Makefile Wed Mar 8 09:53:20 2017 (r314904)
+++ head/sys/modules/linuxkpi/Makefile Wed Mar 8 11:09:27 2017 (r314905)
@@ -10,6 +10,7 @@ SRCS= linux_kmod.c \
linux_pci.c \
linux_radix.c \
linux_rcu.c \
+ linux_slab.c \
linux_tasklet.c \
linux_idr.c \
linux_usb.c \
More information about the svn-src-all
mailing list