svn commit: r314106 - in head/sys: compat/linuxkpi/common/include/linux compat/linuxkpi/common/src conf modules/linuxkpi
Hans Petter Selasky
hselasky at FreeBSD.org
Wed Feb 22 19:39:55 UTC 2017
Author: hselasky
Date: Wed Feb 22 19:39:54 2017
New Revision: 314106
URL: https://svnweb.freebsd.org/changeset/base/314106
Log:
Optimise unmapped LinuxKPI page allocations.
When allocating unmapped pages, take advantage of the direct map on
AMD64 to get the virtual address corresponding to a page. Else all
pages allocated must be mapped because sometimes the virtual address
of a page is requested.
Move all page allocation and deallocation code into an own C-file.
Add support for GFP_DMA32, GFP_KERNEL, GFP_ATOMIC and __GFP_ZERO
allocation flags.
Make a clear separation between mapped and unmapped allocations.
Obtained from: kmacy @
MFC after: 1 week
Sponsored by: Mellanox Technologies
Added:
head/sys/compat/linuxkpi/common/src/linux_page.c (contents, props changed)
Modified:
head/sys/compat/linuxkpi/common/include/linux/gfp.h
head/sys/conf/files
head/sys/modules/linuxkpi/Makefile
Modified: head/sys/compat/linuxkpi/common/include/linux/gfp.h
==============================================================================
--- head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Feb 22 19:31:02 2017 (r314105)
+++ head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Feb 22 19:39:54 2017 (r314106)
@@ -2,7 +2,7 @@
* Copyright (c) 2010 Isilon Systems, Inc.
* Copyright (c) 2010 iX Systems, Inc.
* Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -67,94 +67,106 @@
#define GFP_TEMPORARY M_NOWAIT
#define GFP_NATIVE_MASK (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_ZERO)
-static inline void *
-page_address(struct page *page)
+/*
+ * Resolve a page into a virtual address:
+ *
+ * NOTE: This function only works for pages allocated by the kernel.
+ */
+extern void *linux_page_address(struct page *);
+
+#define page_address(page) linux_page_address(page)
+
+/*
+ * Page management for unmapped pages:
+ */
+extern vm_page_t linux_alloc_pages(gfp_t flags, unsigned int order);
+extern void linux_free_pages(vm_page_t page, unsigned int order);
+
+static inline struct page *
+alloc_page(gfp_t flags)
{
- if (page->object != kmem_object && page->object != kernel_object)
- return (NULL);
- return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS +
- IDX_TO_OFF(page->pindex)));
+ return (linux_alloc_pages(flags, 0));
}
-static inline unsigned long
-linux_get_page(gfp_t mask)
+static inline struct page *
+alloc_pages(gfp_t flags, unsigned int order)
{
- return kmem_malloc(kmem_arena, PAGE_SIZE, mask);
+ return (linux_alloc_pages(flags, order));
}
-#define get_zeroed_page(mask) linux_get_page((mask) | M_ZERO)
-#define alloc_page(mask) virt_to_page(linux_get_page((mask)))
-#define __get_free_page(mask) linux_get_page((mask))
+static inline struct page *
+alloc_pages_node(int node_id, gfp_t flags, unsigned int order)
+{
+
+ return (linux_alloc_pages(flags, order));
+}
static inline void
-free_page(unsigned long page)
+__free_pages(struct page *page, unsigned int order)
{
- if (page == 0)
- return;
- kmem_free(kmem_arena, page, PAGE_SIZE);
+ linux_free_pages(page, order);
}
static inline void
-__free_page(struct page *m)
+__free_page(struct page *page)
{
- if (m->object != kmem_object)
- panic("__free_page: Freed page %p not allocated via wrappers.",
- m);
- kmem_free(kmem_arena, (vm_offset_t)page_address(m), PAGE_SIZE);
+ linux_free_pages(page, 0);
}
-static inline void
-__free_pages(struct page *m, unsigned int order)
+/*
+ * Page management for mapped pages:
+ */
+extern vm_offset_t linux_alloc_kmem(gfp_t flags, unsigned int order);
+extern void linux_free_kmem(vm_offset_t, unsigned int order);
+
+static inline vm_offset_t
+get_zeroed_page(gfp_t flags)
{
- size_t size;
- if (m == NULL)
- return;
- size = PAGE_SIZE << order;
- kmem_free(kmem_arena, (vm_offset_t)page_address(m), size);
+ return (linux_alloc_kmem(flags | __GFP_ZERO, 0));
}
-static inline void free_pages(uintptr_t addr, unsigned int order)
+static inline vm_offset_t
+__get_free_page(gfp_t flags)
{
- if (addr == 0)
- return;
- __free_pages(virt_to_page((void *)addr), order);
+
+ return (linux_alloc_kmem(flags, 0));
}
-/*
- * Alloc pages allocates directly from the buddy allocator on linux so
- * order specifies a power of two bucket of pages and the results
- * are expected to be aligned on the size as well.
- */
-static inline struct page *
-alloc_pages(gfp_t gfp_mask, unsigned int order)
+static inline vm_offset_t
+__get_free_pages(gfp_t flags, unsigned int order)
{
- unsigned long page;
- size_t size;
- size = PAGE_SIZE << order;
- page = kmem_alloc_contig(kmem_arena, size, gfp_mask,
- 0, ~(vm_paddr_t)0, size, 0, VM_MEMATTR_DEFAULT);
- if (page == 0)
- return (NULL);
- return (virt_to_page(page));
+ return (linux_alloc_kmem(flags, order));
}
-static inline uintptr_t __get_free_pages(gfp_t gfp_mask, unsigned int order)
+static inline void
+free_pages(uintptr_t addr, unsigned int order)
{
- struct page *page;
+ if (addr == 0)
+ return;
- page = alloc_pages(gfp_mask, order);
- if (page == NULL)
- return (0);
- return ((uintptr_t)page_address(page));
+ linux_free_kmem(addr, order);
}
-#define alloc_pages_node(node, mask, order) alloc_pages(mask, order)
+static inline void
+free_page(uintptr_t addr)
+{
+ if (addr == 0)
+ return;
+
+ linux_free_kmem(addr, 0);
+}
+
+static inline bool
+gfpflags_allow_blocking(const gfp_t gfp_flags)
+{
+ return ((gfp_flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK);
+}
#define kmalloc_node(chunk, mask, node) kmalloc(chunk, mask)
Added: head/sys/compat/linuxkpi/common/src/linux_page.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/compat/linuxkpi/common/src/linux_page.c Wed Feb 22 19:39:54 2017 (r314106)
@@ -0,0 +1,167 @@
+/*-
+ * Copyright (c) 2010 Isilon Systems, Inc.
+ * Copyright (c) 2016 Matt Macy (mmacy at nextbsd.org)
+ * Copyright (c) 2017 Mellanox Technologies, Ltd.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/rwlock.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+
+#include <machine/bus.h>
+
+#include <linux/gfp.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/vm_pageout.h>
+
+void *
+linux_page_address(struct page *page)
+{
+#ifdef __amd64__
+ return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page)));
+#else
+ if (page->object != kmem_object && page->object != kernel_object)
+ return (NULL);
+ return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS +
+ IDX_TO_OFF(page->pindex)));
+#endif
+}
+
+vm_page_t
+linux_alloc_pages(gfp_t flags, unsigned int order)
+{
+#ifdef __amd64__
+ unsigned long npages = 1UL << order;
+ int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ |
+ VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL);
+ vm_page_t page;
+
+ if (order == 0 && (flags & GFP_DMA32) == 0) {
+ page = vm_page_alloc(NULL, 0, req);
+ if (page == NULL)
+ return (NULL);
+ } else {
+ vm_paddr_t pmax = (flags & GFP_DMA32) ?
+ BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR;
+retry:
+ page = vm_page_alloc_contig(NULL, 0, req,
+ npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
+
+ if (page == NULL) {
+ if (flags & M_WAITOK) {
+ if (!vm_page_reclaim_contig(req,
+ npages, 0, pmax, PAGE_SIZE, 0)) {
+ VM_WAIT;
+ }
+ flags &= ~M_WAITOK;
+ goto retry;
+ }
+ return (NULL);
+ }
+ }
+ if (flags & M_ZERO) {
+ unsigned long x;
+
+ for (x = 0; x != npages; x++) {
+ vm_page_t pgo = page + x;
+
+ if ((pgo->flags & PG_ZERO) == 0)
+ pmap_zero_page(pgo);
+ }
+ }
+#else
+ vm_offset_t vaddr;
+ vm_page_t page;
+
+ vaddr = linux_alloc_kmem(flags, order);
+ if (vaddr == 0)
+ return (NULL);
+
+ page = PHYS_TO_VM_PAGE(vtophys((void *)vaddr));
+
+ KASSERT(vaddr == (vm_offset_t)page_address(page),
+ ("Page address mismatch"));
+#endif
+ return (page);
+}
+
+void
+linux_free_pages(vm_page_t page, unsigned int order)
+{
+#ifdef __amd64__
+ unsigned long npages = 1UL << order;
+ unsigned long x;
+
+ for (x = 0; x != npages; x++) {
+ vm_page_t pgo = page + x;
+
+ vm_page_lock(pgo);
+ vm_page_free(pgo);
+ vm_page_unlock(pgo);
+ }
+#else
+ vm_offset_t vaddr;
+
+ vaddr = (vm_offset_t)page_address(page);
+
+ linux_free_kmem(vaddr, order);
+#endif
+}
+
+vm_offset_t
+linux_alloc_kmem(gfp_t flags, unsigned int order)
+{
+ size_t size = ((size_t)PAGE_SIZE) << order;
+ vm_offset_t addr;
+
+ if ((flags & GFP_DMA32) == 0) {
+ addr = kmem_malloc(kmem_arena, size, flags & GFP_NATIVE_MASK);
+ } else {
+ addr = kmem_alloc_contig(kmem_arena, size,
+ flags & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR_32BIT,
+ PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
+ }
+ return (addr);
+}
+
+void
+linux_free_kmem(vm_offset_t addr, unsigned int order)
+{
+ size_t size = ((size_t)PAGE_SIZE) << order;
+
+ kmem_free(kmem_arena, addr, size);
+}
Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Wed Feb 22 19:31:02 2017 (r314105)
+++ head/sys/conf/files Wed Feb 22 19:39:54 2017 (r314106)
@@ -4286,6 +4286,8 @@ compat/linuxkpi/common/src/linux_current
compile-with "${LINUXKPI_C}"
compat/linuxkpi/common/src/linux_kthread.c optional compat_linuxkpi \
compile-with "${LINUXKPI_C}"
+compat/linuxkpi/common/src/linux_page.c optional compat_linuxkpi \
+ compile-with "${LINUXKPI_C}"
compat/linuxkpi/common/src/linux_pci.c optional compat_linuxkpi pci \
compile-with "${LINUXKPI_C}"
compat/linuxkpi/common/src/linux_tasklet.c optional compat_linuxkpi \
Modified: head/sys/modules/linuxkpi/Makefile
==============================================================================
--- head/sys/modules/linuxkpi/Makefile Wed Feb 22 19:31:02 2017 (r314105)
+++ head/sys/modules/linuxkpi/Makefile Wed Feb 22 19:39:54 2017 (r314106)
@@ -6,6 +6,7 @@ SRCS= linux_kmod.c \
linux_compat.c \
linux_current.c \
linux_kthread.c \
+ linux_page.c \
linux_pci.c \
linux_radix.c \
linux_rcu.c \
More information about the svn-src-all
mailing list