svn commit: r328168 - in head/sys: amd64/include arm/include arm64/include compat/linuxkpi/common/src dev/efidev i386/include kern mips/include powerpc/include riscv/include sparc64/include sys vm
Nathan Whitehorn
nwhitehorn at FreeBSD.org
Fri Jan 19 17:46:34 UTC 2018
Author: nwhitehorn
Date: Fri Jan 19 17:46:31 2018
New Revision: 328168
URL: https://svnweb.freebsd.org/changeset/base/328168
Log:
Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the
kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.
As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.
Reviewed by: kib
Suggestions from: marius, alc, kib
Runtime tested on: amd64, powerpc64, powerpc, mips64
Modified:
head/sys/amd64/include/vmparam.h
head/sys/arm/include/vmparam.h
head/sys/arm64/include/vmparam.h
head/sys/compat/linuxkpi/common/src/linux_page.c
head/sys/dev/efidev/efirt.c
head/sys/i386/include/vmparam.h
head/sys/kern/subr_sfbuf.c
head/sys/mips/include/vmparam.h
head/sys/powerpc/include/vmparam.h
head/sys/riscv/include/vmparam.h
head/sys/sparc64/include/vmparam.h
head/sys/sys/sf_buf.h
head/sys/vm/vm_page.c
Modified: head/sys/amd64/include/vmparam.h
==============================================================================
--- head/sys/amd64/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/amd64/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -190,6 +190,7 @@
* because the result is not actually accessed until later, but the early
* vt fb startup needs to be reworked.
*/
+#define PMAP_HAS_DMAP 1
#define PHYS_TO_DMAP(x) ({ \
KASSERT(dmaplimit == 0 || (x) < dmaplimit, \
("physical address %#jx not covered by the DMAP", \
Modified: head/sys/arm/include/vmparam.h
==============================================================================
--- head/sys/arm/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/arm/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -187,6 +187,8 @@ extern vm_offset_t vm_max_kernel_address;
#define SFBUF
#define SFBUF_MAP
+#define PMAP_HAS_DMAP 0
+
#define DEVMAP_MAX_VADDR ARM_VECTORS_HIGH
#endif /* _MACHINE_VMPARAM_H_ */
Modified: head/sys/arm64/include/vmparam.h
==============================================================================
--- head/sys/arm64/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/arm64/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -176,6 +176,7 @@
#define VIRT_IN_DMAP(va) ((va) >= DMAP_MIN_ADDRESS && \
(va) < (dmap_max_addr))
+#define PMAP_HAS_DMAP 1
#define PHYS_TO_DMAP(pa) \
({ \
KASSERT(PHYS_IN_DMAP(pa), \
Modified: head/sys/compat/linuxkpi/common/src/linux_page.c
==============================================================================
--- head/sys/compat/linuxkpi/common/src/linux_page.c Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/compat/linuxkpi/common/src/linux_page.c Fri Jan 19 17:46:31 2018 (r328168)
@@ -63,19 +63,14 @@ __FBSDID("$FreeBSD$");
#include <linux/preempt.h>
#include <linux/fs.h>
-#if defined(__amd64__) || defined(__aarch64__) || defined(__riscv)
-#define LINUXKPI_HAVE_DMAP
-#else
-#undef LINUXKPI_HAVE_DMAP
-#endif
-
void *
linux_page_address(struct page *page)
{
if (page->object != kmem_object && page->object != kernel_object) {
-#ifdef LINUXKPI_HAVE_DMAP
- return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page)));
+#ifdef PHYS_TO_DMAP
+ return (PMAP_HAS_DMAP ?
+ ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))) : NULL);
#else
return (NULL);
#endif
@@ -87,7 +82,8 @@ linux_page_address(struct page *page)
vm_page_t
linux_alloc_pages(gfp_t flags, unsigned int order)
{
-#ifdef LINUXKPI_HAVE_DMAP
+#ifdef PHYS_TO_DMAP
+ KASSERT(PMAP_HAS_DMAP, ("Direct map unavailable"));
unsigned long npages = 1UL << order;
int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ |
VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL);
@@ -145,23 +141,27 @@ retry:
void
linux_free_pages(vm_page_t page, unsigned int order)
{
-#ifdef LINUXKPI_HAVE_DMAP
- unsigned long npages = 1UL << order;
- unsigned long x;
+#ifdef PHYS_TO_DMAP
+ if (PMAP_HAS_DMAP) {
+ unsigned long npages = 1UL << order;
+ unsigned long x;
- for (x = 0; x != npages; x++) {
- vm_page_t pgo = page + x;
+ for (x = 0; x != npages; x++) {
+ vm_page_t pgo = page + x;
- vm_page_lock(pgo);
- vm_page_free(pgo);
- vm_page_unlock(pgo);
- }
-#else
- vm_offset_t vaddr;
+ vm_page_lock(pgo);
+ vm_page_free(pgo);
+ vm_page_unlock(pgo);
+ }
+ } else {
+#endif
+ vm_offset_t vaddr;
- vaddr = (vm_offset_t)page_address(page);
+ vaddr = (vm_offset_t)page_address(page);
- linux_free_kmem(vaddr, order);
+ linux_free_kmem(vaddr, order);
+#ifdef PHYS_TO_DMAP
+ }
#endif
}
Modified: head/sys/dev/efidev/efirt.c
==============================================================================
--- head/sys/dev/efidev/efirt.c Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/dev/efidev/efirt.c Fri Jan 19 17:46:31 2018 (r328168)
@@ -114,6 +114,11 @@ efi_init(void)
printf("EFI systbl not available\n");
return (0);
}
+ if (!PMAP_HAS_DMAP) {
+ if (bootverbose)
+ printf("EFI systbl requires direct map\n");
+ return (0);
+ }
efi_systbl = (struct efi_systbl *)PHYS_TO_DMAP(efi_systbl_phys);
if (efi_systbl->st_hdr.th_sig != EFI_SYSTBL_SIG) {
efi_systbl = NULL;
Modified: head/sys/i386/include/vmparam.h
==============================================================================
--- head/sys/i386/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/i386/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -202,4 +202,6 @@
#define SFBUF_CPUSET
#define SFBUF_PROCESS_PAGE
+#define PMAP_HAS_DMAP 0
+
#endif /* _MACHINE_VMPARAM_H_ */
Modified: head/sys/kern/subr_sfbuf.c
==============================================================================
--- head/sys/kern/subr_sfbuf.c Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/kern/subr_sfbuf.c Fri Jan 19 17:46:31 2018 (r328168)
@@ -88,10 +88,8 @@ sf_buf_init(void *arg)
vm_offset_t sf_base;
int i;
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
- if (SFBUF_OPTIONAL_DIRECT_MAP)
+ if (PMAP_HAS_DMAP)
return;
-#endif
nsfbufs = NSFBUFS;
TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
@@ -119,10 +117,8 @@ sf_buf_alloc(struct vm_page *m, int flags)
struct sf_buf *sf;
int error;
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
- if (SFBUF_OPTIONAL_DIRECT_MAP)
+ if (PMAP_HAS_DMAP)
return ((struct sf_buf *)m);
-#endif
KASSERT(curthread->td_pinned > 0 || (flags & SFB_CPUPRIVATE) == 0,
("sf_buf_alloc(SFB_CPUPRIVATE): curthread not pinned"));
@@ -181,10 +177,8 @@ void
sf_buf_free(struct sf_buf *sf)
{
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
- if (SFBUF_OPTIONAL_DIRECT_MAP)
+ if (PMAP_HAS_DMAP)
return;
-#endif
mtx_lock(&sf_buf_lock);
sf->ref_count--;
@@ -205,10 +199,8 @@ void
sf_buf_ref(struct sf_buf *sf)
{
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
- if (SFBUF_OPTIONAL_DIRECT_MAP)
+ if (PMAP_HAS_DMAP)
return;
-#endif
mtx_lock(&sf_buf_lock);
KASSERT(sf->ref_count > 0, ("%s: sf %p not allocated", __func__, sf));
Modified: head/sys/mips/include/vmparam.h
==============================================================================
--- head/sys/mips/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/mips/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -189,6 +189,12 @@
#ifndef __mips_n64
#define SFBUF
#define SFBUF_MAP
+#define PMAP_HAS_DMAP 0
+#else
+#define PMAP_HAS_DMAP 1
#endif
+
+#define PHYS_TO_DMAP(x) MIPS_PHYS_TO_DIRECT(x)
+#define DMAP_TO_PHYS(x) MIPS_DIRECT_TO_PHYS(x)
#endif /* !_MACHINE_VMPARAM_H_ */
Modified: head/sys/powerpc/include/vmparam.h
==============================================================================
--- head/sys/powerpc/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/powerpc/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -37,6 +37,10 @@
#ifndef _MACHINE_VMPARAM_H_
#define _MACHINE_VMPARAM_H_
+#ifndef LOCORE
+#include <machine/md_var.h>
+#endif
+
#define USRSTACK SHAREDPAGE
#ifndef MAXTSIZ
@@ -236,17 +240,25 @@ struct pmap_physseg {
*/
#define SFBUF
#define SFBUF_NOMD
-#define SFBUF_OPTIONAL_DIRECT_MAP hw_direct_map
-#define SFBUF_PHYS_DMAP(x) (x)
/*
- * We (usually) have a direct map of all physical memory. All
- * uses of this macro must be gated by a check on hw_direct_map!
- * The location of the direct map may not be 1:1 in future, so use
- * of the macro is recommended; it may also grow an assert that hw_direct_map
- * is set.
+ * We (usually) have a direct map of all physical memory, so provide
+ * a macro to use to get the kernel VA address for a given PA. Returns
+ * 0 if the direct map is unavailable. The location of the direct map
+ * may not be 1:1 in future, so use of the macro is recommended.
*/
-#define PHYS_TO_DMAP(x) x
-#define DMAP_TO_PHYS(x) x
-
+#ifdef __powerpc64__
+#define DMAP_BASE_ADDRESS 0x0000000000000000UL
+#else
+#define DMAP_BASE_ADDRESS 0x00000000UL
+#endif
+
+#define PMAP_HAS_DMAP (hw_direct_map)
+#define PHYS_TO_DMAP(x) ({ \
+ KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \
+ (x) | DMAP_BASE_ADDRESS; })
+#define DMAP_TO_PHYS(x) ({ \
+ KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \
+ (x) &~ DMAP_BASE_ADDRESS; })
+
#endif /* _MACHINE_VMPARAM_H_ */
Modified: head/sys/riscv/include/vmparam.h
==============================================================================
--- head/sys/riscv/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/riscv/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -170,6 +170,7 @@
#define VIRT_IN_DMAP(va) ((va) >= DMAP_MIN_ADDRESS && \
(va) < (dmap_max_addr))
+#define PMAP_HAS_DMAP 1
#define PHYS_TO_DMAP(pa) \
({ \
KASSERT(PHYS_IN_DMAP(pa), \
Modified: head/sys/sparc64/include/vmparam.h
==============================================================================
--- head/sys/sparc64/include/vmparam.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/sparc64/include/vmparam.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -240,10 +240,12 @@ extern vm_offset_t vm_max_kernel_address;
*/
#define ZERO_REGION_SIZE PAGE_SIZE
+#include <machine/tlb.h>
+
#define SFBUF
#define SFBUF_MAP
-#define SFBUF_OPTIONAL_DIRECT_MAP dcache_color_ignore
-#include <machine/tlb.h>
-#define SFBUF_PHYS_DMAP(x) TLB_PHYS_TO_DIRECT(x)
+
+#define PMAP_HAS_DMAP dcache_color_ignore
+#define PHYS_TO_DMAP(x) (TLB_PHYS_TO_DIRECT(x))
#endif /* !_MACHINE_VMPARAM_H_ */
Modified: head/sys/sys/sf_buf.h
==============================================================================
--- head/sys/sys/sf_buf.h Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/sys/sf_buf.h Fri Jan 19 17:46:31 2018 (r328168)
@@ -77,9 +77,6 @@ struct sfstat { /* sendfile statistics */
* that do no invalidate cache on the rest of CPUs.
* SFBUF_NOMD This machine doesn't have machine/sf_buf.h
*
- * SFBUF_OPTIONAL_DIRECT_MAP Value of this define is used as boolean
- * variable that tells whether machine is
- * capable of direct map or not at runtime.
* SFBUF_MAP This machine provides its own sf_buf_map() and
* sf_buf_unmap().
* SFBUF_PROCESS_PAGE This machine provides sf_buf_process_page()
@@ -109,9 +106,6 @@ struct sf_buf;
#ifndef SFBUF_NOMD
#include <machine/sf_buf.h>
#endif
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
-#include <machine/md_var.h>
-#endif
#ifdef SFBUF
struct sf_buf *sf_buf_alloc(struct vm_page *, int);
@@ -121,9 +115,9 @@ void sf_buf_ref(struct sf_buf *);
static inline vm_offset_t
sf_buf_kva(struct sf_buf *sf)
{
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
- if (SFBUF_OPTIONAL_DIRECT_MAP)
- return (SFBUF_PHYS_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf)));
+#ifdef PMAP_HAS_DMAP
+ if (PMAP_HAS_DMAP)
+ return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf)));
#endif
return (sf->kva);
@@ -132,8 +126,8 @@ sf_buf_kva(struct sf_buf *sf)
static inline vm_page_t
sf_buf_page(struct sf_buf *sf)
{
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
- if (SFBUF_OPTIONAL_DIRECT_MAP)
+#ifdef PMAP_HAS_DMAP
+ if (PMAP_HAS_DMAP)
return ((vm_page_t)sf);
#endif
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Fri Jan 19 16:06:52 2018 (r328167)
+++ head/sys/vm/vm_page.c Fri Jan 19 17:46:31 2018 (r328168)
@@ -2937,7 +2937,7 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked)
{
#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
- if ((m->flags & PG_ZERO) != 0) {
+ if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {
uint64_t *p;
int i;
p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
More information about the svn-src-all
mailing list