kernel address space & auto-tuning

[Alan Cox]

Ian and others here,

Could you please test the attached patch?  However, before you apply
this patch, can you run

    sysctl vm.max_kernel_address

and record the output.  I'd like to know how this output changes after
the patch is applied.

Here is an explanation of what the patch does:

1. Recently, a #define for VM_KMEM_SIZE_SCALE was added to
arm/include/vmparam.h.  This is good.  However, it will create a problem
as soon as someone gets arm hardware with more than ~1.5GB of RAM.  This
patch introduces a cap on the size of the kmem submap so that booting on
future larger-memory systems won't fail.

2. It appears that arm is more like sparc64 than x86 in the respect that
the ending address of the kernel address space can vary from machine to
machine.  To be more precise, I'm talking about the kernel address space
into which we can dynamically map and unmap code/data and I'm not
including regions for device access or direct maps.  Thus, the current
#define for VM_MAX_KERNEL_ADDRESS on arm is really incorrect or at least
inconsistent with how this is defined on other architectures.  This
patch borrows from sparc64 in defining a global variable,
vm_max_kernel_address, rather than a constant #define to represent the
end of the kernel address space.

Thanks,
Alan

-------------- next part --------------
Index: arm/arm/machdep.c
===================================================================
--- arm/arm/machdep.c	(revision 246628)
+++ arm/arm/machdep.c	(working copy)
@@ -1178,7 +1178,6 @@ initarm(struct arm_boot_params *abp)
 	struct pv_addr kernel_l1pt;
 	struct pv_addr dpcpu;
 	vm_offset_t dtbp, freemempos, l2_start, lastaddr;
-	vm_offset_t pmap_bootstrap_lastaddr;
 	uint32_t memsize, l2size;
 	char *env;
 	void *kmdp;
@@ -1288,7 +1287,7 @@ initarm(struct arm_boot_params *abp)
 	availmem_regions_sz = curr;
 
 	/* Platform-specific initialisation */
-	pmap_bootstrap_lastaddr = initarm_lastaddr();
+	vm_max_kernel_address = initarm_lastaddr();
 
 	pcpu0_init();
 
@@ -1477,7 +1476,7 @@ initarm(struct arm_boot_params *abp)
 	arm_intrnames_init();
 	arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL);
 	arm_dump_avail_init(memsize, sizeof(dump_avail) / sizeof(dump_avail[0]));
-	pmap_bootstrap(freemempos, pmap_bootstrap_lastaddr, &kernel_l1pt);
+	pmap_bootstrap(freemempos, vm_max_kernel_address, &kernel_l1pt);
 	msgbufp = (void *)msgbufpv.pv_va;
 	msgbufinit(msgbufp, msgbufsize);
 	mutex_init();
Index: arm/arm/pmap-v6.c
===================================================================
--- arm/arm/pmap-v6.c	(revision 246628)
+++ arm/arm/pmap-v6.c	(working copy)
@@ -231,6 +231,8 @@ vm_paddr_t kernel_l1pa;
 
 vm_offset_t kernel_vm_end = 0;
 
+vm_offset_t vm_max_kernel_address;
+
 struct pmap kernel_pmap_store;
 
 static pt_entry_t *csrc_pte, *cdst_pte;
Index: arm/arm/pmap.c
===================================================================
--- arm/arm/pmap.c	(revision 246628)
+++ arm/arm/pmap.c	(working copy)
@@ -220,6 +220,8 @@ vm_paddr_t kernel_l1pa;
 
 vm_offset_t kernel_vm_end = 0;
 
+vm_offset_t vm_max_kernel_address;
+
 struct pmap kernel_pmap_store;
 
 static pt_entry_t *csrc_pte, *cdst_pte;
Index: arm/include/vmparam.h
===================================================================
--- arm/include/vmparam.h	(revision 246628)
+++ arm/include/vmparam.h	(working copy)
@@ -133,7 +133,7 @@
 #define VM_MIN_KERNEL_ADDRESS KERNBASE
 #endif
 
-#define VM_MAX_KERNEL_ADDRESS	0xffffffff
+#define	VM_MAX_KERNEL_ADDRESS	(vm_max_kernel_address)
 
 /*
  * Virtual size (bytes) for various kernel submaps.
@@ -145,6 +145,14 @@
 #define VM_KMEM_SIZE_SCALE	(2)
 #endif
 
+/*
+ * Ceiling on the size of the kmem submap: 60% of the kernel map.
+ */
+#ifndef VM_KMEM_SIZE_MAX
+#define	VM_KMEM_SIZE_MAX	((vm_max_kernel_address - \
+    VM_MIN_KERNEL_ADDRESS + 1) * 3 / 5)
+#endif
+
 #define MAXTSIZ 	(16*1024*1024)
 #ifndef DFLDSIZ
 #define DFLDSIZ         (128*1024*1024)
@@ -166,6 +174,8 @@
 #define UMA_MD_SMALL_ALLOC
 #endif /* ARM_USE_SMALL_ALLOC */
 
+extern vm_offset_t vm_max_kernel_address;
+
 #define	ZERO_REGION_SIZE	(64 * 1024)	/* 64KB */
 
 #endif	/* _MACHINE_VMPARAM_H_ */
Index: vm/vm_kern.c
===================================================================
--- vm/vm_kern.c	(revision 246628)
+++ vm/vm_kern.c	(working copy)
@@ -98,7 +98,7 @@ SYSCTL_ULONG(_vm, OID_AUTO, min_kernel_address, CT
     NULL, VM_MIN_KERNEL_ADDRESS, "Min kernel address");
 
 SYSCTL_ULONG(_vm, OID_AUTO, max_kernel_address, CTLFLAG_RD,
-#ifdef __sparc64__
+#if defined(__arm__) || defined(__sparc64__)
     &vm_max_kernel_address, 0,
 #else
     NULL, VM_MAX_KERNEL_ADDRESS,