PERFORCE change 137268 for review
Marius Strobl
marius at FreeBSD.org
Sun Mar 9 21:55:51 UTC 2008
http://perforce.freebsd.org/chv.cgi?CH=137268
Change 137268 by marius at flak on 2008/03/09 21:55:08
IFC @137264
Affected files ...
.. //depot/projects/usiii/amd64/amd64/identcpu.c#4 integrate
.. //depot/projects/usiii/amd64/amd64/mp_machdep.c#2 integrate
.. //depot/projects/usiii/amd64/amd64/pmap.c#4 integrate
.. //depot/projects/usiii/amd64/conf/GENERIC#6 integrate
.. //depot/projects/usiii/amd64/include/pmap.h#3 integrate
.. //depot/projects/usiii/amd64/include/smp.h#3 integrate
.. //depot/projects/usiii/arm/arm/pmap.c#5 integrate
.. //depot/projects/usiii/arm/arm/uio_machdep.c#2 integrate
.. //depot/projects/usiii/arm/conf/AVILA#4 integrate
.. //depot/projects/usiii/arm/include/pmap.h#3 integrate
.. //depot/projects/usiii/arm/xscale/ixp425/ixp425_iic.c#2 integrate
.. //depot/projects/usiii/boot/i386/pxeldr/pxeldr.S#2 integrate
.. //depot/projects/usiii/compat/linux/linux_futex.c#2 integrate
.. //depot/projects/usiii/conf/NOTES#7 integrate
.. //depot/projects/usiii/conf/files#6 integrate
.. //depot/projects/usiii/conf/files.powerpc#5 integrate
.. //depot/projects/usiii/conf/kmod.mk#3 integrate
.. //depot/projects/usiii/conf/options#6 integrate
.. //depot/projects/usiii/conf/options.powerpc#4 integrate
.. //depot/projects/usiii/ddb/db_command.c#3 integrate
.. //depot/projects/usiii/ddb/db_examine.c#3 integrate
.. //depot/projects/usiii/ddb/db_input.c#3 integrate
.. //depot/projects/usiii/dev/acpica/acpi_cpu.c#5 integrate
.. //depot/projects/usiii/dev/agp/agp_nvidia.c#2 integrate
.. //depot/projects/usiii/dev/ata/ata-chipset.c#5 integrate
.. //depot/projects/usiii/dev/ata/ata-pci.h#4 integrate
.. //depot/projects/usiii/dev/bge/if_bge.c#6 integrate
.. //depot/projects/usiii/dev/bge/if_bgereg.h#3 integrate
.. //depot/projects/usiii/dev/cmx/cmx.c#1 branch
.. //depot/projects/usiii/dev/cmx/cmx_pccard.c#1 branch
.. //depot/projects/usiii/dev/cmx/cmxreg.h#1 branch
.. //depot/projects/usiii/dev/cmx/cmxvar.h#1 branch
.. //depot/projects/usiii/dev/gem/if_gem.c#6 integrate
.. //depot/projects/usiii/dev/hptrr/hptrr_os_bsd.c#3 integrate
.. //depot/projects/usiii/dev/hptrr/hptrr_osm_bsd.c#3 integrate
.. //depot/projects/usiii/dev/hptrr/ldm.h#3 integrate
.. //depot/projects/usiii/dev/hptrr/osm.h#3 integrate
.. //depot/projects/usiii/dev/ic/quicc.h#1 branch
.. //depot/projects/usiii/dev/igb/if_igb.c#2 integrate
.. //depot/projects/usiii/dev/lge/if_lge.c#4 integrate
.. //depot/projects/usiii/dev/mii/brgphy.c#3 integrate
.. //depot/projects/usiii/dev/mii/ciphy.c#2 integrate
.. //depot/projects/usiii/dev/mii/miidevs#4 integrate
.. //depot/projects/usiii/dev/mii/rgephy.c#3 integrate
.. //depot/projects/usiii/dev/msk/if_msk.c#5 integrate
.. //depot/projects/usiii/dev/nfe/if_nfe.c#5 integrate
.. //depot/projects/usiii/dev/nfe/if_nfereg.h#2 integrate
.. //depot/projects/usiii/dev/nfe/if_nfevar.h#2 integrate
.. //depot/projects/usiii/dev/pccard/pccarddevs#3 integrate
.. //depot/projects/usiii/dev/quicc/quicc_bfe.h#1 branch
.. //depot/projects/usiii/dev/quicc/quicc_bfe_ocp.c#1 branch
.. //depot/projects/usiii/dev/quicc/quicc_bus.h#1 branch
.. //depot/projects/usiii/dev/quicc/quicc_core.c#1 branch
.. //depot/projects/usiii/dev/re/if_re.c#4 integrate
.. //depot/projects/usiii/dev/scc/scc_bfe.h#2 integrate
.. //depot/projects/usiii/dev/scc/scc_bfe_quicc.c#1 branch
.. //depot/projects/usiii/dev/scc/scc_bus.h#2 integrate
.. //depot/projects/usiii/dev/scc/scc_dev_quicc.c#1 branch
.. //depot/projects/usiii/dev/tsec/if_tsec.c#1 branch
.. //depot/projects/usiii/dev/tsec/if_tsec.h#1 branch
.. //depot/projects/usiii/dev/tsec/if_tsecreg.h#1 branch
.. //depot/projects/usiii/dev/uart/uart.h#2 integrate
.. //depot/projects/usiii/dev/uart/uart_bus_ocp.c#1 branch
.. //depot/projects/usiii/dev/uart/uart_bus_scc.c#2 integrate
.. //depot/projects/usiii/dev/uart/uart_cpu_powerpc.c#3 integrate
.. //depot/projects/usiii/dev/uart/uart_dev_quicc.c#1 branch
.. //depot/projects/usiii/dev/usb/ehci.c#4 integrate
.. //depot/projects/usiii/dev/usb/if_axe.c#4 integrate
.. //depot/projects/usiii/dev/usb/usbdevs#6 integrate
.. //depot/projects/usiii/dev/usb/uslcom.c#1 branch
.. //depot/projects/usiii/fs/nwfs/nwfs_node.c#4 integrate
.. //depot/projects/usiii/fs/nwfs/nwfs_subr.c#2 integrate
.. //depot/projects/usiii/fs/smbfs/smbfs.h#2 integrate
.. //depot/projects/usiii/fs/smbfs/smbfs_node.c#4 integrate
.. //depot/projects/usiii/fs/smbfs/smbfs_vfsops.c#4 integrate
.. //depot/projects/usiii/geom/part/g_part_vtoc8.c#1 branch
.. //depot/projects/usiii/i386/conf/GENERIC#6 integrate
.. //depot/projects/usiii/i386/cpufreq/est.c#3 integrate
.. //depot/projects/usiii/i386/i386/identcpu.c#3 integrate
.. //depot/projects/usiii/i386/i386/mp_machdep.c#3 integrate
.. //depot/projects/usiii/i386/include/_types.h#2 integrate
.. //depot/projects/usiii/i386/include/float.h#2 integrate
.. //depot/projects/usiii/i386/include/smp.h#3 integrate
.. //depot/projects/usiii/ia64/ia64/mp_machdep.c#2 integrate
.. //depot/projects/usiii/kern/init_main.c#5 integrate
.. //depot/projects/usiii/kern/init_sysent.c#4 integrate
.. //depot/projects/usiii/kern/kern_cpuset.c#1 branch
.. //depot/projects/usiii/kern/kern_descrip.c#5 integrate
.. //depot/projects/usiii/kern/kern_lock.c#5 integrate
.. //depot/projects/usiii/kern/kern_priv.c#2 integrate
.. //depot/projects/usiii/kern/kern_shutdown.c#4 integrate
.. //depot/projects/usiii/kern/kern_sig.c#4 integrate
.. //depot/projects/usiii/kern/kern_thread.c#5 integrate
.. //depot/projects/usiii/kern/sched_4bsd.c#5 integrate
.. //depot/projects/usiii/kern/sched_ule.c#6 integrate
.. //depot/projects/usiii/kern/subr_bus.c#4 integrate
.. //depot/projects/usiii/kern/subr_smp.c#5 integrate
.. //depot/projects/usiii/kern/subr_witness.c#5 integrate
.. //depot/projects/usiii/kern/syscalls.c#4 integrate
.. //depot/projects/usiii/kern/syscalls.master#5 integrate
.. //depot/projects/usiii/kern/systrace_args.c#4 integrate
.. //depot/projects/usiii/kern/vfs_bio.c#6 integrate
.. //depot/projects/usiii/kern/vfs_subr.c#6 integrate
.. //depot/projects/usiii/modules/Makefile#7 integrate
.. //depot/projects/usiii/modules/cmx/Makefile#1 branch
.. //depot/projects/usiii/modules/cxgb/cxgb_t3fw/Makefile#2 integrate
.. //depot/projects/usiii/modules/geom/Makefile#4 integrate
.. //depot/projects/usiii/modules/geom/geom_apple/Makefile#2 delete
.. //depot/projects/usiii/modules/geom/geom_gpt/Makefile#2 delete
.. //depot/projects/usiii/modules/geom/geom_part/Makefile#1 branch
.. //depot/projects/usiii/modules/uslcom/Makefile#1 branch
.. //depot/projects/usiii/modules/zfs/Makefile#4 integrate
.. //depot/projects/usiii/net/bpf_filter.c#4 integrate
.. //depot/projects/usiii/net/if.c#3 integrate
.. //depot/projects/usiii/net/if_ethersubr.c#4 integrate
.. //depot/projects/usiii/net/if_gif.c#3 integrate
.. //depot/projects/usiii/netgraph/ng_base.c#5 integrate
.. //depot/projects/usiii/netgraph/ng_ksocket.c#2 integrate
.. //depot/projects/usiii/netgraph/ng_l2tp.c#4 integrate
.. //depot/projects/usiii/netgraph/ng_nat.c#2 integrate
.. //depot/projects/usiii/netgraph/ng_nat.h#2 integrate
.. //depot/projects/usiii/netgraph/ng_pppoe.c#6 integrate
.. //depot/projects/usiii/netinet/in.h#2 integrate
.. //depot/projects/usiii/netinet/in_pcb.c#4 integrate
.. //depot/projects/usiii/netinet/ip_fw_nat.c#2 integrate
.. //depot/projects/usiii/netinet/ip_fw_pfil.c#3 integrate
.. //depot/projects/usiii/netinet/libalias/alias.c#2 integrate
.. //depot/projects/usiii/netinet/libalias/alias_db.c#2 integrate
.. //depot/projects/usiii/netinet/libalias/alias_ftp.c#2 integrate
.. //depot/projects/usiii/netinet/libalias/alias_irc.c#2 integrate
.. //depot/projects/usiii/netinet/libalias/alias_local.h#2 integrate
.. //depot/projects/usiii/netinet/libalias/alias_proxy.c#2 integrate
.. //depot/projects/usiii/netinet/libalias/alias_smedia.c#2 integrate
.. //depot/projects/usiii/netinet/tcp_input.c#4 integrate
.. //depot/projects/usiii/netinet/tcp_output.c#3 integrate
.. //depot/projects/usiii/netipsec/key.c#2 integrate
.. //depot/projects/usiii/netipsec/key_debug.c#2 integrate
.. //depot/projects/usiii/netncp/ncp_conn.h#2 integrate
.. //depot/projects/usiii/netsmb/smb_conn.h#2 integrate
.. //depot/projects/usiii/nfs4client/nfs4_idmap.c#3 integrate
.. //depot/projects/usiii/nfsclient/nfs_vfsops.c#5 integrate
.. //depot/projects/usiii/nfsserver/nfs_serv.c#4 integrate
.. //depot/projects/usiii/pci/if_rl.c#3 integrate
.. //depot/projects/usiii/pci/if_rlreg.h#4 integrate
.. //depot/projects/usiii/powerpc/aim/locore.S#3 integrate
.. //depot/projects/usiii/powerpc/aim/machdep.c#3 integrate
.. //depot/projects/usiii/powerpc/aim/swtch.S#2 integrate
.. //depot/projects/usiii/powerpc/aim/trap.c#2 integrate
.. //depot/projects/usiii/powerpc/aim/trap_subr.S#2 integrate
.. //depot/projects/usiii/powerpc/aim/vm_machdep.c#2 integrate
.. //depot/projects/usiii/powerpc/booke/clock.c#1 branch
.. //depot/projects/usiii/powerpc/booke/copyinout.c#1 branch
.. //depot/projects/usiii/powerpc/booke/interrupt.c#1 branch
.. //depot/projects/usiii/powerpc/booke/locore.S#1 branch
.. //depot/projects/usiii/powerpc/booke/machdep.c#1 branch
.. //depot/projects/usiii/powerpc/booke/pmap.c#1 branch
.. //depot/projects/usiii/powerpc/booke/support.S#1 branch
.. //depot/projects/usiii/powerpc/booke/swtch.S#1 branch
.. //depot/projects/usiii/powerpc/booke/trap.c#1 branch
.. //depot/projects/usiii/powerpc/booke/trap_subr.S#1 branch
.. //depot/projects/usiii/powerpc/booke/uio_machdep.c#1 branch
.. //depot/projects/usiii/powerpc/booke/vm_machdep.c#1 branch
.. //depot/projects/usiii/powerpc/conf/MPC85XX#1 branch
.. //depot/projects/usiii/powerpc/conf/NOTES#6 integrate
.. //depot/projects/usiii/powerpc/include/frame.h#2 integrate
.. //depot/projects/usiii/powerpc/include/hid.h#3 integrate
.. //depot/projects/usiii/powerpc/include/intr_machdep.h#3 integrate
.. //depot/projects/usiii/powerpc/include/kdb.h#2 integrate
.. //depot/projects/usiii/powerpc/include/md_var.h#3 integrate
.. //depot/projects/usiii/powerpc/include/mmuvar.h#2 integrate
.. //depot/projects/usiii/powerpc/include/ocpbus.h#1 branch
.. //depot/projects/usiii/powerpc/include/openpicvar.h#3 integrate
.. //depot/projects/usiii/powerpc/include/pcb.h#2 integrate
.. //depot/projects/usiii/powerpc/include/pcpu.h#2 integrate
.. //depot/projects/usiii/powerpc/include/pmap.h#2 integrate
.. //depot/projects/usiii/powerpc/include/psl.h#2 integrate
.. //depot/projects/usiii/powerpc/include/pte.h#2 integrate
.. //depot/projects/usiii/powerpc/include/sf_buf.h#2 integrate
.. //depot/projects/usiii/powerpc/include/spr.h#3 integrate
.. //depot/projects/usiii/powerpc/include/tlb.h#1 branch
.. //depot/projects/usiii/powerpc/include/trap.h#2 integrate
.. //depot/projects/usiii/powerpc/include/trap_aim.h#1 branch
.. //depot/projects/usiii/powerpc/include/trap_booke.h#1 branch
.. //depot/projects/usiii/powerpc/include/vmparam.h#4 integrate
.. //depot/projects/usiii/powerpc/mpc85xx/nexus.c#1 branch
.. //depot/projects/usiii/powerpc/mpc85xx/ocpbus.c#1 branch
.. //depot/projects/usiii/powerpc/mpc85xx/ocpbus.h#1 branch
.. //depot/projects/usiii/powerpc/mpc85xx/opic.c#1 branch
.. //depot/projects/usiii/powerpc/mpc85xx/pci_ocp.c#1 branch
.. //depot/projects/usiii/powerpc/powermac/openpic_macio.c#3 integrate
.. //depot/projects/usiii/powerpc/powerpc/autoconf.c#2 integrate
.. //depot/projects/usiii/powerpc/powerpc/cpu.c#3 integrate
.. //depot/projects/usiii/powerpc/powerpc/db_trace.c#3 integrate
.. //depot/projects/usiii/powerpc/powerpc/gdb_machdep.c#2 integrate
.. //depot/projects/usiii/powerpc/powerpc/genassym.c#4 integrate
.. //depot/projects/usiii/powerpc/powerpc/intr_machdep.c#4 integrate
.. //depot/projects/usiii/powerpc/powerpc/mp_machdep.c#2 integrate
.. //depot/projects/usiii/powerpc/powerpc/openpic.c#3 integrate
.. //depot/projects/usiii/powerpc/powerpc/pic_if.m#3 integrate
.. //depot/projects/usiii/powerpc/psim/openpic_iobus.c#3 integrate
.. //depot/projects/usiii/security/audit/audit.c#5 integrate
.. //depot/projects/usiii/security/audit/audit_bsm_klib.c#5 integrate
.. //depot/projects/usiii/security/audit/audit_private.h#3 integrate
.. //depot/projects/usiii/security/audit/audit_syscalls.c#4 integrate
.. //depot/projects/usiii/security/audit/audit_trigger.c#2 integrate
.. //depot/projects/usiii/security/audit/audit_worker.c#5 integrate
.. //depot/projects/usiii/sparc64/include/tlb.h#3 integrate
.. //depot/projects/usiii/sparc64/sparc64/cache.c#2 integrate
.. //depot/projects/usiii/sparc64/sparc64/cheetah.c#3 integrate
.. //depot/projects/usiii/sparc64/sparc64/iommu.c#2 integrate
.. //depot/projects/usiii/sparc64/sparc64/mp_machdep.c#2 integrate
.. //depot/projects/usiii/sparc64/sparc64/pmap.c#4 integrate
.. //depot/projects/usiii/sparc64/sparc64/spitfire.c#2 integrate
.. //depot/projects/usiii/sparc64/sparc64/tlb.c#2 integrate
.. //depot/projects/usiii/sun4v/sun4v/mp_machdep.c#2 integrate
.. //depot/projects/usiii/sys/_types.h#2 integrate
.. //depot/projects/usiii/sys/buf.h#4 integrate
.. //depot/projects/usiii/sys/bufobj.h#2 integrate
.. //depot/projects/usiii/sys/bus.h#3 integrate
.. //depot/projects/usiii/sys/cpuset.h#1 branch
.. //depot/projects/usiii/sys/fcntl.h#3 integrate
.. //depot/projects/usiii/sys/lockmgr.h#5 integrate
.. //depot/projects/usiii/sys/mount.h#3 integrate
.. //depot/projects/usiii/sys/param.h#6 integrate
.. //depot/projects/usiii/sys/pcpu.h#4 integrate
.. //depot/projects/usiii/sys/proc.h#6 integrate
.. //depot/projects/usiii/sys/sched.h#2 integrate
.. //depot/projects/usiii/sys/smp.h#3 integrate
.. //depot/projects/usiii/sys/syscall.h#4 integrate
.. //depot/projects/usiii/sys/syscall.mk#4 integrate
.. //depot/projects/usiii/sys/sysproto.h#4 integrate
.. //depot/projects/usiii/sys/types.h#3 integrate
.. //depot/projects/usiii/sys/vnode.h#5 integrate
.. //depot/projects/usiii/sys/vtoc.h#1 branch
.. //depot/projects/usiii/ufs/ffs/ffs_vfsops.c#5 integrate
.. //depot/projects/usiii/ufs/ufs/extattr.h#2 integrate
.. //depot/projects/usiii/ufs/ufs/ufs_extattr.c#3 integrate
.. //depot/projects/usiii/vm/vm_fault.c#5 integrate
.. //depot/projects/usiii/vm/vm_map.h#3 integrate
Differences ...
==== //depot/projects/usiii/amd64/amd64/identcpu.c#4 (text+ko) ====
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/amd64/amd64/identcpu.c,v 1.157 2008/02/02 23:17:27 das Exp $");
+__FBSDID("$FreeBSD: src/sys/amd64/amd64/identcpu.c,v 1.158 2008/03/02 07:58:40 jeff Exp $");
#include "opt_cpu.h"
@@ -97,6 +97,10 @@
{ "Sledgehammer", CPUCLASS_K8 }, /* CPU_SLEDGEHAMMER */
};
+int cpu_cores;
+int cpu_logical;
+
+
extern int pq_l2size;
extern int pq_l2nways;
@@ -360,11 +364,13 @@
if ((regs[0] & 0x1f) != 0)
cmp = ((regs[0] >> 26) & 0x3f) + 1;
}
+ cpu_cores = cmp;
+ cpu_logical = htt / cmp;
if (cmp > 1)
printf("\n Cores per package: %d", cmp);
if ((htt / cmp) > 1)
printf("\n Logical CPUs per core: %d",
- htt / cmp);
+ cpu_logical);
}
}
/* Avoid ugly blank lines: only print newline when we have to. */
==== //depot/projects/usiii/amd64/amd64/mp_machdep.c#2 (text+ko) ====
@@ -25,7 +25,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/amd64/amd64/mp_machdep.c,v 1.287 2007/08/02 21:17:58 peter Exp $");
+__FBSDID("$FreeBSD: src/sys/amd64/amd64/mp_machdep.c,v 1.288 2008/03/02 07:58:40 jeff Exp $");
#include "opt_cpu.h"
#include "opt_kstack_pages.h"
@@ -83,12 +83,6 @@
extern struct pcpu __pcpu[];
-/*
- * CPU topology map datastructures for HTT.
- */
-static struct cpu_group mp_groups[MAXCPU];
-static struct cpu_top mp_top;
-
/* AP uses this during bootstrap. Do not staticize. */
char *bootSTK;
static int bootAP;
@@ -182,40 +176,38 @@
mem_range_softc.mr_op->initAP(&mem_range_softc);
}
-void
-mp_topology(void)
+struct cpu_group *
+cpu_topo(void)
{
- struct cpu_group *group;
- int apic_id;
- int groups;
- int cpu;
-
- /* Build the smp_topology map. */
- /* Nothing to do if there is no HTT support. */
- if (hyperthreading_cpus <= 1)
- return;
- group = &mp_groups[0];
- groups = 1;
- for (cpu = 0, apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) {
- if (!cpu_info[apic_id].cpu_present)
- continue;
- /*
- * If the current group has members and we're not a logical
- * cpu, create a new group.
- */
- if (group->cg_count != 0 &&
- (apic_id % hyperthreading_cpus) == 0) {
- group++;
- groups++;
- }
- group->cg_count++;
- group->cg_mask |= 1 << cpu;
- cpu++;
+ if (cpu_cores == 0)
+ cpu_cores = 1;
+ if (cpu_logical == 0)
+ cpu_logical = 1;
+ if (mp_ncpus % (cpu_cores * cpu_logical) != 0) {
+ printf("WARNING: Non-uniform processors.\n");
+ printf("WARNING: Using suboptimal topology.\n");
+ return (smp_topo_none());
}
-
- mp_top.ct_count = groups;
- mp_top.ct_group = mp_groups;
- smp_topology = &mp_top;
+ /*
+ * No multi-core or hyper-threaded.
+ */
+ if (cpu_logical * cpu_cores == 1)
+ return (smp_topo_none());
+ /*
+ * Only HTT no multi-core.
+ */
+ if (cpu_logical > 1 && cpu_cores == 1)
+ return (smp_topo_1level(CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
+ /*
+ * Only multi-core no HTT.
+ */
+ if (cpu_cores > 1 && cpu_logical == 1)
+ return (smp_topo_1level(CG_SHARE_NONE, cpu_cores, 0));
+ /*
+ * Both HTT and multi-core.
+ */
+ return (smp_topo_2level(CG_SHARE_NONE, cpu_cores,
+ CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
}
/*
@@ -409,9 +401,6 @@
}
set_interrupt_apic_ids();
-
- /* Last, setup the cpu topology now that we have probed CPUs */
- mp_topology();
}
==== //depot/projects/usiii/amd64/amd64/pmap.c#4 (text+ko) ====
@@ -7,7 +7,7 @@
* All rights reserved.
* Copyright (c) 2003 Peter Wemm
* All rights reserved.
- * Copyright (c) 2005 Alan L. Cox <alc at cs.rice.edu>
+ * Copyright (c) 2005-2008 Alan L. Cox <alc at cs.rice.edu>
* All rights reserved.
*
* This code is derived from software contributed to Berkeley by
@@ -77,7 +77,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.605 2008/01/17 18:25:51 alc Exp $");
+__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.606 2008/03/04 18:50:15 alc Exp $");
/*
* Manages physical address maps.
@@ -107,10 +107,12 @@
#include "opt_msgbuf.h"
#include "opt_pmap.h"
+#include "opt_vm.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
+#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mman.h>
@@ -134,6 +136,7 @@
#include <vm/vm_extern.h>
#include <vm/vm_pageout.h>
#include <vm/vm_pager.h>
+#include <vm/vm_reserv.h>
#include <vm/uma.h>
#include <machine/cpu.h>
@@ -162,6 +165,9 @@
#define PV_STAT(x) do { } while (0)
#endif
+#define pa_index(pa) ((pa) >> PDRSHIFT)
+#define pa_to_pvh(pa) (&pv_table[pa_index(pa)])
+
struct pmap kernel_pmap_store;
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
@@ -173,6 +179,12 @@
vm_offset_t kernel_vm_end;
pt_entry_t pg_nx;
+SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
+
+static int pg_ps_enabled;
+SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RD, &pg_ps_enabled, 0,
+ "Are large page mappings enabled?");
+
static u_int64_t KPTphys; /* phys addr of kernel level 1 */
static u_int64_t KPDphys; /* phys addr of kernel level 2 */
u_int64_t KPDPphys; /* phys addr of kernel level 3 */
@@ -185,6 +197,7 @@
* Data for the pv entry allocation mechanism
*/
static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
+static struct md_page *pv_table;
static int shpgperproc = PMAP_SHPGPERPROC;
/*
@@ -201,11 +214,29 @@
static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try);
+static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_page_t m);
+static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
+static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
+ vm_offset_t va);
+static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
+ vm_prot_t prot);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);
+static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
+static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
+static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
+ vm_prot_t prot);
+static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
+ vm_page_t *free);
static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free);
+static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
vm_page_t *free);
static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
@@ -361,21 +392,6 @@
}
-static __inline pt_entry_t *
-pmap_pte_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *ptepde)
-{
- pd_entry_t *pde;
-
- pde = pmap_pde(pmap, va);
- if (pde == NULL || (*pde & PG_V) == 0)
- return NULL;
- *ptepde = *pde;
- if ((*pde & PG_PS) != 0) /* compat with i386 pmap_pte() */
- return ((pt_entry_t *)pde);
- return (pmap_pde_to_pte(pde, va));
-}
-
-
PMAP_INLINE pt_entry_t *
vtopte(vm_offset_t va)
{
@@ -521,6 +537,7 @@
*/
PMAP_LOCK_INIT(kernel_pmap);
kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + KPML4phys);
+ kernel_pmap->pm_root = NULL;
kernel_pmap->pm_active = -1; /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvchunk);
nkpt = NKPT;
@@ -620,8 +637,28 @@
void
pmap_init(void)
{
+ pd_entry_t *pd;
+ vm_page_t mpte;
+ vm_size_t s;
+ int i, pv_npg;
/*
+ * Initialize the vm page array entries for the kernel pmap's
+ * page table pages.
+ */
+ pd = pmap_pde(kernel_pmap, VM_MIN_KERNEL_ADDRESS);
+ for (i = 0; i < nkpt; i++) {
+ if ((pd[i] & (PG_PS | PG_V)) == (PG_PS | PG_V))
+ continue;
+ mpte = PHYS_TO_VM_PAGE(pd[i] & PG_FRAME);
+ KASSERT(mpte >= vm_page_array &&
+ mpte < &vm_page_array[vm_page_array_size],
+ ("pmap_init: page table page is out of range"));
+ mpte->pindex = pmap_pde_pindex(VM_MIN_KERNEL_ADDRESS) + i;
+ mpte->phys_addr = pd[i] & PG_FRAME;
+ }
+
+ /*
* Initialize the address space (zone) for the pv entries. Set a
* high water mark so that the system can recover from excessive
* numbers of pv entries.
@@ -630,9 +667,28 @@
pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
pv_entry_high_water = 9 * (pv_entry_max / 10);
+
+ /*
+ * Are large page mappings enabled?
+ */
+ TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
+
+ /*
+ * Calculate the size of the pv head table for superpages.
+ */
+ for (i = 0; phys_avail[i + 1]; i += 2);
+ pv_npg = round_2mpage(phys_avail[(i - 2) + 1]) / NBPDR;
+
+ /*
+ * Allocate memory for the pv head table for superpages.
+ */
+ s = (vm_size_t)(pv_npg * sizeof(struct md_page));
+ s = round_page(s);
+ pv_table = (struct md_page *)kmem_alloc(kernel_map, s);
+ for (i = 0; i < pv_npg; i++)
+ TAILQ_INIT(&pv_table[i].pv_list);
}
-SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
static int
pmap_pventry_proc(SYSCTL_HANDLER_ARGS)
{
@@ -663,6 +719,25 @@
SYSCTL_PROC(_vm_pmap, OID_AUTO, shpgperproc, CTLTYPE_INT|CTLFLAG_RW,
&shpgperproc, 0, pmap_shpgperproc_proc, "IU", "Page share factor per proc");
+SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
+ "2MB page mapping counters");
+
+static u_long pmap_pde_demotions;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
+ &pmap_pde_demotions, 0, "2MB page demotions");
+
+static u_long pmap_pde_mappings;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
+ &pmap_pde_mappings, 0, "2MB page mappings");
+
+static u_long pmap_pde_p_failures;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
+ &pmap_pde_p_failures, 0, "2MB page promotion failures");
+
+static u_long pmap_pde_promotions;
+SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
+ &pmap_pde_promotions, 0, "2MB page promotions");
+
/***************************************************
* Low level helper routines.....
@@ -1097,8 +1172,105 @@
while (free != NULL) {
m = free;
free = m->right;
- vm_page_free_zero(m);
+ /* Preserve the page's PG_ZERO setting. */
+ vm_page_free_toq(m);
+ }
+}
+
+/*
+ * Schedule the specified unused page table page to be freed. Specifically,
+ * add the page to the specified list of pages that will be released to the
+ * physical memory manager after the TLB has been updated.
+ */
+static __inline void
+pmap_add_delayed_free_list(vm_page_t m, vm_page_t *free, boolean_t set_PG_ZERO)
+{
+
+ if (set_PG_ZERO)
+ m->flags |= PG_ZERO;
+ else
+ m->flags &= ~PG_ZERO;
+ m->right = *free;
+ *free = m;
+}
+
+/*
+ * Inserts the specified page table page into the specified pmap's collection
+ * of idle page table pages. Each of a pmap's page table pages is responsible
+ * for mapping a distinct range of virtual addresses. The pmap's collection is
+ * ordered by this virtual address range.
+ */
+static void
+pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
+{
+ vm_page_t root;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ root = pmap->pm_root;
+ if (root == NULL) {
+ mpte->left = NULL;
+ mpte->right = NULL;
+ } else {
+ root = vm_page_splay(mpte->pindex, root);
+ if (mpte->pindex < root->pindex) {
+ mpte->left = root->left;
+ mpte->right = root;
+ root->left = NULL;
+ } else if (mpte->pindex == root->pindex)
+ panic("pmap_insert_pt_page: pindex already inserted");
+ else {
+ mpte->right = root->right;
+ mpte->left = root;
+ root->right = NULL;
+ }
+ }
+ pmap->pm_root = mpte;
+}
+
+/*
+ * Looks for a page table page mapping the specified virtual address in the
+ * specified pmap's collection of idle page table pages. Returns NULL if there
+ * is no page table page corresponding to the specified virtual address.
+ */
+static vm_page_t
+pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
+{
+ vm_page_t mpte;
+ vm_pindex_t pindex = pmap_pde_pindex(va);
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ if ((mpte = pmap->pm_root) != NULL && mpte->pindex != pindex) {
+ mpte = vm_page_splay(pindex, mpte);
+ if ((pmap->pm_root = mpte)->pindex != pindex)
+ mpte = NULL;
+ }
+ return (mpte);
+}
+
+/*
+ * Removes the specified page table page from the specified pmap's collection
+ * of idle page table pages. The specified page table page must be a member of
+ * the pmap's collection.
+ */
+static void
+pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
+{
+ vm_page_t root;
+
+ PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ if (mpte != pmap->pm_root) {
+ root = vm_page_splay(mpte->pindex, pmap->pm_root);
+ KASSERT(mpte == root,
+ ("pmap_remove_pt_page: mpte %p is missing from pmap %p",
+ mpte, pmap));
+ }
+ if (mpte->left == NULL)
+ root = mpte->right;
+ else {
+ root = vm_page_splay(mpte->pindex, mpte->left);
+ root->right = mpte->right;
}
+ pmap->pm_root = root;
}
/*
@@ -1177,8 +1349,7 @@
* Put page on a list so that it is released after
* *ALL* TLB shootdown is done
*/
- m->right = *free;
- *free = m;
+ pmap_add_delayed_free_list(m, free, TRUE);
return 1;
}
@@ -1205,6 +1376,7 @@
PMAP_LOCK_INIT(pmap);
pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + KPML4phys);
+ pmap->pm_root = NULL;
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1241,6 +1413,7 @@
/* install self-referential address mapping entry(s) */
pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
+ pmap->pm_root = NULL;
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1416,7 +1589,7 @@
{
vm_pindex_t ptepindex;
pd_entry_t *pd;
- vm_page_t m, free;
+ vm_page_t m;
KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
(flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
@@ -1437,13 +1610,13 @@
* normal 4K page.
*/
if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
- *pd = 0;
- pd = NULL;
- pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
- free = NULL;
- pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va), &free);
- pmap_invalidate_all(kernel_pmap);
- pmap_free_zero_pages(free);
+ if (!pmap_demote_pde(pmap, pd, va)) {
+ /*
+ * Invalidation of the 2MB page mapping may have caused
+ * the deallocation of the underlying PD page.
+ */
+ pd = NULL;
+ }
}
/*
@@ -1483,6 +1656,8 @@
KASSERT(pmap->pm_stats.resident_count == 0,
("pmap_release: pmap resident count %ld != 0",
pmap->pm_stats.resident_count));
+ KASSERT(pmap->pm_root == NULL,
+ ("pmap_release: pmap has reserved page table page(s)"));
m = PHYS_TO_VM_PAGE(pmap->pm_pml4[PML4PML4I] & PG_FRAME);
@@ -1649,11 +1824,16 @@
* drastic measures to free some pages so we can allocate
* another pv entry chunk. This is normally called to
* unmap inactive pages, and if necessary, active pages.
+ *
+ * We do not, however, unmap 2mpages because subsequent accesses will
+ * allocate per-page pv entries until repromotion occurs, thereby
+ * exacerbating the shortage of free pv entries.
*/
static void
pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
{
- pd_entry_t ptepde;
+ struct md_page *pvh;
+ pd_entry_t *pde;
pmap_t pmap;
pt_entry_t *pte, tpte;
pv_entry_t next_pv, pv;
@@ -1672,10 +1852,10 @@
else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
continue;
pmap->pm_stats.resident_count--;
- pte = pmap_pte_pde(pmap, va, &ptepde);
- if (pte == NULL) {
- panic("null pte in pmap_collect");
- }
+ pde = pmap_pde(pmap, va);
+ KASSERT((*pde & PG_PS) == 0, ("pmap_collect: found"
+ " a 2mpage in page %p's pv list", m));
+ pte = pmap_pde_to_pte(pde, va);
tpte = pte_load_clear(pte);
KASSERT((tpte & PG_W) == 0,
("pmap_collect: wired pte %#lx", tpte));
@@ -1688,12 +1868,15 @@
vm_page_dirty(m);
}
free = NULL;
- pmap_unuse_pt(pmap, va, ptepde, &free);
+ pmap_unuse_pt(pmap, va, *pde, &free);
pmap_invalidate_page(pmap, va);
pmap_free_zero_pages(free);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
- if (TAILQ_EMPTY(&m->md.pv_list))
- vm_page_flag_clear(m, PG_WRITEABLE);
+ if (TAILQ_EMPTY(&m->md.pv_list)) {
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ if (TAILQ_EMPTY(&pvh->pv_list))
+ vm_page_flag_clear(m, PG_WRITEABLE);
+ }
free_pv_entry(pmap, pv);
if (pmap != locked_pmap)
PMAP_UNLOCK(pmap);
@@ -1828,24 +2011,133 @@
return (pv);
}
-static void
-pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
+/*
+ * First find and then remove the pv entry for the specified pmap and virtual
+ * address from the specified pv list. Returns the pv entry if found and NULL
+ * otherwise. This operation can be performed on pv lists for either 4KB or
+ * 2MB page mappings.
+ */
+static __inline pv_entry_t
+pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
{
pv_entry_t pv;
- PMAP_LOCK_ASSERT(pmap, MA_OWNED);
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
- if (pmap == PV_PMAP(pv) && va == pv->pv_va)
+ TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
+ if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
+ TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
break;
+ }
}
- KASSERT(pv != NULL, ("pmap_remove_entry: pv not found"));
- TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
- if (TAILQ_EMPTY(&m->md.pv_list))
- vm_page_flag_clear(m, PG_WRITEABLE);
+ return (pv);
+}
+
+/*
+ * After demotion from a 2MB page mapping to 512 4KB page mappings,
+ * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
+ * entries for each of the 4KB page mappings.
+ */
+static void
+pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+{
+ struct md_page *pvh;
+ pv_entry_t pv;
+ vm_offset_t va_last;
+ vm_page_t m;
+
+ mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((pa & PDRMASK) == 0,
+ ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
+
+ /*
+ * Transfer the 2mpage's pv entry for this mapping to the first
+ * page's pv list.
+ */
+ pvh = pa_to_pvh(pa);
+ va = trunc_2mpage(va);
+ pv = pmap_pvh_remove(pvh, pmap, va);
+ KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
+ m = PHYS_TO_VM_PAGE(pa);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+ /* Instantiate the remaining NPTEPG - 1 pv entries. */
+ va_last = va + NBPDR - PAGE_SIZE;
+ do {
+ m++;
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
+ ("pmap_pv_demote_pde: page %p is not managed", m));
+ va += PAGE_SIZE;
+ pmap_insert_entry(pmap, va, m);
+ } while (va < va_last);
+}
+
+/*
+ * After promotion from 512 4KB page mappings to a single 2MB page mapping,
+ * replace the many pv entries for the 4KB page mappings by a single pv entry
+ * for the 2MB page mapping.
+ */
+static void
+pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+{
+ struct md_page *pvh;
+ pv_entry_t pv;
+ vm_offset_t va_last;
+ vm_page_t m;
+
+ mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ KASSERT((pa & PDRMASK) == 0,
+ ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
+
+ /*
+ * Transfer the first page's pv entry for this mapping to the
+ * 2mpage's pv list. Aside from avoiding the cost of a call
+ * to get_pv_entry(), a transfer avoids the possibility that
+ * get_pv_entry() calls pmap_collect() and that pmap_collect()
+ * removes one of the mappings that is being promoted.
+ */
+ m = PHYS_TO_VM_PAGE(pa);
+ va = trunc_2mpage(va);
+ pv = pmap_pvh_remove(&m->md, pmap, va);
+ KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
+ pvh = pa_to_pvh(pa);
+ TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+ /* Free the remaining NPTEPG - 1 pv entries. */
+ va_last = va + NBPDR - PAGE_SIZE;
+ do {
+ m++;
+ va += PAGE_SIZE;
+ pmap_pvh_free(&m->md, pmap, va);
+ } while (va < va_last);
+}
+
+/*
+ * First find and then destroy the pv entry for the specified pmap and virtual
+ * address. This operation can be performed on pv lists for either 4KB or 2MB
+ * page mappings.
+ */
+static void
+pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
+{
+ pv_entry_t pv;
+
+ pv = pmap_pvh_remove(pvh, pmap, va);
+ KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
free_pv_entry(pmap, pv);
}
+static void
+pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
+{
+ struct md_page *pvh;
+
+ mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ pmap_pvh_free(&m->md, pmap, va);
+ if (TAILQ_EMPTY(&m->md.pv_list)) {
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ if (TAILQ_EMPTY(&pvh->pv_list))
+ vm_page_flag_clear(m, PG_WRITEABLE);
+ }
+}
+
/*
* Create a pv entry for page at pa for
* (pmap, va).
@@ -1882,6 +2174,174 @@
}
/*
+ * Create the pv entry for a 2MB page mapping.
+ */
+static boolean_t
+pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_page_t m)
+{
+ struct md_page *pvh;
+ pv_entry_t pv;
+
+ mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ if (pv_entry_count < pv_entry_high_water &&
+ (pv = get_pv_entry(pmap, TRUE)) != NULL) {
+ pv->pv_va = va;
+ pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+ TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
+ return (TRUE);
+ } else
+ return (FALSE);
+}
+
+/*
>>> TRUNCATED FOR MAIL (1000 lines) <<<
More information about the p4-projects
mailing list