svn commit: r223640 - in projects/largeSMP/sys/i386: i386 xen
Attilio Rao
attilio at FreeBSD.org
Tue Jun 28 13:13:06 UTC 2011
Author: attilio
Date: Tue Jun 28 13:13:06 2011
New Revision: 223640
URL: http://svn.freebsd.org/changeset/base/223640
Log:
Remove pc_cpumask usage from i386 and XEN.
Tested by: pluknet
Modified:
projects/largeSMP/sys/i386/i386/mp_machdep.c
projects/largeSMP/sys/i386/i386/pmap.c
projects/largeSMP/sys/i386/xen/mp_machdep.c
projects/largeSMP/sys/i386/xen/pmap.c
Modified: projects/largeSMP/sys/i386/i386/mp_machdep.c
==============================================================================
--- projects/largeSMP/sys/i386/i386/mp_machdep.c Tue Jun 28 12:51:08 2011 (r223639)
+++ projects/largeSMP/sys/i386/i386/mp_machdep.c Tue Jun 28 13:13:06 2011 (r223640)
@@ -658,12 +658,11 @@ cpu_mp_announce(void)
void
init_secondary(void)
{
- cpuset_t tcpuset;
struct pcpu *pc;
vm_offset_t addr;
int gsel_tss;
int x, myid;
- u_int cr0;
+ u_int cpuid, cr0;
/* bootAP is set in start_ap() to our ID. */
myid = bootAP;
@@ -758,8 +757,9 @@ init_secondary(void)
#endif
/* A quick check from sanity claus */
+ cpuid = PCPU_GET(cpuid);
if (PCPU_GET(apic_id) != lapic_id()) {
- printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
+ printf("SMP: cpuid = %d\n", cpuid);
printf("SMP: actual apic_id = %d\n", lapic_id());
printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
panic("cpuid mismatch! boom!!");
@@ -781,14 +781,13 @@ init_secondary(void)
smp_cpus++;
- CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
- printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
- tcpuset = PCPU_GET(cpumask);
+ CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
+ printf("SMP: AP CPU #%d Launched!\n", cpuid);
/* Determine if we are a logical CPU. */
/* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */
if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0)
- CPU_OR(&logical_cpus_mask, &tcpuset);
+ CPU_SET(cpuid, &logical_cpus_mask);
if (bootverbose)
lapic_dump("AP");
@@ -1242,9 +1241,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask
if (othercpus < 1)
return;
} else {
- sched_pin();
- CPU_NAND(&mask, PCPU_PTR(cpumask));
- sched_unpin();
+ CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
return;
}
@@ -1465,7 +1462,7 @@ ipi_all_but_self(u_int ipi)
int
ipi_nmi_handler()
{
- cpuset_t cpumask;
+ u_int cpuid;
/*
* As long as there is not a simple way to know about a NMI's
@@ -1473,13 +1470,11 @@ ipi_nmi_handler()
* the global pending bitword an IPI_STOP_HARD has been issued
* and should be handled.
*/
- sched_pin();
- cpumask = PCPU_GET(cpumask);
- sched_unpin();
- if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ if (!CPU_ISSET(cpuid, &ipi_nmi_pending))
return (1);
- CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask);
+ CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending);
cpustop_handler();
return (0);
}
@@ -1491,25 +1486,21 @@ ipi_nmi_handler()
void
cpustop_handler(void)
{
- cpuset_t cpumask;
u_int cpu;
- sched_pin();
cpu = PCPU_GET(cpuid);
- cpumask = PCPU_GET(cpumask);
- sched_unpin();
savectx(&stoppcbs[cpu]);
/* Indicate that we are stopped */
- CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_SET_ATOMIC(cpu, &stopped_cpus);
/* Wait for restart */
- while (!CPU_OVERLAP(&started_cpus, &cpumask))
+ while (!CPU_ISSET(cpu, &started_cpus))
ia32_pause();
- CPU_NAND_ATOMIC(&started_cpus, &cpumask);
- CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_CLR_ATOMIC(cpu, &started_cpus);
+ CPU_CLR_ATOMIC(cpu, &stopped_cpus);
if (cpu == 0 && cpustop_restartfunc != NULL) {
cpustop_restartfunc();
Modified: projects/largeSMP/sys/i386/i386/pmap.c
==============================================================================
--- projects/largeSMP/sys/i386/i386/pmap.c Tue Jun 28 12:51:08 2011 (r223639)
+++ projects/largeSMP/sys/i386/i386/pmap.c Tue Jun 28 13:13:06 2011 (r223640)
@@ -1012,11 +1012,11 @@ pmap_invalidate_cache(void)
}
struct pde_action {
- cpuset_t store; /* processor that updates the PDE */
cpuset_t invalidate; /* processors that invalidate their TLB */
vm_offset_t va;
pd_entry_t *pde;
pd_entry_t newpde;
+ u_int store; /* processor that updates the PDE */
};
static void
@@ -1026,9 +1026,7 @@ pmap_update_pde_kernel(void *arg)
pd_entry_t *pde;
pmap_t pmap;
- sched_pin();
- if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) {
- sched_unpin();
+ if (act->store == PCPU_GET(cpuid)) {
/*
* Elsewhere, this operation requires allpmaps_lock for
@@ -1039,8 +1037,7 @@ pmap_update_pde_kernel(void *arg)
pde = pmap_pde(pmap, act->va);
pde_store(pde, act->newpde);
}
- } else
- sched_unpin();
+ }
}
static void
@@ -1048,12 +1045,8 @@ pmap_update_pde_user(void *arg)
{
struct pde_action *act = arg;
- sched_pin();
- if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) {
- sched_unpin();
+ if (act->store == PCPU_GET(cpuid))
pde_store(act->pde, act->newpde);
- } else
- sched_unpin();
}
static void
@@ -1077,24 +1070,25 @@ static void
pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
{
struct pde_action act;
- cpuset_t active, cpumask, other_cpus;
+ cpuset_t active, other_cpus;
+ u_int cpuid;
sched_pin();
- cpumask = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
other_cpus = all_cpus;
- CPU_CLR(PCPU_GET(cpuid), &other_cpus);
+ CPU_CLR(cpuid, &other_cpus);
if (pmap == kernel_pmap)
active = all_cpus;
else
active = pmap->pm_active;
if (CPU_OVERLAP(&active, &other_cpus)) {
- act.store = cpumask;
+ act.store = cpuid;
act.invalidate = active;
act.va = va;
act.pde = pde;
act.newpde = newpde;
- CPU_OR(&cpumask, &active);
- smp_rendezvous_cpus(cpumask,
+ CPU_SET(cpuid, &active);
+ smp_rendezvous_cpus(active,
smp_no_rendevous_barrier, pmap == kernel_pmap ?
pmap_update_pde_kernel : pmap_update_pde_user,
pmap_update_pde_teardown, &act);
@@ -1103,7 +1097,7 @@ pmap_update_pde(pmap_t pmap, vm_offset_t
pmap_kenter_pde(va, newpde);
else
pde_store(pde, newpde);
- if (CPU_OVERLAP(&active, &cpumask))
+ if (CPU_ISSET(cpuid, &active))
pmap_update_pde_invalidate(va, newpde);
}
sched_unpin();
@@ -1928,12 +1922,12 @@ pmap_lazyfix_action(void)
}
static void
-pmap_lazyfix_self(cpuset_t mymask)
+pmap_lazyfix_self(u_int cpuid)
{
if (rcr3() == lazyptd)
load_cr3(PCPU_GET(curpcb)->pcb_cr3);
- CPU_NAND_ATOMIC(lazymask, &mymask);
+ CPU_CLR_ATOMIC(cpuid, lazymask);
}
@@ -1941,7 +1935,7 @@ static void
pmap_lazyfix(pmap_t pmap)
{
cpuset_t mymask, mask;
- u_int spins;
+ u_int cpuid, spins;
int lsb;
mask = pmap->pm_active;
@@ -1959,10 +1953,13 @@ pmap_lazyfix(pmap_t pmap)
#else
lazyptd = vtophys(pmap->pm_pdir);
#endif
- mymask = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
+
+ /* Use a cpuset just for having an easy check. */
+ CPU_SETOF(cpuid, &mymask);
if (!CPU_CMP(&mask, &mymask)) {
lazymask = &pmap->pm_active;
- pmap_lazyfix_self(mymask);
+ pmap_lazyfix_self(cpuid);
} else {
atomic_store_rel_int((u_int *)&lazymask,
(u_int)&pmap->pm_active);
@@ -5101,17 +5098,19 @@ void
pmap_activate(struct thread *td)
{
pmap_t pmap, oldpmap;
+ u_int cpuid;
u_int32_t cr3;
critical_enter();
pmap = vmspace_pmap(td->td_proc->p_vmspace);
oldpmap = PCPU_GET(curpmap);
+ cpuid = PCPU_GET(cpuid);
#if defined(SMP)
- CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
+ CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
#else
- CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR(cpuid, &oldpmap->pm_active);
+ CPU_SET(cpuid, &pmap->pm_active);
#endif
#ifdef PAE
cr3 = vtophys(pmap->pm_pdpt);
Modified: projects/largeSMP/sys/i386/xen/mp_machdep.c
==============================================================================
--- projects/largeSMP/sys/i386/xen/mp_machdep.c Tue Jun 28 12:51:08 2011 (r223639)
+++ projects/largeSMP/sys/i386/xen/mp_machdep.c Tue Jun 28 13:13:06 2011 (r223640)
@@ -523,8 +523,8 @@ xen_smp_intr_init_cpus(void *unused)
void
init_secondary(void)
{
- cpuset_t tcpuset;
vm_offset_t addr;
+ u_int cpuid;
int gsel_tss;
@@ -601,18 +601,18 @@ init_secondary(void)
#endif
smp_cpus++;
- CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
- printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
- tcpuset = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
+ CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
+ printf("SMP: AP CPU #%d Launched!\n", cpuid);
/* Determine if we are a logical CPU. */
if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0)
- CPU_OR(&logical_cpus_mask, &tcpuset);
+ CPU_SET(cpuid, &logical_cpus_mask);
/* Determine if we are a hyperthread. */
if (hyperthreading_cpus > 1 &&
PCPU_GET(apic_id) % hyperthreading_cpus != 0)
- CPU_OR(&hyperthreading_cpus_mask, &tcpuset);
+ CPU_SET(cpuid, &hyperthreading_cpus_mask);
#if 0
if (bootverbose)
lapic_dump("AP");
@@ -1020,9 +1020,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask
if (othercpus < 1)
return;
} else {
- critical_enter();
- CPU_NAND(&mask, PCPU_PTR(cpumask));
- critical_exit();
+ CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
return;
}
@@ -1185,7 +1183,7 @@ ipi_all_but_self(u_int ipi)
int
ipi_nmi_handler()
{
- cpuset_t cpumask;
+ u_int cpuid;
/*
* As long as there is not a simple way to know about a NMI's
@@ -1193,13 +1191,11 @@ ipi_nmi_handler()
* the global pending bitword an IPI_STOP_HARD has been issued
* and should be handled.
*/
- sched_pin();
- cpumask = PCPU_GET(cpumask);
- sched_unpin();
- if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask))
+ cpuid = PCPU_GET(cpuid);
+ if (!CPU_ISSET(cpuid, &ipi_nmi_pending))
return (1);
- CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask);
+ CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending);
cpustop_handler();
return (0);
}
@@ -1211,25 +1207,21 @@ ipi_nmi_handler()
void
cpustop_handler(void)
{
- cpuset_t cpumask;
int cpu;
- sched_pin();
- cpumask = PCPU_GET(cpumask);
cpu = PCPU_GET(cpuid);
- sched_unpin();
savectx(&stoppcbs[cpu]);
/* Indicate that we are stopped */
- CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_SET_ATOMIC(cpu, &stopped_cpus);
/* Wait for restart */
- while (!CPU_OVERLAP(&started_cpus, &cpumask))
+ while (!CPU_ISSET(cpu, &started_cpus))
ia32_pause();
- CPU_NAND_ATOMIC(&started_cpus, &cpumask);
- CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
+ CPU_CLR_ATOMIC(cpu, &started_cpus);
+ CPU_CLR_ATOMIC(cpu, &stopped_cpus);
if (cpu == 0 && cpustop_restartfunc != NULL) {
cpustop_restartfunc();
Modified: projects/largeSMP/sys/i386/xen/pmap.c
==============================================================================
--- projects/largeSMP/sys/i386/xen/pmap.c Tue Jun 28 12:51:08 2011 (r223639)
+++ projects/largeSMP/sys/i386/xen/pmap.c Tue Jun 28 13:13:06 2011 (r223640)
@@ -802,7 +802,8 @@ pmap_cache_bits(int mode, boolean_t is_p
void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
+ u_int cpuid;
CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x",
pmap, va);
@@ -812,10 +813,10 @@ pmap_invalidate_page(pmap_t pmap, vm_off
invlpg(va);
smp_invlpg(va);
} else {
- cpumask = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
other_cpus = all_cpus;
- CPU_CLR(PCPU_GET(cpuid), &other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
invlpg(va);
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
@@ -828,8 +829,9 @@ pmap_invalidate_page(pmap_t pmap, vm_off
void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
vm_offset_t addr;
+ u_int cpuid;
CTR3(KTR_PMAP, "pmap_invalidate_page: pmap=%p eva=0x%x sva=0x%x",
pmap, sva, eva);
@@ -840,10 +842,10 @@ pmap_invalidate_range(pmap_t pmap, vm_of
invlpg(addr);
smp_invlpg_range(sva, eva);
} else {
- cpumask = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
other_cpus = all_cpus;
- CPU_CLR(PCPU_GET(cpuid), &other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
CPU_AND(&other_cpus, &pmap->pm_active);
@@ -857,7 +859,8 @@ pmap_invalidate_range(pmap_t pmap, vm_of
void
pmap_invalidate_all(pmap_t pmap)
{
- cpuset_t cpumask, other_cpus;
+ cpuset_t other_cpus;
+ u_int cpuid;
CTR1(KTR_PMAP, "pmap_invalidate_page: pmap=%p", pmap);
@@ -866,10 +869,10 @@ pmap_invalidate_all(pmap_t pmap)
invltlb();
smp_invltlb();
} else {
- cpumask = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
other_cpus = all_cpus;
- CPU_CLR(PCPU_GET(cpuid), &other_cpus);
- if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
+ CPU_CLR(cpuid, &other_cpus);
+ if (CPU_ISSET(cpuid, &pmap->pm_active))
invltlb();
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
@@ -1711,12 +1714,12 @@ pmap_lazyfix_action(void)
}
static void
-pmap_lazyfix_self(cpuset_t mymask)
+pmap_lazyfix_self(u_int cpuid)
{
if (rcr3() == lazyptd)
load_cr3(PCPU_GET(curpcb)->pcb_cr3);
- CPU_NAND_ATOMIC(lazymask, &mymask);
+ CPU_CLR_ATOMIC(cpuid, lazymask);
}
@@ -1724,7 +1727,7 @@ static void
pmap_lazyfix(pmap_t pmap)
{
cpuset_t mymask, mask;
- u_int spins;
+ u_int cpuid, spins;
int lsb;
mask = pmap->pm_active;
@@ -1742,10 +1745,13 @@ pmap_lazyfix(pmap_t pmap)
#else
lazyptd = vtophys(pmap->pm_pdir);
#endif
- mymask = PCPU_GET(cpumask);
+ cpuid = PCPU_GET(cpuid);
+
+ /* Use a cpuset just for having an easy check. */
+ CPU_SETOF(cpuid, &mymask);
if (!CPU_CMP(&mask, &mymask)) {
lazymask = &pmap->pm_active;
- pmap_lazyfix_self(mymask);
+ pmap_lazyfix_self(cpuid);
} else {
atomic_store_rel_int((u_int *)&lazymask,
(u_int)&pmap->pm_active);
@@ -4129,17 +4135,19 @@ void
pmap_activate(struct thread *td)
{
pmap_t pmap, oldpmap;
+ u_int cpuid;
u_int32_t cr3;
critical_enter();
pmap = vmspace_pmap(td->td_proc->p_vmspace);
oldpmap = PCPU_GET(curpmap);
+ cpuid = PCPU_GET(cpuid);
#if defined(SMP)
- CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
+ CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
#else
- CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask));
- CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask));
+ CPU_CLR(cpuid, &oldpmap->pm_active);
+ CPU_SET(cpuid, &pmap->pm_active);
#endif
#ifdef PAE
cr3 = vtophys(pmap->pm_pdpt);
More information about the svn-src-projects
mailing list