git: 166784e27501 - stable/13 - amd64 pmap: convert to counter(9), add PV and pagetable page counts
Mark Johnston
markj at FreeBSD.org
Wed Sep 1 13:31:08 UTC 2021
The branch stable/13 has been updated by markj:
URL: https://cgit.FreeBSD.org/src/commit/?id=166784e27501f96446566aeff62acf575f3f157d
commit 166784e27501f96446566aeff62acf575f3f157d
Author: Jason A. Harmening <jah at FreeBSD.org>
AuthorDate: 2021-02-25 05:08:42 +0000
Commit: Mark Johnston <markj at FreeBSD.org>
CommitDate: 2021-09-01 13:29:01 +0000
amd64 pmap: convert to counter(9), add PV and pagetable page counts
This change converts most of the counters in the amd64 pmap from
global atomics to scalable counter(9) counters. Per discussion
with kib@, it also removes the handrolled per-CPU PCID save count
as it isn't considered generally useful.
The bulk of these counters remain guarded by PV_STATS, as it seems
unlikely that they will be useful outside of very specific debugging
scenarios. However, this change does add two new counters that
are available without PV_STATS. pt_page_count and pv_page_count
track the number of active physical-to-virtual list pages and page
table pages, respectively. These will be useful in evaluating
the memory footprint of pmap structures under various workloads,
which will help to guide future changes in this area.
Reviewed by: kib
(cherry picked from commit e4b8deb222278b2a12c9c67021b406625f5be301)
---
sys/amd64/amd64/genassym.c | 1 -
sys/amd64/amd64/pmap.c | 274 +++++++++++++++++++++++++--------------------
sys/amd64/include/pcpu.h | 3 +-
3 files changed, 155 insertions(+), 123 deletions(-)
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index f89c53e75508..7c29368828d0 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -227,7 +227,6 @@ ASSYM(PC_GS32P, offsetof(struct pcpu, pc_gs32p));
ASSYM(PC_LDT, offsetof(struct pcpu, pc_ldt));
ASSYM(PC_COMMONTSS, offsetof(struct pcpu, pc_common_tss));
ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss));
-ASSYM(PC_PM_SAVE_CNT, offsetof(struct pcpu, pc_pm_save_cnt));
ASSYM(PC_KCR3, offsetof(struct pcpu, pc_kcr3));
ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3));
ASSYM(PC_UCR3_LOAD_MASK, offsetof(struct pcpu, pc_ucr3_load_mask));
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 0ac312a95a8c..3086aa0b053f 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -536,21 +536,9 @@ static int pmap_pkru_copy(pmap_t dst_pmap, pmap_t src_pmap);
static int pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
static void pmap_pkru_deassign_all(pmap_t pmap);
-static int
-pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS)
-{
- int i;
- uint64_t res;
-
- res = 0;
- CPU_FOREACH(i) {
- res += cpuid_to_pcpu[i]->pc_pm_save_cnt;
- }
- return (sysctl_handle_64(oidp, &res, 0, req));
-}
-SYSCTL_PROC(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLTYPE_U64 | CTLFLAG_RD |
- CTLFLAG_MPSAFE, NULL, 0, pmap_pcid_save_cnt_proc, "QU",
- "Count of saved TLB context on switch");
+static COUNTER_U64_DEFINE_EARLY(pcid_save_cnt);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLFLAG_RD,
+ &pcid_save_cnt, "Count of saved TLB context on switch");
static LIST_HEAD(, pmap_invl_gen) pmap_invl_gen_tracker =
LIST_HEAD_INITIALIZER(&pmap_invl_gen_tracker);
@@ -770,19 +758,30 @@ pmap_di_store_invl(struct pmap_invl_gen *ptr, struct pmap_invl_gen *old_val,
return (res);
}
+static COUNTER_U64_DEFINE_EARLY(pv_page_count);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_page_count, CTLFLAG_RD,
+ &pv_page_count, "Current number of allocated pv pages");
+
+static COUNTER_U64_DEFINE_EARLY(pt_page_count);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pt_page_count, CTLFLAG_RD,
+ &pt_page_count, "Current number of allocated page table pages");
+
#ifdef PV_STATS
-static long invl_start_restart;
-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_start_restart, CTLFLAG_RD,
- &invl_start_restart, 0,
- "");
-static long invl_finish_restart;
-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD,
- &invl_finish_restart, 0,
- "");
+
+static COUNTER_U64_DEFINE_EARLY(invl_start_restart);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_start_restart,
+ CTLFLAG_RD, &invl_start_restart,
+ "Number of delayed TLB invalidation request restarts");
+
+static COUNTER_U64_DEFINE_EARLY(invl_finish_restart);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD,
+ &invl_finish_restart,
+ "Number of delayed TLB invalidation completion restarts");
+
static int invl_max_qlen;
SYSCTL_INT(_vm_pmap, OID_AUTO, invl_max_qlen, CTLFLAG_RD,
&invl_max_qlen, 0,
- "");
+ "Maximum delayed TLB invalidation request queue length");
#endif
#define di_delay locks_delay
@@ -820,7 +819,7 @@ again:
PV_STAT(i++);
prevl = (uintptr_t)atomic_load_ptr(&p->next);
if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) {
- PV_STAT(atomic_add_long(&invl_start_restart, 1));
+ PV_STAT(counter_u64_add(invl_start_restart, 1));
lock_delay(&lda);
goto again;
}
@@ -834,7 +833,7 @@ again:
#endif
if (!pmap_di_load_invl(p, &prev) || prev.next != NULL) {
- PV_STAT(atomic_add_long(&invl_start_restart, 1));
+ PV_STAT(counter_u64_add(invl_start_restart, 1));
lock_delay(&lda);
goto again;
}
@@ -863,7 +862,7 @@ again:
*/
if (!pmap_di_store_invl(p, &prev, &new_prev)) {
critical_exit();
- PV_STAT(atomic_add_long(&invl_start_restart, 1));
+ PV_STAT(counter_u64_add(invl_start_restart, 1));
lock_delay(&lda);
goto again;
}
@@ -927,7 +926,7 @@ again:
for (p = &pmap_invl_gen_head; p != NULL; p = (void *)prevl) {
prevl = (uintptr_t)atomic_load_ptr(&p->next);
if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) {
- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
lock_delay(&lda);
goto again;
}
@@ -940,7 +939,7 @@ again:
* thread before us finished its DI and started it again.
*/
if (__predict_false(p == NULL)) {
- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
lock_delay(&lda);
goto again;
}
@@ -952,7 +951,7 @@ again:
atomic_clear_ptr((uintptr_t *)&invl_gen->next,
PMAP_INVL_GEN_NEXT_INVALID);
critical_exit();
- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
lock_delay(&lda);
goto again;
}
@@ -988,12 +987,15 @@ DB_SHOW_COMMAND(di_queue, pmap_di_queue)
#endif
#ifdef PV_STATS
-static long invl_wait;
-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait, CTLFLAG_RD, &invl_wait, 0,
+static COUNTER_U64_DEFINE_EARLY(invl_wait);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait,
+ CTLFLAG_RD, &invl_wait,
"Number of times DI invalidation blocked pmap_remove_all/write");
-static long invl_wait_slow;
-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD, &invl_wait_slow, 0,
- "Number of slow invalidation waits for lockless DI");
+
+static COUNTER_U64_DEFINE_EARLY(invl_wait_slow);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD,
+ &invl_wait_slow, "Number of slow invalidation waits for lockless DI");
+
#endif
#ifdef NUMA
@@ -1067,7 +1069,7 @@ pmap_delayed_invl_wait_l(vm_page_t m)
while (*m_gen > pmap_invl_gen) {
#ifdef PV_STATS
if (!accounted) {
- atomic_add_long(&invl_wait, 1);
+ counter_u64_add(invl_wait, 1);
accounted = true;
}
#endif
@@ -1087,7 +1089,7 @@ pmap_delayed_invl_wait_u(vm_page_t m)
lock_delay_arg_init(&lda, &di_delay);
while (*m_gen > atomic_load_long(&pmap_invl_gen_head.gen)) {
if (fast || !pmap_invl_callout_inited) {
- PV_STAT(atomic_add_long(&invl_wait, 1));
+ PV_STAT(counter_u64_add(invl_wait, 1));
lock_delay(&lda);
fast = false;
} else {
@@ -1120,7 +1122,7 @@ pmap_delayed_invl_wait_u(vm_page_t m)
atomic_load_long(&pmap_invl_gen_head.gen)) {
callout_reset(&pmap_invl_callout, 1,
pmap_delayed_invl_callout_func, NULL);
- PV_STAT(atomic_add_long(&invl_wait_slow, 1));
+ PV_STAT(counter_u64_add(invl_wait_slow, 1));
pmap_delayed_invl_wait_block(m_gen,
&pmap_invl_gen_head.gen);
}
@@ -2460,28 +2462,28 @@ SYSCTL_UINT(_vm_pmap, OID_AUTO, large_map_pml4_entries,
static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"2MB page mapping counters");
-static u_long pmap_pde_demotions;
-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
- &pmap_pde_demotions, 0, "2MB page demotions");
+static COUNTER_U64_DEFINE_EARLY(pmap_pde_demotions);
+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, demotions,
+ CTLFLAG_RD, &pmap_pde_demotions, "2MB page demotions");
-static u_long pmap_pde_mappings;
-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
- &pmap_pde_mappings, 0, "2MB page mappings");
+static COUNTER_U64_DEFINE_EARLY(pmap_pde_mappings);
+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
+ &pmap_pde_mappings, "2MB page mappings");
-static u_long pmap_pde_p_failures;
-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
- &pmap_pde_p_failures, 0, "2MB page promotion failures");
+static COUNTER_U64_DEFINE_EARLY(pmap_pde_p_failures);
+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
+ &pmap_pde_p_failures, "2MB page promotion failures");
-static u_long pmap_pde_promotions;
-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
- &pmap_pde_promotions, 0, "2MB page promotions");
+static COUNTER_U64_DEFINE_EARLY(pmap_pde_promotions);
+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
+ &pmap_pde_promotions, "2MB page promotions");
static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"1GB page mapping counters");
-static u_long pmap_pdpe_demotions;
-SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
- &pmap_pdpe_demotions, 0, "1GB page demotions");
+static COUNTER_U64_DEFINE_EARLY(pmap_pdpe_demotions);
+SYSCTL_COUNTER_U64(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
+ &pmap_pdpe_demotions, "1GB page demotions");
/***************************************************
* Low level helper routines.....
@@ -4017,6 +4019,8 @@ _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
pmap_unwire_ptp(pmap, va, pml4pg, free);
}
+ counter_u64_add(pt_page_count, -1);
+
/*
* Put page on a list so that it is released after
* *ALL* TLB shootdown is done
@@ -4197,6 +4201,8 @@ pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags)
pmltop_pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK);
+ counter_u64_add(pt_page_count, 1);
+
pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg);
pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys);
@@ -4228,6 +4234,7 @@ pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags)
if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) {
pmltop_pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_WAITOK);
+ counter_u64_add(pt_page_count, 1);
pmap->pm_pmltopu = (pml4_entry_t *)PHYS_TO_DMAP(
VM_PAGE_TO_PHYS(pmltop_pgu));
if (pmap_is_la57(pmap))
@@ -4415,6 +4422,7 @@ pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp,
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
return (NULL);
+
if ((m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
@@ -4512,6 +4520,8 @@ pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp,
}
pmap_resident_count_inc(pmap, 1);
+ counter_u64_add(pt_page_count, 1);
+
return (m);
}
@@ -4674,12 +4684,14 @@ pmap_release(pmap_t pmap)
vm_page_unwire_noq(m);
vm_page_free_zero(m);
+ counter_u64_add(pt_page_count, -1);
if (pmap->pm_pmltopu != NULL) {
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->
pm_pmltopu));
vm_page_unwire_noq(m);
vm_page_free(m);
+ counter_u64_add(pt_page_count, -1);
}
if (pmap->pm_type == PT_X86 &&
(cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0)
@@ -4795,6 +4807,7 @@ pmap_growkernel(vm_offset_t addr)
panic("pmap_growkernel: no memory to grow kernel");
if ((nkpg->flags & PG_ZERO) == 0)
pmap_zero_page(nkpg);
+ counter_u64_add(pt_page_count, 1);
paddr = VM_PAGE_TO_PHYS(nkpg);
*pdpe = (pdp_entry_t)(paddr | X86_PG_V | X86_PG_RW |
X86_PG_A | X86_PG_M);
@@ -4817,6 +4830,7 @@ pmap_growkernel(vm_offset_t addr)
panic("pmap_growkernel: no memory to grow kernel");
if ((nkpg->flags & PG_ZERO) == 0)
pmap_zero_page(nkpg);
+ counter_u64_add(pt_page_count, 1);
paddr = VM_PAGE_TO_PHYS(nkpg);
newpdir = paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M;
pde_store(pde, newpdir);
@@ -4853,28 +4867,39 @@ pv_to_chunk(pv_entry_t pv)
static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
#ifdef PV_STATS
-static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
-
-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
- "Current number of pv entry chunks");
-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
- "Current number of pv entry chunks allocated");
-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
- "Current number of pv entry chunks frees");
-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
- "Number of times tried to get a chunk page but failed.");
-
-static long pv_entry_frees, pv_entry_allocs, pv_entry_count;
-static int pv_entry_spare;
-
-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
- "Current number of pv entry frees");
-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
- "Current number of pv entry allocs");
-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
- "Current number of pv entries");
-SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
- "Current number of spare pv entries");
+
+static COUNTER_U64_DEFINE_EARLY(pc_chunk_count);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD,
+ &pc_chunk_count, "Current number of pv entry cnunks");
+
+static COUNTER_U64_DEFINE_EARLY(pc_chunk_allocs);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD,
+ &pc_chunk_allocs, "Total number of pv entry chunks allocated");
+
+static COUNTER_U64_DEFINE_EARLY(pc_chunk_frees);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD,
+ &pc_chunk_frees, "Total number of pv entry chunks freed");
+
+static COUNTER_U64_DEFINE_EARLY(pc_chunk_tryfail);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD,
+ &pc_chunk_tryfail,
+ "Number of failed attempts to get a pv entry chunk page");
+
+static COUNTER_U64_DEFINE_EARLY(pv_entry_frees);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD,
+ &pv_entry_frees, "Total number of pv entries freed");
+
+static COUNTER_U64_DEFINE_EARLY(pv_entry_allocs);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD,
+ &pv_entry_allocs, "Total number of pv entries allocated");
+
+static COUNTER_U64_DEFINE_EARLY(pv_entry_count);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD,
+ &pv_entry_count, "Current number of pv entries");
+
+static COUNTER_U64_DEFINE_EARLY(pv_entry_spare);
+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD,
+ &pv_entry_spare, "Current number of spare pv entries");
#endif
static void
@@ -5047,15 +5072,15 @@ reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain)
}
/* Every freed mapping is for a 4 KB page. */
pmap_resident_count_dec(pmap, freed);
- PV_STAT(atomic_add_long(&pv_entry_frees, freed));
- PV_STAT(atomic_add_int(&pv_entry_spare, freed));
- PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
+ PV_STAT(counter_u64_add(pv_entry_frees, freed));
+ PV_STAT(counter_u64_add(pv_entry_spare, freed));
+ PV_STAT(counter_u64_add(pv_entry_count, -freed));
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 &&
pc->pc_map[2] == PC_FREE2) {
- PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
- PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
- PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
+ PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV));
+ PV_STAT(counter_u64_add(pc_chunk_count, -1));
+ PV_STAT(counter_u64_add(pc_chunk_frees, 1));
/* Entire chunk is free; return it. */
m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
dump_drop_page(m_pc->phys_addr);
@@ -5128,9 +5153,9 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv)
int idx, field, bit;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- PV_STAT(atomic_add_long(&pv_entry_frees, 1));
- PV_STAT(atomic_add_int(&pv_entry_spare, 1));
- PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
+ PV_STAT(counter_u64_add(pv_entry_frees, 1));
+ PV_STAT(counter_u64_add(pv_entry_spare, 1));
+ PV_STAT(counter_u64_add(pv_entry_count, -1));
pc = pv_to_chunk(pv);
idx = pv - &pc->pc_pventry[0];
field = idx / 64;
@@ -5154,9 +5179,10 @@ free_pv_chunk_dequeued(struct pv_chunk *pc)
{
vm_page_t m;
- PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
- PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
- PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
+ PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV));
+ PV_STAT(counter_u64_add(pc_chunk_count, -1));
+ PV_STAT(counter_u64_add(pc_chunk_frees, 1));
+ counter_u64_add(pv_page_count, -1);
/* entire chunk is free, return it */
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
dump_drop_page(m->phys_addr);
@@ -5219,7 +5245,7 @@ get_pv_entry(pmap_t pmap, struct rwlock **lockp)
vm_page_t m;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
+ PV_STAT(counter_u64_add(pv_entry_allocs, 1));
retry:
pc = TAILQ_FIRST(&pmap->pm_pvchunk);
if (pc != NULL) {
@@ -5239,8 +5265,8 @@ retry:
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
pc_list);
}
- PV_STAT(atomic_add_long(&pv_entry_count, 1));
- PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
+ PV_STAT(counter_u64_add(pv_entry_count, 1));
+ PV_STAT(counter_u64_add(pv_entry_spare, -1));
return (pv);
}
}
@@ -5249,15 +5275,16 @@ retry:
VM_ALLOC_WIRED);
if (m == NULL) {
if (lockp == NULL) {
- PV_STAT(pc_chunk_tryfail++);
+ PV_STAT(counter_u64_add(pc_chunk_tryfail, 1));
return (NULL);
}
m = reclaim_pv_chunk(pmap, lockp);
if (m == NULL)
goto retry;
- }
- PV_STAT(atomic_add_int(&pc_chunk_count, 1));
- PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
+ } else
+ counter_u64_add(pv_page_count, 1);
+ PV_STAT(counter_u64_add(pc_chunk_count, 1));
+ PV_STAT(counter_u64_add(pc_chunk_allocs, 1));
dump_add_page(m->phys_addr);
pc = (void *)PHYS_TO_DMAP(m->phys_addr);
pc->pc_pmap = pmap;
@@ -5270,8 +5297,8 @@ retry:
mtx_unlock(&pvc->pvc_lock);
pv = &pc->pc_pventry[0];
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
- PV_STAT(atomic_add_long(&pv_entry_count, 1));
- PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
+ PV_STAT(counter_u64_add(pv_entry_count, 1));
+ PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV - 1));
return (pv);
}
@@ -5355,9 +5382,10 @@ retry:
if (m == NULL)
goto retry;
reclaimed = true;
- }
- PV_STAT(atomic_add_int(&pc_chunk_count, 1));
- PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
+ } else
+ counter_u64_add(pv_page_count, 1);
+ PV_STAT(counter_u64_add(pc_chunk_count, 1));
+ PV_STAT(counter_u64_add(pc_chunk_allocs, 1));
dump_add_page(m->phys_addr);
pc = (void *)PHYS_TO_DMAP(m->phys_addr);
pc->pc_pmap = pmap;
@@ -5366,7 +5394,7 @@ retry:
pc->pc_map[2] = PC_FREE2;
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
TAILQ_INSERT_TAIL(&new_tail[vm_page_domain(m)], pc, pc_lru);
- PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
+ PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV));
/*
* The reclaim might have freed a chunk from the current pmap.
@@ -5441,7 +5469,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
m->md.pv_gen++;
/* Instantiate the remaining NPTEPG - 1 pv entries. */
- PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
+ PV_STAT(counter_u64_add(pv_entry_allocs, NPTEPG - 1));
va_last = va + NBPDR - PAGE_SIZE;
for (;;) {
pc = TAILQ_FIRST(&pmap->pm_pvchunk);
@@ -5471,8 +5499,8 @@ out:
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
}
- PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
- PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
+ PV_STAT(counter_u64_add(pv_entry_count, NPTEPG - 1));
+ PV_STAT(counter_u64_add(pv_entry_spare, -(NPTEPG - 1)));
}
#if VM_NRESERVLEVEL > 0
@@ -5732,6 +5760,8 @@ pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
return (FALSE);
}
+ counter_u64_add(pt_page_count, 1);
+
if (!in_kernel) {
mpte->ref_count = NPTEPG;
pmap_resident_count_inc(pmap, 1);
@@ -5796,7 +5826,7 @@ pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
if ((oldpde & PG_MANAGED) != 0)
pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp);
- atomic_add_long(&pmap_pde_demotions, 1);
+ counter_u64_add(pmap_pde_demotions, 1);
CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx in pmap %p",
va, pmap);
return (TRUE);
@@ -6515,7 +6545,7 @@ pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V) ||
!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,
newpde))) {
- atomic_add_long(&pmap_pde_p_failures, 1);
+ counter_u64_add(pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@@ -6540,7 +6570,7 @@ setpde:
for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
oldpte = *pte;
if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
- atomic_add_long(&pmap_pde_p_failures, 1);
+ counter_u64_add(pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@@ -6559,7 +6589,7 @@ setpte:
(va & ~PDRMASK), pmap);
}
if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
- atomic_add_long(&pmap_pde_p_failures, 1);
+ counter_u64_add(pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@@ -6581,7 +6611,7 @@ setpte:
"mpte %p pidx %#lx va %#lx va pde pidx %#lx",
mpte, mpte->pindex, va, pmap_pde_pindex(va)));
if (pmap_insert_pt_page(pmap, mpte, true)) {
- atomic_add_long(&pmap_pde_p_failures, 1);
+ counter_u64_add(pmap_pde_p_failures, 1);
CTR2(KTR_PMAP,
"pmap_promote_pde: failure for va %#lx in pmap %p", va,
pmap);
@@ -6607,7 +6637,7 @@ setpte:
else
pde_store(pde, PG_PROMOTED | PG_PS | newpde);
- atomic_add_long(&pmap_pde_promotions, 1);
+ counter_u64_add(pmap_pde_promotions, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
" in pmap %p", va, pmap);
}
@@ -7184,7 +7214,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,
*/
pde_store(pde, newpde);
- atomic_add_long(&pmap_pde_mappings, 1);
+ counter_u64_add(pmap_pde_mappings, 1);
CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p",
va, pmap);
return (KERN_SUCCESS);
@@ -7445,7 +7475,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
pde_store(pde, pa | PG_PS | PG_M | PG_A |
PG_U | PG_RW | PG_V);
pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
- atomic_add_long(&pmap_pde_mappings, 1);
+ counter_u64_add(pmap_pde_mappings, 1);
} else {
/* Continue on if the PDE is already valid. */
pdpg->ref_count--;
@@ -7673,7 +7703,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
*pde = srcptepaddr & ~PG_W;
pmap_resident_count_inc(dst_pmap, NBPDR /
PAGE_SIZE);
- atomic_add_long(&pmap_pde_mappings, 1);
+ counter_u64_add(pmap_pde_mappings, 1);
} else
pmap_abort_ptp(dst_pmap, addr, dst_pdpg);
continue;
@@ -8158,9 +8188,9 @@ pmap_remove_pages(pmap_t pmap)
freed++;
}
}
- PV_STAT(atomic_add_long(&pv_entry_frees, freed));
- PV_STAT(atomic_add_int(&pv_entry_spare, freed));
- PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
+ PV_STAT(counter_u64_add(pv_entry_frees, freed));
+ PV_STAT(counter_u64_add(pv_entry_spare, freed));
+ PV_STAT(counter_u64_add(pv_entry_count, -freed));
if (allfree) {
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, pc_list);
@@ -9071,6 +9101,7 @@ pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va)
" in pmap %p", va, pmap);
return (FALSE);
}
+ counter_u64_add(pt_page_count, 1);
pdpgpa = VM_PAGE_TO_PHYS(pdpg);
firstpde = (pd_entry_t *)PHYS_TO_DMAP(pdpgpa);
newpdpe = pdpgpa | PG_M | PG_A | (oldpdpe & PG_U) | PG_RW | PG_V;
@@ -9098,7 +9129,7 @@ pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va)
*/
pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va));
- pmap_pdpe_demotions++;
+ counter_u64_add(pmap_pdpe_demotions, 1);
CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
" in pmap %p", va, pmap);
return (TRUE);
@@ -9620,7 +9651,7 @@ pmap_activate_sw_pcid_pti(struct thread *td, pmap_t pmap, u_int cpuid)
PCPU_SET(kcr3, kcr3 | CR3_PCID_SAVE);
PCPU_SET(ucr3, ucr3 | CR3_PCID_SAVE);
if (cached)
- PCPU_INC(pm_save_cnt);
+ counter_u64_add(pcid_save_cnt, 1);
pmap_activate_sw_pti_post(td, pmap);
}
@@ -9641,7 +9672,7 @@ pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap,
cached);
PCPU_SET(curpmap, pmap);
if (cached)
- PCPU_INC(pm_save_cnt);
+ counter_u64_add(pcid_save_cnt, 1);
}
static void
@@ -10086,8 +10117,11 @@ pmap_large_map_getptp_unlocked(void)
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
VM_ALLOC_ZERO);
- if (m != NULL && (m->flags & PG_ZERO) == 0)
- pmap_zero_page(m);
+ if (m != NULL) {
+ if ((m->flags & PG_ZERO) == 0)
+ pmap_zero_page(m);
+ counter_u64_add(pt_page_count, 1);
+ }
return (m);
}
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
index 4b163636c775..6326fbdae0be 100644
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -76,7 +76,6 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
struct system_segment_descriptor *pc_ldt; \
/* Pointer to the CPU TSS descriptor */ \
struct system_segment_descriptor *pc_tss; \
- uint64_t pc_pm_save_cnt; \
u_int pc_cmci_mask; /* MCx banks for CMCI */ \
uint64_t pc_dbreg[16]; /* ddb debugging regs */ \
uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \
@@ -89,7 +88,7 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
uint32_t pc_ibpb_set; \
void *pc_mds_buf; \
void *pc_mds_buf64; \
- uint32_t pc_pad[2]; \
+ uint32_t pc_pad[4]; \
uint8_t pc_mds_tmp[64]; \
u_int pc_ipi_bitmap; \
struct amd64tss pc_common_tss; \
More information about the dev-commits-src-branches
mailing list