PERFORCE change 96842 for review
Kip Macy
kmacy at FreeBSD.org
Tue May 9 00:18:03 UTC 2006
http://perforce.freebsd.org/chv.cgi?CH=96842
Change 96842 by kmacy at kmacy_storage:sun4v_rwbuf on 2006/05/09 00:17:16
use direct area for referencing TSB at TL < 2 once the TSB is installed
clean up prologue to tsb miss handler - greatly simplifying
don't always clear TSB in invalidate_range and invalidate_page - allowing us to pre-load the TSB in places
make more aggressive use of membars when updating the TSB
Affected files ...
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/pmap.h#12 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#11 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#50 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#48 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#13 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte.c#9 edit
Differences ...
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/pmap.h#12 (text+ko) ====
@@ -102,8 +102,8 @@
void pmap_bootstrap(vm_offset_t ekva);
vm_paddr_t pmap_kextract(vm_offset_t va);
-void pmap_invalidate_page(pmap_t pmap, vm_offset_t va);
-void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
+void pmap_invalidate_page(pmap_t pmap, vm_offset_t va, int cleartsb);
+void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int cleartsb);
void pmap_invalidate_all(pmap_t pmap);
void pmap_scrub_pages(vm_paddr_t pa, int64_t size);
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#11 (text+ko) ====
@@ -32,6 +32,8 @@
void tsb_set_tte(struct hv_tsb_info *tsb, vm_offset_t va, tte_t tte_data, uint64_t ctx);
+void tsb_set_tte_real(struct hv_tsb_info *tsb, vm_offset_t va, tte_t tte_data, uint64_t ctx);
+
tte_t tsb_get_tte(struct hv_tsb_info *tsb, vm_offset_t va);
tte_t tsb_lookup_tte(vm_offset_t va, uint64_t context);
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#50 (text+ko) ====
@@ -1393,67 +1393,41 @@
! %g3==flag bits, TSB (RA)
! %g4==fault type,entry tag
! %g5==tag
-! %g6==context,hash size, temp
+! %g6==context
! %g7 temp
ENTRY(tsb_miss_handler)
ldxa [%g1 + %g7]ASI_REAL, %g6 ! load in the context
- rdpr %tl, %g7 ! need to use real addresses
-
-#ifdef notyet
- rdpr %cansave, %g1
-#else
- mov 0, %g1
-#endif
-
- brz,pn %g1, 0f
- nop
- save
-0:
- cmp %g7, 1 ! for tl > 1
- bne,pn %xcc, 2f
- nop
- mov ASI_LDTD_N, %g3
- wr %g0, ASI_N, %asi
- GET_PCPU_SCRATCH
- cmp %g6, %g0 ! kernel?
- be,pn %xcc, 1f
- nop
GET_HASH_SCRATCH_USER(%g2)
GET_TSB_SCRATCH_USER(%g4)
- brz,pn %g1, 4f
- nop
- ba,pt %xcc, 5f
- nop
-1:
+
+ brnz,pn %g6, 2f
+ nop
GET_HASH_SCRATCH_KERNEL(%g2)
GET_TSB_SCRATCH_KERNEL(%g4)
- brz,pn %g1, 4f
- nop
- ba,pt %xcc, 5f
- nop
2:
- mov ASI_LDTD_REAL, %g3
- wr %g0, ASI_REAL, %asi
- GET_PCPU_PHYS_SCRATCH(%g4)
- cmp %g6, %g0 ! kernel?
- be,pn %xcc, 3f
- nop
- GET_HASH_PHYS_SCRATCH_USER(%g4, %g2)
- GET_TSB_SCRATCH_USER(%g4)
- brz,pn %g1, 4f
- nop
- ba,pt %xcc, 5f
- nop
-3:
+ rdpr %tl, %g1 ! need to use real addresses?
+ mov ASI_LDTD_N, %g3
+ wr %g0, ASI_N, %asi
+ dec %g1
+ GET_PCPU_SCRATCH
- GET_HASH_PHYS_SCRATCH_KERNEL(%g4, %g2)
- GET_TSB_SCRATCH_KERNEL(%g4)
- brnz,pt %g1, 5f
+ brz,pt %g1, 3f ! for tl == 1
nop
-4:
+ sethi %uhi(VM_MIN_DIRECT_ADDRESS), %g1
+ wr %g0, ASI_REAL, %asi
+ sllx %g1, 32, %g1
+ mov ASI_LDTD_REAL, %g3
+ andn %g2, %g1, %g2
+ andn %g4, %g1, %g4
+ andn %g7, %g1, %g7
+3:
+#ifdef notyet
+ rdpr %cansave, %g1
+ /* XXX use save operation if %g1 > 0 and tl == 1 */
+#endif
rdpr %tl, %g1
dec %g1
sll %g1, RW_SHIFT, %g1
@@ -1461,7 +1435,6 @@
add PCPU_REG, %g1, %g1
SAVE_LOCALS_ASI(%g1)
mov 0, %g1 ! cansave is 0
-5:
! %g1 == %cansave
! %g2 == hash scratch value
@@ -1588,10 +1561,10 @@
sllx %g3, TTE_SHIFT, %g3 ! masked byte offset
add %g3, %l3, %g3 ! TTE RA
- mov 8, %l4
#ifdef PMAP_DEBUG
- ldda [%g3]ASI_LDTD_REAL, %l2
+ ldxa [%g3]%asi, %l2
+ ldxa [%g3 + 8]%asi, %l3
cmp %l3, %l7
bne,pt %xcc, 12f
cmp %l2, %l6
@@ -1612,12 +1585,10 @@
#endif
12:
#endif
- stxa %g0, [%g3 + %l4]ASI_REAL ! invalidate data
- membar #StoreStore
- stxa %l6, [%g3]ASI_REAL ! store tag
- membar #StoreStore
- stxa %l7, [%g3 + %l4]ASI_REAL ! store data
- stxa %l7, [%g2 + 8]%asi ! update TTE with ref bit
+ stxa %g0, [%g3 + 8]%asi ! invalidate data
+ stxa %l6, [%g3]%asi ! store tag
+ stxa %l7, [%g3 + 8]%asi ! store data
+ stxa %l7, [%g2 + 8]%asi ! update TTE with ref bit
membar #StoreLoad
THE_LOCK_EXIT(%l5, %l0, %l7)
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#48 (text+ko) ====
@@ -332,7 +332,7 @@
vm_page_dirty(m);
}
- pmap_invalidate_page(pmap, va);
+ pmap_invalidate_page(pmap, va, TRUE);
TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
if (TAILQ_EMPTY(&m->md.pv_list))
@@ -639,7 +639,7 @@
for (i = 0; i < KSTACK_PAGES; i++) {
pa = kstack0_phys + i * PAGE_SIZE;
va = kstack0 + i * PAGE_SIZE;
- tsb_set_tte(&kernel_td[TSB8K_INDEX], va,
+ tsb_set_tte_real(&kernel_td[TSB8K_INDEX], va,
pa | TTE_KERNEL | VTD_8K, 0);
}
/*
@@ -674,7 +674,7 @@
va = translations[i].om_start + off;
pa = TTE_GET_PA(translations[i].om_tte) + off;
tsb_assert_invalid(&kernel_td[TSB8K_INDEX], va);
- tsb_set_tte(&kernel_td[TSB8K_INDEX], va, pa |
+ tsb_set_tte_real(&kernel_td[TSB8K_INDEX], va, pa |
TTE_KERNEL | VTD_8K, 0);
}
}
@@ -687,7 +687,7 @@
pa = PAGE_SIZE_4M;
for (i = 0; phys_avail[i + 2] != 0; i += 2)
for (; pa < phys_avail[i + 1]; pa += PAGE_SIZE_4M) {
- tsb_set_tte(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa),
+ tsb_set_tte_real(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa),
pa | TTE_KERNEL | VTD_4M, 0);
}
@@ -966,6 +966,8 @@
otte_data = tte_hash_update(pmap->pm_hash, va, tte_data | TTE_MINFLAGS);
+ tsb_set_tte(&pmap->pm_tsb, va, tte_data|TTE_MINFLAGS, pmap->pm_context);
+
invlva = FALSE;
if ((otte_data & ~(VTD_W|VTD_REF)) != tte_data) {
if (otte_data & VTD_V) {
@@ -988,7 +990,7 @@
}
if (invlva)
- pmap_invalidate_page(pmap, va);
+ pmap_invalidate_page(pmap, va, FALSE);
sched_unpin();
PMAP_UNLOCK(pmap);
@@ -1229,11 +1231,12 @@
#endif
void
-pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va, int cleartsb)
{
spinlock_enter();
- tsb_clear_tte(&pmap->pm_tsb, va);
+ if (cleartsb == TRUE)
+ tsb_clear_tte(&pmap->pm_tsb, va);
DPRINTF("pmap_invalidate_page(va=0x%lx)\n", va);
invlpg(va, pmap->pm_context);
@@ -1247,7 +1250,7 @@
}
void
-pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int cleartsb)
{
vm_offset_t tva;
#ifdef SMP
@@ -1256,14 +1259,12 @@
spinlock_enter();
- if (pmap != kernel_pmap)
- DPRINTF("pmap_invalidate_range(sva=%lx, eva=%lx)\n", sva, eva);
-
- if ((((eva - sva) >> PAGE_SHIFT) < MAX_TSB_CLEARS) ||
- (pmap->pm_context == 0)) {
- tsb_clear_range(&pmap->pm_tsb, sva, eva);
- } else {
- tsb_clear(&pmap->pm_tsb);
+ if (cleartsb == TRUE) {
+ if ((((eva - sva) >> PAGE_SHIFT) < MAX_TSB_CLEARS) ||
+ (pmap->pm_context == 0))
+ tsb_clear_range(&pmap->pm_tsb, sva, eva);
+ else
+ tsb_clear(&pmap->pm_tsb);
}
/* XXX */
@@ -1342,6 +1343,7 @@
pmap_kenter(vm_offset_t va, vm_paddr_t pa)
{
tte_hash_update(kernel_pmap->pm_hash, va, pa | TTE_KERNEL | VTD_8K);
+ tsb_set_tte(&kernel_td[TSB8K_INDEX], va, pa | TTE_KERNEL | VTD_8K, 0);
}
/*
@@ -1402,7 +1404,7 @@
start += PAGE_SIZE;
}
- pmap_invalidate_range(kernel_pmap, sva, va);
+ pmap_invalidate_range(kernel_pmap, sva, va, FALSE);
*virt = va;
return (sva);
@@ -1569,7 +1571,7 @@
sched_unpin();
if (anychanged)
- pmap_invalidate_range(pmap, sva, eva);
+ pmap_invalidate_range(pmap, sva, eva, TRUE);
vm_page_unlock_queues();
PMAP_UNLOCK(pmap);
@@ -1591,7 +1593,7 @@
va += PAGE_SIZE;
m++;
}
- pmap_invalidate_range(kernel_pmap, sva, va);
+ pmap_invalidate_range(kernel_pmap, sva, va, FALSE);
}
/*
@@ -1608,7 +1610,7 @@
pmap_kremove(va);
va += PAGE_SIZE;
}
- pmap_invalidate_range(kernel_pmap, sva, va);
+ pmap_invalidate_range(kernel_pmap, sva, va, TRUE);
}
@@ -1661,7 +1663,7 @@
sched_unpin();
vm_page_unlock_queues();
if (invlva)
- pmap_invalidate_range(pmap, start, end);
+ pmap_invalidate_range(pmap, start, end, TRUE);
PMAP_UNLOCK(pmap);
@@ -1711,7 +1713,7 @@
vm_page_dirty(m);
}
- pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
+ pmap_invalidate_page(pv->pv_pmap, pv->pv_va, TRUE);
TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
m->md.pv_list_count--;
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#13 (text+ko) ====
@@ -51,6 +51,8 @@
#include <machine/mmu.h>
#include <machine/tte.h>
#include <machine/tsb.h>
+#include <machine/vmparam.h>
+#include <machine/tlb.h>
CTASSERT(sizeof(tte_t) == sizeof(uint64_t));
#define TSB_MASK(tsb) ((tsb->hvtsb_ntte) - 1)
@@ -92,7 +94,7 @@
hvtsb->hvtsb_pa = VM_PAGE_TO_PHYS(m);
tsb_pages = hvtsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT);
- *scratchval = hvtsb->hvtsb_pa | tsb_pages;
+ *scratchval = TLB_PHYS_TO_DIRECT(hvtsb->hvtsb_pa) | tsb_pages;
return vtophys(hvtsb);
}
@@ -127,7 +129,7 @@
}
void
-tsb_set_tte(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx)
+tsb_set_tte_real(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx)
{
vm_paddr_t tsb_store_pa;
uint64_t tsb_index, tsb_shift, tte_tag;
@@ -152,6 +154,30 @@
store_real(tsb_store_pa + sizeof(uint64_t), tte_data);
}
+
+void
+tsb_set_tte(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx)
+{
+
+ uint64_t tsb_index, tsb_shift, tte_tag;
+ tte_t *entry;
+
+ tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+ tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+ entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+ tte_tag = (ctx << TTARGET_CTX_SHIFT) | (va >> TTARGET_VA_SHIFT);
+ tte_data &= ~VTD_V;
+ /* store new value with valid bit cleared
+ * to avoid invalid intermediate value;
+ */
+ *(entry + 1) = tte_data;
+ tte_data |= VTD_V;
+ *(entry) = tte_tag;
+ *(entry + 1) = tte_data;
+ membar(Sync);
+}
+
+
void
tsb_clear(hv_tsb_info_t *tsb)
{
@@ -161,40 +187,60 @@
void
tsb_clear_tte(hv_tsb_info_t *tsb, vm_offset_t va)
{
- vm_paddr_t tsb_store_pa;
+ tte_t *entry;
uint64_t tsb_index, tsb_shift;
tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
- tsb_store_pa = tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t);
+ entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+
+ *(entry) = 0;
+ *(entry + 1) = 0;
- store_real(tsb_store_pa, 0);
- store_real(tsb_store_pa + sizeof(uint64_t), 0);
+ membar(Sync);
}
void
tsb_clear_range(hv_tsb_info_t *tsb, vm_offset_t sva, vm_offset_t eva)
{
- vm_paddr_t tsb_store_pa;
vm_offset_t tva;
uint64_t tsb_index, tsb_shift, tsb_mask;
-
+ tte_t *entry;
+
tsb_mask = TSB_MASK(tsb);
tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
for (tva = sva; tva < eva; tva += PAGE_SIZE) {
tsb_index = (tva >> tsb_shift) & tsb_mask;
- tsb_store_pa = tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t);
-
- store_real(tsb_store_pa, 0);
- store_real(tsb_store_pa + sizeof(uint64_t), 0);
+ entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+ *(entry) = 0;
+ *(entry + 1) = 0;
}
+ membar(Sync);
}
tte_t
tsb_get_tte(hv_tsb_info_t *tsb, vm_offset_t va)
{
+ tte_t *entry;
+ uint64_t tsb_index, tsb_shift, tte_tag, tte_data;
+
+ tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz);
+ tsb_index = (va >> tsb_shift) & TSB_MASK(tsb);
+ entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t));
+ tte_tag = *(entry);
+ tte_data = *(entry + 1);
+
+ if ((tte_tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M))
+ return tte_data;
+
+ return (0UL);
+}
+#if 0
+tte_t
+tsb_get_tte_real(hv_tsb_info_t *tsb, vm_offset_t va)
+{
vm_paddr_t tsb_load_pa;
uint64_t tsb_index, tsb_shift, tte_tag, tte_data;
@@ -213,6 +259,7 @@
return (0UL);
}
+#endif
tte_t
tsb_lookup_tte(vm_offset_t va, uint64_t ctx)
@@ -238,7 +285,7 @@
{
uint64_t tsb_pages, tsb_scratch;
tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT);
- tsb_scratch = tsb->hvtsb_pa | tsb_pages;
+ tsb_scratch = TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa) | tsb_pages;
set_tsb_kernel_scratchpad(tsb_scratch);
membar(Sync);
@@ -250,7 +297,7 @@
{
uint64_t tsb_pages, tsb_scratch;
tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT);
- tsb_scratch = tsb->hvtsb_pa | tsb_pages;
+ tsb_scratch = TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa) | tsb_pages;
set_tsb_user_scratchpad(tsb_scratch);
membar(Sync);
return tsb_scratch;
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte.c#9 (text+ko) ====
@@ -81,7 +81,7 @@
if (otte_data & VTD_W)
vm_page_dirty(m);
- pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
+ pmap_invalidate_page(pv->pv_pmap, pv->pv_va, TRUE);
}
@@ -112,7 +112,7 @@
otte_data = tte_hash_clear_bits(pmap->pm_hash, va, flags);
if (otte_data & flags)
- pmap_invalidate_page(pmap, va);
+ pmap_invalidate_page(pmap, va, TRUE);
}
void
More information about the p4-projects
mailing list