PERFORCE change 96586 for review
Kip Macy
kmacy at FreeBSD.org
Wed May 3 05:00:04 UTC 2006
http://perforce.freebsd.org/chv.cgi?CH=96586
Change 96586 by kmacy at kmacy_storage:sun4v_rwbuf on 2006/05/03 04:59:12
simplify tte_hash interface
- hide locking from outside callers
- add tte_hash_update to get an atomic view of the tte before a change
- change tte_hash_delete to return the value of the tte being deleted
to avoid redundant lookup
Affected files ...
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#11 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#40 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte.c#7 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#16 edit
Differences ...
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#11 (text+ko) ====
@@ -14,22 +14,23 @@
tte_hash_t tte_hash_create(uint64_t context, uint64_t *scratchval);
-void tte_hash_bucket_unlock(tte_hash_t hash, vm_offset_t va);
-
void tte_hash_destroy(tte_hash_t th);
-int tte_hash_delete(tte_hash_t hash, vm_offset_t va, int locked);
+tte_t tte_hash_delete(tte_hash_t hash, vm_offset_t va);
void tte_hash_delete_all(tte_hash_t hash);
void tte_hash_insert(tte_hash_t hash, vm_offset_t va, tte_t data);
-tte_t *tte_hash_lookup(tte_hash_t hash, vm_offset_t va, int leave_locked);
+tte_t tte_hash_lookup(tte_hash_t hash, vm_offset_t va);
+
+tte_t tte_hash_lookup_nolock(tte_hash_t hash, vm_offset_t va);
uint64_t tte_hash_set_scratchpad_kernel(tte_hash_t th);
uint64_t tte_hash_set_scratchpad_user(tte_hash_t th, uint64_t context);
+tte_t tte_hash_update(tte_hash_t hash, vm_offset_t va, tte_t tte_data);
#endif /* _MACHINE_TTE_HASH_H_ */
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#40 (text+ko) ====
@@ -126,7 +126,7 @@
static uma_zone_t pvzone;
static struct vm_object pvzone_obj;
static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
-int pmap_debug = 0;
+int pmap_debug = 1;
static struct mtx pmap_ctx_lock;
static uint16_t ctx_stack[PMAP_CONTEXT_MAX];
@@ -196,7 +196,7 @@
static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va);
-static void pmap_remove_tte(pmap_t pmap, uint64_t *tte, vm_offset_t va);
+static void pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va);
/*
* Quick sort callout for comparing memory regions.
@@ -277,7 +277,7 @@
static const struct timeval printinterval = { 60, 0 };
static struct timeval lastprint;
struct vpgqueues *vpq;
- uint64_t *tte, tte_data;
+ uint64_t tte_data;
pmap_t pmap;
pv_entry_t allocated_pv, next_pv, pv;
vm_offset_t va;
@@ -319,9 +319,7 @@
continue;
pmap->pm_stats.resident_count--;
- tte = tte_hash_lookup(pmap->pm_hash, va, TRUE);
- tte_data = *tte;
- tte_hash_delete(pmap->pm_hash, va, TRUE);
+ tte_data = tte_hash_delete(pmap->pm_hash, va);
KASSERT((tte_data & VTD_WIRED) == 0,
("get_pv_entry: wired pte %#jx", (uintmax_t)tte_data));
@@ -815,12 +813,8 @@
{
vm_offset_t addr, end_addr;
-
end_addr = src_addr + len;
- if (dst_addr != src_addr)
- return;
-
/*
* Don't let optional prefaulting of pages make us go
* way below the low water mark of free pages or way
@@ -840,13 +834,13 @@
}
sched_pin();
for (addr = src_addr; addr < end_addr; addr += PAGE_SIZE) {
- tte_t *src_tte, *dst_tte, tte_data;
+ tte_t tte_data;
vm_page_t m;
- src_tte = tte_hash_lookup(src_pmap->pm_hash, addr, FALSE);
- tte_data = src_tte ? *src_tte : 0;
+ tte_data = tte_hash_lookup_nolock(src_pmap->pm_hash, addr);
+
if ((tte_data & VTD_MANAGED) != 0) {
- if ((dst_tte = tte_hash_lookup(dst_pmap->pm_hash, addr, FALSE)) == NULL) {
+ if (tte_hash_lookup_nolock(dst_pmap->pm_hash, addr) == 0) {
m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
tte_hash_insert(dst_pmap->pm_hash, addr, tte_data & ~(VTD_W|VTD_REF));
dst_pmap->pm_stats.resident_count++;
@@ -885,7 +879,7 @@
boolean_t wired)
{
vm_paddr_t pa, opa;
- uint64_t tte_data, otte_data, *otte;
+ uint64_t tte_data, otte_data;
vm_page_t om;
int invlva;
@@ -900,8 +894,7 @@
sched_pin();
tte_data = pa = VM_PAGE_TO_PHYS(m);
- otte = tte_hash_lookup(pmap->pm_hash, va, TRUE);
- otte_data = otte ? *otte : 0;
+ otte_data = tte_hash_lookup(pmap->pm_hash, va);
opa = TTE_GET_PA(otte_data);
/*
* Mapping has not changed, must be protection or wiring change.
@@ -972,9 +965,10 @@
else
tte_data |= TTE_MINFLAGS;
- if ((otte_data & ~(VTD_W|VTD_REF|VTD_LOCK)) != tte_data) {
+ otte_data = tte_hash_update(pmap->pm_hash, va, tte_data);
+ invlva = FALSE;
+ if ((otte_data & ~(VTD_W|VTD_REF)) != tte_data) {
if (otte_data & VTD_V) {
- invlva = FALSE;
if (otte_data & VTD_REF) {
if (otte_data & VTD_MANAGED)
vm_page_flag_set(om, PG_REFERENCED);
@@ -985,28 +979,16 @@
if ((otte_data & VTD_MANAGED) &&
pmap_track_modified(pmap, va))
vm_page_dirty(om);
- if ((prot & VM_PROT_WRITE) == 0)
+#if 0
+ if ((prot & VM_PROT_WRITE) == 0) /* XXX double check */
+#endif
invlva = TRUE;
}
- DPRINTF("update *otte 0x%lx -> 0x%lx\n", otte_data, tte_data);
- *otte = tte_data;
+ }
+ }
- if (!(otte_data & VTD_LOCK))
- tte_hash_bucket_unlock(pmap->pm_hash, va);
-#ifdef notyet
- if (invlva)
-#endif
- pmap_invalidate_page(pmap, va);
- } else {
- tte_hash_insert(pmap->pm_hash, va, tte_data);
- membar(Sync);
- }
- } else {
- if (!(*otte & VTD_LOCK))
- tte_hash_bucket_unlock(pmap->pm_hash, va);
- else
- *otte = (*otte & ~VTD_LOCK);
- }
+ if (invlva)
+ pmap_invalidate_page(pmap, va);
sched_unpin();
PMAP_UNLOCK(pmap);
@@ -1067,10 +1049,10 @@
pmap_extract(pmap_t pmap, vm_offset_t va)
{
vm_paddr_t pa;
- tte_t *tte;
+ tte_t tte_data;
- tte = tte_hash_lookup(pmap->pm_hash, va, FALSE);
- pa = TTE_GET_PA(*tte) | (va & TTE_GET_PAGE_MASK(*tte));
+ tte_data = tte_hash_lookup(pmap->pm_hash, va);
+ pa = TTE_GET_PA(tte_data) | (va & TTE_GET_PAGE_MASK(tte_data));
return (pa);
}
@@ -1083,17 +1065,17 @@
vm_page_t
pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
{
- tte_t *tte_data;
+ tte_t tte_data;
vm_page_t m;
m = NULL;
vm_page_lock_queues();
PMAP_LOCK(pmap);
sched_pin();
- tte_data = tte_hash_lookup(pmap->pm_hash, va, FALSE);
+ tte_data = tte_hash_lookup_nolock(pmap->pm_hash, va);
if (tte_data != 0 &&
- ((*tte_data & VTD_SW_W) || (prot & VM_PROT_WRITE) == 0)) {
- m = PHYS_TO_VM_PAGE(TTE_GET_PA(*tte_data));
+ ((tte_data & VTD_SW_W) || (prot & VM_PROT_WRITE) == 0)) {
+ m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
vm_page_hold(m);
}
sched_unpin();
@@ -1252,7 +1234,6 @@
invlpg(va, pmap->pm_context);
#ifdef SMP
-
pmap_ipi(pmap, (void *)tl_invlpg, (uint64_t)va, (uint64_t)pmap->pm_context);
#endif
spinlock_exit();
@@ -1349,7 +1330,7 @@
boolean_t
pmap_is_prefaultable(pmap_t pmap, vm_offset_t va)
{
- return (tte_hash_lookup(pmap->pm_hash, va, FALSE) == NULL);
+ return (tte_hash_lookup(pmap->pm_hash, va) == 0);
}
/*
@@ -1369,7 +1350,7 @@
vm_paddr_t
pmap_kextract(vm_offset_t va)
{
- uint64_t *tte, tte_data;
+ tte_t tte_data;
vm_paddr_t pa;
pa = 0;
@@ -1381,8 +1362,8 @@
if ((pa == 0) && (tte_data = tsb_lookup_tte(va, 0)) != 0)
pa = TTE_GET_PA(tte_data) | (va & TTE_GET_PAGE_MASK(tte_data));
- if ((pa == 0) && (tte = tte_hash_lookup(kernel_pmap->pm_hash, va, FALSE)) != NULL)
- pa = TTE_GET_PA(*tte) | (va & TTE_GET_PAGE_MASK(*tte));
+ if ((pa == 0) && (tte_data = tte_hash_lookup(kernel_pmap->pm_hash, va)) != 0)
+ pa = TTE_GET_PA(tte_data) | (va & TTE_GET_PAGE_MASK(tte_data));
return pa;
}
@@ -1390,7 +1371,7 @@
static void
pmap_kremove(vm_offset_t va)
{
- tte_hash_delete(kernel_pmap->pm_hash, va, FALSE);
+ tte_hash_delete(kernel_pmap->pm_hash, va);
}
static void
@@ -1540,7 +1521,6 @@
{
int anychanged;
- uint64_t *tte;
vm_offset_t tva;
DPRINTF("pmap_protect(0x%lx, 0x%lx, %d)\n", sva, eva, prot);
@@ -1562,40 +1542,32 @@
for (tva = sva; tva < eva; tva += PAGE_SIZE) {
uint64_t otte_data, tte_data;
vm_page_t m;
- retry:
- tte = tte_hash_lookup(pmap->pm_hash, tva, TRUE);
- if (tte == NULL)
+
+ if ((otte_data = tte_hash_lookup(pmap->pm_hash, tva)) == 0)
continue;
- otte_data = tte_data = *tte;
+ tte_data = otte_data;
+
+ tte_data &= ~(VTD_SW_W | VTD_W | VTD_REF);
+ otte_data = tte_hash_update(pmap->pm_hash, tva, tte_data);
+
+ if (tte_data != otte_data)
+ anychanged = 1;
- if (tte_data & VTD_MANAGED) {
+ if (otte_data & VTD_MANAGED) {
m = NULL;
- if (tte_data & VTD_REF) {
- m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
+
+ if (otte_data & VTD_REF) {
+ m = PHYS_TO_VM_PAGE(TTE_GET_PA(otte_data));
vm_page_flag_set(m, PG_REFERENCED);
- tte_data &= ~VTD_REF;
}
- if ((tte_data & VTD_W) && pmap_track_modified(pmap, tva)) {
- m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
+ if ((otte_data & VTD_W) && pmap_track_modified(pmap, tva)) {
+ m = PHYS_TO_VM_PAGE(TTE_GET_PA(otte_data));
vm_page_dirty(m);
}
- }
-
- tte_data &= ~(VTD_SW_W | VTD_W);
-
- if (tte_data != otte_data) {
- if (!atomic_cmpset_long(tte, otte_data, tte_data)) {
- /* XXX this should never happen with hash bucket locks - FIXME */
- panic("bucket locked but tte data changed");
- tte_hash_bucket_unlock(pmap->pm_hash, tva);
- goto retry;
- }
- anychanged = 1;
+
}
- tte_hash_bucket_unlock(pmap->pm_hash, tva);
}
-
sched_unpin();
if (anychanged)
@@ -1669,33 +1641,30 @@
{
int invlva;
vm_offset_t tva;
- uint64_t *tte;
+ uint64_t tte_data;
/*
* Perform an unsynchronized read. This is, however, safe.
*/
if (pmap->pm_stats.resident_count == 0)
return;
+ DPRINTF("pmap_remove(start=0x%lx, end=0x%lx)\n",
+ start, end);
invlva = 0;
vm_page_lock_queues();
sched_pin();
PMAP_LOCK(pmap);
for (tva = start; tva < end; tva += PAGE_SIZE) {
- if ((tte = tte_hash_lookup(pmap->pm_hash, tva, TRUE)) == NULL)
+ if ((tte_data = tte_hash_delete(pmap->pm_hash, tva)) == 0)
continue;
- pmap_remove_tte(pmap, tte, tva);
- tte_hash_delete(pmap->pm_hash, tva, TRUE);
-
+ pmap_remove_tte(pmap, tte_data, tva);
invlva = 1;
}
sched_unpin();
vm_page_unlock_queues();
- if (invlva) {
- if (pmap == kernel_pmap)
- pmap_invalidate_range(pmap, start, end);
- else
- pmap_invalidate_all(pmap);
- }
+ if (invlva)
+ pmap_invalidate_range(pmap, start, end);
+
PMAP_UNLOCK(pmap);
}
@@ -1717,7 +1686,7 @@
pmap_remove_all(vm_page_t m)
{
pv_entry_t pv;
- uint64_t *tte, tte_data;
+ uint64_t tte_data;
DPRINTF("pmap_remove_all 0x%lx\n", VM_PAGE_TO_PHYS(m));
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
@@ -1725,9 +1694,9 @@
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
PMAP_LOCK(pv->pv_pmap);
pv->pv_pmap->pm_stats.resident_count--;
- tte = tte_hash_lookup(pv->pv_pmap->pm_hash, pv->pv_va, TRUE);
- tte_data = *tte;
- tte_hash_delete(pv->pv_pmap->pm_hash, pv->pv_va, TRUE);
+
+ tte_data = tte_hash_delete(pv->pv_pmap->pm_hash, pv->pv_va);
+
if (tte_data & VTD_WIRED)
pv->pv_pmap->pm_stats.wired_count--;
if (tte_data & VTD_REF)
@@ -1790,17 +1759,17 @@
vm_page_t m;
pv_entry_t pv, npv;
- tte_t *tte, tte_data;
-
+ tte_t tte_data;
+
+ DPRINTF("pmap_remove_pages(ctx=0x%lx)\n", pmap->pm_context);
vm_page_lock_queues();
PMAP_LOCK(pmap);
sched_pin();
for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
- tte = tte_hash_lookup(pmap->pm_hash, pv->pv_va, FALSE);
- tte_data = tte ? *tte : 0;
+ tte_data = tte_hash_delete(pmap->pm_hash, pv->pv_va);
if (tte_data == 0) {
- printf("TTE at %p IS ZERO @ VA %016lx\n", tte, pv->pv_va);
+ printf("TTE IS ZERO @ VA %016lx\n", pv->pv_va);
panic("bad tte");
}
@@ -1815,7 +1784,9 @@
}
m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
pmap->pm_stats.resident_count--;
-
+ if ((uint64_t)m < KERNBASE)
+ panic("m=%p va=0x%lx tte_data=0x%lx",
+ m, pv->pv_va, tte_data);
if (tte_data & VTD_W) {
vm_page_dirty(m);
@@ -1823,7 +1794,7 @@
npv = TAILQ_NEXT(pv, pv_plist);
TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
-
+
m->md.pv_list_count--;
TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
if (TAILQ_EMPTY(&m->md.pv_list))
@@ -1831,7 +1802,6 @@
free_pv_entry(pv);
}
- tte_hash_delete_all(pmap->pm_hash);
sched_unpin();
pmap_invalidate_all(pmap);
@@ -1852,31 +1822,31 @@
}
}
static void
-pmap_remove_tte(pmap_t pmap, uint64_t *tte, vm_offset_t va)
+pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va)
{
vm_page_t m;
if (pmap != kernel_pmap)
- DPRINTF("pmap_remove_tte(va=0x%lx, pa=0x%lx)\n", va, TTE_GET_PA(*tte));
+ DPRINTF("pmap_remove_tte(va=0x%lx, pa=0x%lx)\n", va, TTE_GET_PA(tte_data));
+
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- if (*tte & VTD_WIRED)
- pmap->pm_stats.wired_count -= 1;
+ if (tte_data & VTD_WIRED)
+ pmap->pm_stats.wired_count--;
- pmap->pm_stats.resident_count -= 1;
+ pmap->pm_stats.resident_count--;
- if (*tte & VTD_MANAGED) {
- m = PHYS_TO_VM_PAGE(TTE_GET_PA(*tte));
- if (*tte & VTD_W) {
+ if (tte_data & VTD_MANAGED) {
+ m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data));
+ if (tte_data & VTD_W) {
if (pmap_track_modified(pmap, va))
vm_page_dirty(m);
}
- if (*tte & VTD_REF)
+ if (tte_data & VTD_REF)
vm_page_flag_set(m, PG_REFERENCED);
pmap_remove_entry(pmap, m, va);
}
-
}
/*
@@ -1921,21 +1891,6 @@
void
pmap_zero_page_idle(vm_page_t m)
{
- int i;
- for (i = 0; i < (nucleus_memory >> PAGE_SHIFT_4M); i++) {
- if (VM_PAGE_TO_PHYS(m) >= nucleus_mappings[i] &&
- VM_PAGE_TO_PHYS(m) < nucleus_mappings[i] + PAGE_SIZE_4M)
- panic("zeroing nucleus");
-
- if (pmap_kextract(TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m))) >=
- nucleus_mappings[i] &&
- pmap_kextract(TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m))) <
- nucleus_mappings[i] + PAGE_SIZE_4M)
- panic("zeroing nucleus in direct area");
- }
-
-
-
bzero((char *)TLB_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)), PAGE_SIZE);
}
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte.c#7 (text+ko) ====
@@ -24,8 +24,6 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
-
-
* $ Exp $
*/
@@ -59,7 +57,6 @@
tte_clear_phys_bit(vm_page_t m, uint64_t flags)
{
pv_entry_t pv;
- uint64_t tte_data, *tte;
if ((m->flags & PG_FICTITIOUS) ||
(flags == VTD_SW_W && (m->flags & PG_WRITEABLE) == 0))
@@ -71,6 +68,7 @@
* setting RO do we need to clear the VAC?
*/
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ tte_t otte_data, tte_data;
/*
* don't write protect pager mappings
*/
@@ -78,24 +76,23 @@
if (!pmap_track_modified(pv->pv_pmap, pv->pv_va))
continue;
}
- PMAP_LOCK(pv->pv_pmap);
- tte = tte_hash_lookup(pv->pv_pmap->pm_hash, pv->pv_va, FALSE);
- retry:
- tte_data = *tte;
- if (tte_data & flags) {
+ otte_data = tte_hash_lookup(pv->pv_pmap->pm_hash, pv->pv_va);
+
+ if (otte_data & flags) {
if (flags == VTD_SW_W) {
- if (!atomic_cmpset_long((u_long *)tte, tte_data,
- tte_data & ~(VTD_SW_W | VTD_W)))
- goto retry;
- if (tte_data & VTD_W) {
- vm_page_dirty(m);
- }
+ tte_data = otte_data & ~(VTD_SW_W | VTD_W);
} else {
- atomic_clear_long((u_long *)tte, flags);
+ tte_data = otte_data & ~flags;
}
+ otte_data = tte_hash_update(pv->pv_pmap->pm_hash, pv->pv_va, tte_data);
+
+ if (otte_data & VTD_W)
+ vm_page_dirty(m);
+
pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
}
- PMAP_UNLOCK(pv->pv_pmap);
+
+
}
if (flags == VTD_SW_W)
vm_page_flag_clear(m, PG_WRITEABLE);
@@ -118,12 +115,12 @@
void
tte_clear_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags)
{
- tte_t *tte;
+ tte_t tte_data;
- tte = tte_hash_lookup(pmap->pm_hash, va, FALSE);
+ tte_data = tte_hash_lookup(pmap->pm_hash, va);
- if (tte) {
- atomic_clear_long((u_long *)tte, flags);
+ if (tte_data) {
+ tte_hash_update(pmap->pm_hash, va, tte_data & ~flags);
pmap_invalidate_page(pmap, va);
}
}
@@ -137,11 +134,9 @@
boolean_t
tte_get_virt_bit(struct pmap *pmap, vm_offset_t va, uint64_t flags)
{
- tte_t ttedata, *tte;
+ tte_t tte_data;
- tte = tte_hash_lookup(pmap->pm_hash, va, FALSE);
+ tte_data = tte_hash_lookup(pmap->pm_hash, va);
- ttedata = tte ? *tte : 0;
-
- return ((ttedata & flags) == flags);
+ return ((tte_data & flags) == flags);
}
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#16 (text+ko) ====
@@ -59,6 +59,8 @@
#define HASH_MASK(th) ((th->th_size << (PAGE_SHIFT - THE_SHIFT)) - 1)
#define HASH_VALID 0x1
+#define DEBUG
+
struct tte_hash_entry;
typedef union {
@@ -121,7 +123,8 @@
hash_bucket_lock(tte_hash_field_t fields)
{
uint64_t data;
-
+
+ spinlock_enter();
data = fields[0].tte.data & ~VTD_LOCK;
while (atomic_cmpset_long(&fields[0].tte.data, data, data | VTD_LOCK))
data = fields[0].tte.data & ~VTD_LOCK;
@@ -130,25 +133,6 @@
}
-void
-tte_hash_bucket_unlock(tte_hash_t th, vm_offset_t va)
-{
- uint64_t hash_shift, hash_index;
- tte_hash_field_t fields;
-
- /* XXX - only handle 8K pages for now */
- hash_shift = PAGE_SHIFT;
- hash_index = (va >> hash_shift) & HASH_MASK(th);
- fields = (th->th_hashtable[hash_index].the_fields);
-
-#ifdef DEBUG
- if ((fields[0].tte.data & VTD_LOCK) == 0)
- panic("trying to unlock bucket that isn't locked");
-#endif
- fields[0].tte.data &= ~VTD_LOCK;
- membar(StoreLoad);
-}
-
static __inline void
hash_bucket_unlock_inline(tte_hash_field_t fields)
{
@@ -158,6 +142,7 @@
#endif
fields[0].tte.data &= ~VTD_LOCK;
membar(StoreLoad);
+ spinlock_exit();
}
void
@@ -233,12 +218,6 @@
}
void
-tte_hash_clear(tte_hash_t th)
-{
- bzero(th->th_hashtable, th->th_size*PAGE_SIZE);
-}
-
-void
tte_hash_destroy(tte_hash_t th)
{
vm_page_t m, hash_pages[MAX_HASH_SIZE];
@@ -264,52 +243,74 @@
}
-int
-tte_hash_delete(tte_hash_t th, vm_offset_t va, int locked)
+static __inline tte_t
+tte_hash_lookup_inline(tte_hash_t th, vm_offset_t va, int *index)
+{
+ uint64_t hash_shift, hash_index;
+ tte_hash_field_t fields;
+ int i;
+ tte_t entry;
+ /* XXX - only handle 8K pages for now */
+
+ hash_shift = PAGE_SHIFT;
+ hash_index = (va >> hash_shift) & HASH_MASK(th);
+ fields = (th->th_hashtable[hash_index].the_fields);
+ entry = 0;
+
+ for (i = 0; i < 4 && fields[i].tte.tag != 0; i++) {
+ if (((fields[i].tte.tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M))) {
+ entry = (fields[i].tte.data & ~VTD_LOCK);
+ break;
+ }
+ }
+ if (index)
+ *index = i;
+ /*
+ * XXX handle the case of collisions > 3
+ */
+ return (entry);
+}
+
+
+tte_t
+tte_hash_delete(tte_hash_t th, vm_offset_t va)
{
uint64_t hash_shift, hash_index;
tte_hash_field_t fields;
int i, vaindex, lastindex;
+ tte_t tte_data;
+
/* XXX - only handle 8K pages for now */
hash_shift = PAGE_SHIFT;
hash_index = (va >> hash_shift) & HASH_MASK(th);
fields = (th->th_hashtable[hash_index].the_fields);
+
+ tte_data = 0;
-#if 0
- KASSERT(tte_hash_lookup(th, va, FALSE) != 0, ("attempting to delete non-existent entry"));
-#endif
- if (locked == FALSE)
- hash_bucket_lock(fields);
+ hash_bucket_lock(fields);
+
+ tte_data = tte_hash_lookup_inline(th, va, &vaindex);
- for (i = 0; i <= 3; i++)
- if ((fields[i].tte.tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M))
- break;
- vaindex = i;
+ if (tte_data == 0)
+ goto done;
- for (i = 0; i <= 3; i++)
- if (fields[i].tte.tag == 0)
- break;
+ for (i = 0; (i < 4) && (fields[i + 1].tte.tag != 0); i++)
+ ;
lastindex = i;
- if (vaindex != lastindex && (vaindex < 4)) {
+ if (vaindex != lastindex) {
fields[vaindex].tte.tag = fields[lastindex].tte.tag;
- fields[vaindex].tte.data = fields[lastindex].tte.data;
+ fields[vaindex].tte.data = fields[lastindex].tte.data |
+ (fields[vaindex].tte.data & VTD_LOCK);
}
fields[lastindex].tte.tag = 0;
- fields[lastindex].tte.data = 0;
-
- if (lastindex != 0)
- hash_bucket_unlock_inline(fields);
+ fields[lastindex].tte.data = 0 | (fields[lastindex].tte.data & VTD_LOCK);
- return (vaindex < 4);
-}
+done:
+ hash_bucket_unlock_inline(fields);
-void
-tte_hash_delete_all(tte_hash_t th)
-{
-
- bzero(th->th_hashtable, th->th_size*PAGE_SIZE);
+ return (tte_data);
}
void
@@ -318,77 +319,69 @@
uint64_t hash_shift, hash_index, tte_tag;
tte_hash_field_t fields;
- int i;
+ int cookie;
+ tte_t otte_data;
+
-
-#if 0
- tte_t *tte;
- tte = tte_hash_lookup(th, va, FALSE);
- if (tte)
- panic("mapping for va=0x%lx already exists tte_data=0x%lx\n", va, *tte);
-#endif
/* XXX - only handle 8K pages for now */
hash_shift = PAGE_SHIFT;
hash_index = (va >> hash_shift) & HASH_MASK(th);
fields = (th->th_hashtable[hash_index].the_fields);
- hash_bucket_lock(fields);
tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
- for (i = 0; i <= 3; i++) {
- if ((fields[i].tte.tag == 0) || (fields[i].tte.tag == tte_tag)) {
- fields[i].tte.data = tte_data | (i ? 0 : VTD_LOCK);
- fields[i].tte.tag = tte_tag;
- goto done;
- }
+ hash_bucket_lock(fields);
+ otte_data = tte_hash_lookup_inline(th, va, &cookie);
+#ifdef DEBUG
+ if (otte_data)
+ panic("mapping for va=0x%lx already exists tte_data=0x%lx\n", va, otte_data);
+#endif
+ fields[cookie].tte.data = tte_data | (fields[cookie].tte.data & VTD_LOCK);
+ fields[cookie].tte.tag = tte_tag;
- }
- panic("collision handling unimplemented - please re-consider");
+ if (cookie == 4)
+ panic("collision handling unimplemented - please re-consider");
-done:
hash_bucket_unlock_inline(fields);
th->th_entries++;
}
+
+
+tte_t
+tte_hash_lookup_nolock(tte_hash_t th, vm_offset_t va)
+{
+ return tte_hash_lookup_inline(th, va, NULL);
+}
+
+
/*
* If leave_locked is true the tte's data field will be returned to
* the caller with the hash bucket left locked
*/
-tte_t *
-tte_hash_lookup(tte_hash_t th, vm_offset_t va, int leave_locked)
+tte_t
+tte_hash_lookup(tte_hash_t th, vm_offset_t va)
{
uint64_t hash_shift, hash_index;
tte_hash_field_t fields;
- int i;
- tte_t *entry;
+ tte_t tte_data;
/* XXX - only handle 8K pages for now */
hash_shift = PAGE_SHIFT;
hash_index = (va >> hash_shift) & HASH_MASK(th);
fields = (th->th_hashtable[hash_index].the_fields);
- entry = NULL;
hash_bucket_lock(fields);
- for (i = 0; i <= 3; i++) {
- if (((fields[i].tte.tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M)) &&
- (fields[i].tte.data != 0)) {
- entry = &(fields[i].tte.data);
- break;
- }
- }
- if (entry == NULL || leave_locked == FALSE)
- hash_bucket_unlock_inline(fields);
+ tte_data = tte_hash_lookup_inline(th, va, NULL);
+ hash_bucket_unlock_inline(fields);
- /*
- * XXX handle the case of collisions > 3
- *
- */
- return (entry);
+ return (tte_data);
}
+
uint64_t
tte_hash_set_scratchpad_kernel(tte_hash_t th)
{
@@ -418,3 +411,30 @@
return hash_scratch;
}
+tte_t
+tte_hash_update(tte_hash_t th, vm_offset_t va, tte_t tte_data)
+{
+ uint64_t hash_shift, hash_index;
+ tte_hash_field_t fields;
+ int cookie;
+ tte_t otte_data;
+
+ /* XXX - only handle 8K pages for now */
+ hash_shift = PAGE_SHIFT;
+ hash_index = (va >> hash_shift) & HASH_MASK(th);
+ fields = (th->th_hashtable[hash_index].the_fields);
+
+ hash_bucket_lock(fields);
+ otte_data = tte_hash_lookup_inline(th, va, &cookie);
+#ifdef TTE_DEBUG
+ printf("tte_hash_update(va=0x%lx, tte_data=0x%lx, index=%d)\n", va, tte_data, cookie);
+#endif
+ fields[cookie].tte.tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
+ fields[cookie].tte.data = tte_data | (fields[cookie].tte.data & VTD_LOCK);
+ hash_bucket_unlock_inline(fields);
+
+ if (otte_data == 0)
+ th->th_entries++;
+
+ return otte_data;
+}
More information about the p4-projects
mailing list