PERFORCE change 92773 for review
Kip Macy
kmacy at FreeBSD.org
Sun Mar 5 02:18:22 PST 2006
http://perforce.freebsd.org/chv.cgi?CH=92773
Change 92773 by kmacy at kmacy_storage:sun4v_work on 2006/03/05 10:18:20
add basic TSB miss handling support
Affected files ...
.. //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#6 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/param.h#5 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/sun4v_cpufunc.h#2 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#8 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#3 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#17 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#16 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#6 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#7 edit
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#3 edit
Differences ...
==== //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#6 (text+ko) ====
@@ -71,6 +71,11 @@
#include <machine/tsb.h>
#include <machine/tstate.h>
#include <machine/utrap.h>
+#ifdef SUN4V
+#include <machine/mmu.h>
+#include <machine/tte_hash.h>
+#endif
+
ASSYM(KERNBASE, KERNBASE);
ASSYM(VM_MIN_PROM_ADDRESS, VM_MIN_PROM_ADDRESS);
@@ -100,13 +105,16 @@
ASSYM(TLB_DEMAP_CONTEXT, TLB_DEMAP_CONTEXT);
ASSYM(TLB_DEMAP_PAGE, TLB_DEMAP_PAGE);
+#ifndef SUN4V
ASSYM(TSB_BUCKET_MASK, TSB_BUCKET_MASK);
ASSYM(TSB_BUCKET_SHIFT, TSB_BUCKET_SHIFT);
-
+#endif
ASSYM(INT_SHIFT, INT_SHIFT);
ASSYM(PTR_SHIFT, PTR_SHIFT);
ASSYM(PAGE_SHIFT, PAGE_SHIFT);
+ASSYM(PAGE_MASK, PAGE_MASK);
+ASSYM(PAGE_MASK_4M, PAGE_MASK_4M);
ASSYM(PAGE_SHIFT_8K, PAGE_SHIFT_8K);
ASSYM(PAGE_SHIFT_4M, PAGE_SHIFT_4M);
ASSYM(PAGE_SIZE, PAGE_SIZE);
@@ -170,6 +178,11 @@
ASSYM(TS_MAX, TS_MAX);
ASSYM(TLB_DIRECT_TO_TTE_MASK, TLB_DIRECT_TO_TTE_MASK);
ASSYM(TV_SIZE_BITS, TV_SIZE_BITS);
+#else
+ASSYM(VTD_REF, VTD_REF);
+ASSYM(TTARGET_VA_MASK, TTARGET_VA_MASK);
+ASSYM(TTARGET_VA_BITS, TTARGET_VA_BITS);
+ASSYM(THE_SHIFT, THE_SHIFT);
#endif
ASSYM(V_INTR, offsetof(struct vmmeter, v_intr));
@@ -198,6 +211,11 @@
ASSYM(PC_RQ_SIZE, offsetof(struct pcpu, pc_rq_size));
ASSYM(PC_NRQ_BASE, offsetof(struct pcpu, pc_nrq_ra));
ASSYM(PC_NRQ_SIZE, offsetof(struct pcpu, pc_nrq_size));
+
+ASSYM(PC_KWBUF_FULL, offsetof(struct pcpu, pc_kwbuf_full));
+ASSYM(PC_KWBUF_SP, offsetof(struct pcpu, pc_kwbuf_sp));
+ASSYM(PC_KWBUF, offsetof(struct pcpu, pc_kwbuf));
+
#else
ASSYM(PC_PMAP, offsetof(struct pcpu, pc_pmap));
ASSYM(PM_TSB, offsetof(struct pmap, pm_tsb));
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/param.h#5 (text+ko) ====
@@ -120,6 +120,7 @@
/*
* Mach derived conversion macros
*/
+#ifndef LOCORE
#define round_page(x) (((unsigned long)(x) + PAGE_MASK) & ~PAGE_MASK)
#define trunc_page(x) ((unsigned long)(x) & ~PAGE_MASK)
@@ -130,10 +131,8 @@
#define sparc64_ptob(x) ((unsigned long)(x) << PAGE_SHIFT)
#define pgtok(x) ((unsigned long)(x) * (PAGE_SIZE / 1024))
+#endif /* LOCORE */
-#define NPGPTD 1 /* number of page table directory pages */
-#define NBPTD (NPGPTD << PAGE_SHIFT) /* number of bytes in a page table directory */
-#define NPDEPG (PAGE_SIZE/(sizeof (vm_offset_t)))
#endif /* !_MACHINE_PARAM_H_ */
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/sun4v_cpufunc.h#2 (text+ko) ====
@@ -32,7 +32,7 @@
#define _MACHINE_SUN4V_CPUFUNC_H_
#include <machine/hypervisor_api.h>
void set_mmfsa_scratchpad(vm_paddr_t mmfsa);
-void set_hash_scratchpad(void *hash);
+void set_hash_scratchpad(vm_offset_t hash);
void set_tsb_scratchpad(vm_paddr_t tsb);
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#8 (text+ko) ====
@@ -38,4 +38,6 @@
void tsb_clear_range(struct hv_tsb_info *tsb, vm_offset_t sva, vm_offset_t eva);
+void tsb_set_scratchpad(struct hv_tsb_info *tsb);
+
#endif /* !_MACHINE_TSB_H_ */
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#3 (text+ko) ====
@@ -1,6 +1,7 @@
#ifndef _MACHINE_TTE_HASH_H_
#define _MACHINE_TTE_HASH_H_
+#define THE_SHIFT 6 /* size of hash entry is 64-bytes */
struct tte_hash;
typedef struct tte_hash *tte_hash_t;
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#17 (text+ko) ====
@@ -42,6 +42,7 @@
#include <machine/trap.h>
#include <machine/tstate.h>
#include <machine/wstate.h>
+#include <machine/hypervisorvar.h>
#include "assym.s"
@@ -228,7 +229,7 @@
wrpr %g1, WSTATE_NESTED, %wstate
save %sp, -(CCFSZ + TF_SIZEOF), %sp
#endif
- nop
+ illtrap
.endm
.macro tl1_setup type
@@ -252,32 +253,40 @@
.macro insn_excptn
MAGIC_TRAP_ON
+ illtrap
.align 32
.endm
.macro insn_miss
MAGIC_TRAP_ON
+ illtrap
.align 32
.endm
.macro data_excptn
MAGIC_TRAP_ON
+ illtrap
.align 32
.endm
.macro data_miss
- MAGIC_TRAP_ON
-
+ GET_MMFSA_SCRATCH(%g1) ! insn 1
+ GET_HASH_SCRATCH(%g2) ! insn 2,3
+ GET_TSB_SCRATCH(%g3) ! insn 4,5
+ ba,pt %xcc, tsb_miss
+ add %g1, MMFSA_D_, %g1 ! set fsa to data
.align 32
.endm
.macro data_prot
MAGIC_TRAP_ON
+ illtrap
.align 32
.endm
.macro tl0_align
MAGIC_TRAP_ON
+ illtrap
.align 32
.endm
@@ -1007,6 +1016,134 @@
END(tl0_intr)
+! %g1==mmfsa (RA)
+! %g2==hash base (VA)
+! %g3==TSB (RA)
+! internal usage:
+! %g1==absolute index
+! %g4==fault type,entry tag
+! %g5==fault address,entry data
+! %g6==hash size,tag, temp
+! %g7 temp
+ENTRY(tsb_miss)
+ ldda [%g0 + %g1]ASI_LDTD_REAL, %g4
+ ! %g4 == fault type %g5 == fault address
+ ! ignore context for now
+ ! XXX only handle normal miss for now
+ mov 1, %g7
+ sllx %g7, PAGE_SHIFT, %g7
+ sub %g7, 1, %g7 ! %g7==PAGE_MASK
+
+ MAGIC_TRAP_ON
+ and %g2, %g7, %g6 ! size stored in lower 13 bits
+ andn %g2, %g7, %g2 ! actual VA of hash
+
+ ! XXX only handle 8k page miss
+ ! calculate hash index
+ srlx %g5, PAGE_SHIFT, %g1 ! absolute hash index
+ sllx %g6, (PAGE_SHIFT - THE_SHIFT), %g6 ! size of hash in THEs
+ sub %g6, 1, %g6 ! THE_MASK
+ and %g1, %g6, %g6 ! masked hash index
+ sllx %g6, THE_SHIFT, %g6 ! masked hash offset
+ srlx %g5, PAGE_SHIFT_4M, %g7 ! VA tag
+ ! fetch hash entries - exit when we find what were looking for
+
+ ! %g2==entry base
+ add %g2, %g6, %g2 ! base + offset == entry base
+
+ ! entry 0
+tsb_miss_lookup_0:
+ mov 1, %g6
+ sllx %g6, TTARGET_VA_BITS, %g6
+ subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK
+
+ ldda [%g2 + %g0]ASI_LDTD_N, %g4
+ and %g4, %g6, %g6 ! mask off context bits
+ cmp %g6, %g0 ! entry tag == 0
+ be,pn %xcc, 1f
+ nop
+ cmp %g6, %g7 ! entry tag == VA tag?
+ be,pn %xcc, 2f
+ nop
+ ! entry 1
+tsb_miss_lookup_1:
+ mov 1, %g6
+ sllx %g6, TTARGET_VA_BITS, %g6
+ subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK
+
+ add %g2, 16, %g2
+ ldda [%g2 + %g0]ASI_LDTD_N, %g4
+ and %g4, %g6, %g6 ! mask off context bits
+ cmp %g6, %g0 ! entry tag == 0
+ be,pn %xcc, 1f
+ nop
+ cmp %g6, %g7 ! entry tag == VA tag?
+ be,pn %xcc, 2f
+ nop
+ ! entry 2
+tsb_miss_lookup_2:
+ mov 1, %g6
+ sllx %g6, TTARGET_VA_BITS, %g6
+ subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK
+
+ add %g2, 16, %g2
+ ldda [%g2 + %g0]ASI_LDTD_N, %g4
+ and %g4, %g6, %g6 ! mask off context bits
+ cmp %g6, %g0 ! entry tag == 0
+ be,pn %xcc, 1f
+ nop
+ cmp %g6, %g7 ! entry tag == VA tag?
+ be,pn %xcc, 2f
+ nop
+ ! entry 3
+tsb_miss_lookup_3:
+ mov 1, %g6
+ sllx %g6, TTARGET_VA_BITS, %g6
+ subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK
+
+ add %g2, 16, %g2
+ ldda [%g2 + %g0]ASI_LDTD_N, %g4
+ and %g4, %g6, %g6 ! mask off context bits
+ cmp %g6, %g0 ! entry tag == 0
+ be,pn %xcc, 1f
+ nop
+ cmp %g6, %g7 ! entry tag == VA tag?
+ be,pn %xcc, 2f
+ nop
+tsb_miss_not_found:
+1: ! not found
+ ! we need to jump to tl0_trap to drop us back down to tl0
+ ! and take us to trap(...) to service the fault
+ ! skipping this step for the moment so we just do an illtrap
+ illtrap
+
+tsb_miss_found:
+2: !found
+ ! set referenced bit unconditionally for now
+ or %g5, VTD_REF, %g5
+ stx %g5, [%g2 + 8] ! set ref bit
+
+ mov 1, %g7
+ sllx %g7, PAGE_SHIFT, %g7
+ sub %g7, 1, %g7 ! %g7==PAGE_MASK
+
+ and %g3, %g7, %g6 ! size of TSB in pages
+
+ andn %g3, %g7, %g3 ! TSB real address
+ sllx %g6, (PAGE_SHIFT - TTE_SHIFT), %g6 ! nttes
+ subx %g6, 1, %g6 ! TSB_MASK
+ and %g6, %g1, %g6 ! masked index
+ sllx %g6, TTE_SHIFT, %g6 ! masked byte offset
+ add %g6, %g3, %g6 ! TTE RA
+ mov 8, %g7
+ stxa %g4, [%g6]ASI_REAL ! store tag
+ stxa %g5, [%g6 + %g7]ASI_REAL ! store data
+ MAGIC_TRAP_OFF
+ retry
+END(tsb_miss)
+
+
+
/*
* Freshly forked processes come here when switched to for the first time.
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#16 (text+ko) ====
@@ -355,7 +355,7 @@
{
struct pmap *pm;
vm_offset_t off, va, kernel_hash;
- vm_paddr_t pa;
+ vm_paddr_t pa, kernel_hash_pa;
vm_size_t physsz, virtsz;
ihandle_t pmem, vmem;
int i, sz, j;
@@ -411,18 +411,26 @@
virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT));
vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz;
-
+ /*
+ * Set the start and end of kva. The kernel is loaded at the first
+ * available 4 meg super page, so round up to the end of the page.
+ */
+ virtual_avail = roundup2(ekva, PAGE_SIZE_4M);
+ virtual_end = vm_max_kernel_address;
+ kernel_vm_end = vm_max_kernel_address;
/*
* Allocate and map a 4MB page for the kernel hashtable
*
*/
- pa = pmap_bootstrap_alloc(PAGE_SIZE_4M);
- if (pa & PAGE_MASK_4M)
- panic("pmap_bootstrap: hashtable unaligned\n");
+ kernel_hash_pa = pmap_bootstrap_alloc(PAGE_SIZE_4M);
+ if (kernel_hash_pa & PAGE_MASK_4M)
+ panic("pmap_bootstrap: hashtable pa unaligned\n");
kernel_hash = virtual_avail;
+ if (kernel_hash & PAGE_MASK_4M)
+ panic("pmap_bootstrap: hashtable va unaligned\n");
virtual_avail += PAGE_SIZE_4M;
- pmap_scrub_pages(pa, PAGE_SIZE_4M);
+ pmap_scrub_pages(kernel_hash_pa, PAGE_SIZE_4M);
/*
* Set up TSB descriptors for the hypervisor
@@ -453,7 +461,7 @@
kernel_pmap->pm_tsb.hvtsb_rsvd = 0;
kernel_pmap->pm_tsb.hvtsb_pa = pa;
- set_tsb_scratchpad(pa);
+ tsb_set_scratchpad(&kernel_pmap->pm_tsb);
/*
* Initialize kernel TSB for 4M pages
@@ -480,7 +488,7 @@
*
*/
tsb_set_tte(&kernel_td[TSB4M_INDEX], kernel_hash,
- pa | TTE_KERNEL | VTD_4M, 0);
+ kernel_hash_pa | TTE_KERNEL | VTD_4M, 0);
/*
* allocate MMU fault status areas for all CPUS
@@ -501,15 +509,6 @@
#endif
/*
- * Set the start and end of kva. The kernel is loaded at the first
- * available 4 meg super page, so round up to the end of the page.
- */
- virtual_avail = roundup2(ekva, PAGE_SIZE_4M);
- virtual_end = vm_max_kernel_address;
- kernel_vm_end = vm_max_kernel_address;
-
-
- /*
* Allocate a kernel stack with guard page for thread0 and map it into
* the kernel tsb.
*/
@@ -603,6 +602,7 @@
* attempts to do updates until they're legal
*/
pm->pm_hash = tte_hash_kernel_create(kernel_hash, PAGE_SIZE_4M);
+ tte_hash_set_scratchpad(pm->pm_hash);
/*
* XXX - We should read the kernel mappings into the hash table
@@ -989,7 +989,7 @@
void
pmap_kenter(vm_offset_t va, vm_paddr_t pa)
{
- tsb_set_tte(&kernel_td[TSB8K_INDEX], va, pa | TTE_KERNEL | VTD_8K, 0);
+ tte_hash_insert(kernel_pmap->pm_hash, va, pa | TTE_KERNEL | VTD_8K);
}
/*
@@ -1022,7 +1022,7 @@
tsb_get_tte(&kernel_td[TSB4M_INDEX], va) != 0)
tsb_clear_tte(&kernel_td[TSB4M_INDEX], va);
else
- tsb_clear_tte(&kernel_td[TSB8K_INDEX], va);
+ tte_hash_delete(kernel_pmap->pm_hash, va);
}
static void
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#6 (text+ko) ====
@@ -39,7 +39,8 @@
#define PCB_REG %g6
-
+#define MAGIC_TRAP_ON ta 0x77
+#define MAGIC_TRAP_OFF ta 0x78
/*
* void cpu_throw(struct thread *old, struct thread *new)
*/
@@ -54,6 +55,7 @@
* void cpu_switch(struct thread *old, struct thread *new)
*/
ENTRY(cpu_switch)
+ MAGIC_TRAP_ON
GET_PCB(PCB_REG)
save %sp, -CCFSZ, %sp
mov %i1, %i0
@@ -243,7 +245,7 @@
stxa %i4, [%i5] ASI_IMMU
mov AA_DMMU_PCXR, %i5
stxa %i3, [%i5] ASI_DMMU
- membar #Sync
+ membar #Sync /* Check the hypervisor spec */
#endif
/*
* Done. Return and load the new process's window from the stack.
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#7 (text+ko) ====
@@ -184,3 +184,10 @@
return tte_data;
}
+void
+tsb_set_scratchpad(hv_tsb_info_t *tsb)
+{
+ uint64_t tsb_pages;
+ tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT);
+ set_tsb_scratchpad(tsb->hvtsb_pa | tsb_pages);
+}
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#3 (text+ko) ====
@@ -26,7 +26,7 @@
#define HASH_SIZE 4
#define MAX_HASH_SIZE 16
-#define HASH_MASK(th) ((th->th_size << PAGE_SHIFT) - 1)
+#define HASH_MASK(th) ((th->th_size << (PAGE_SHIFT - THE_SHIFT)) - 1)
#define HASH_VALID 0x1
struct tte_hash_entry;
@@ -106,6 +106,8 @@
th->th_size = (size >> PAGE_SHIFT);
th->th_entries = 0;
th->th_context = 0;
+ printf("setting kernel hashtable to %lx\n", va);
+ th->th_hashtable = (tte_hash_entry_t)va;
return th;
}
@@ -217,15 +219,17 @@
hash_shift = PAGE_SHIFT;
hash_index = (va >> hash_shift) & HASH_MASK(th);
+
fields = (th->th_hashtable[hash_index].the_fields);
- tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)||(va >> TTARGET_VA_SHIFT));
-
+ tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT));
for (i = 0; i <= 3; i++) {
- if ((fields[i].tte.data == 0) || (fields[i].tte.tag == tte_tag)) {
+ if ((fields[i].tte.tag == 0) || (fields[i].tte.tag == tte_tag)) {
fields[i].tte.data = tte_data;
fields[i].tte.tag = tte_tag;
+ printf("data: 0x%016lx tag: 0x%016lx\n", fields[i].tte.data, fields[i].tte.tag);
goto done;
- }
+ }
+
}
panic("collision handling unimplemented - please re-consider");
@@ -259,6 +263,11 @@
void
tte_hash_set_scratchpad(tte_hash_t th)
{
- set_hash_scratchpad(th->th_hashtable);
+ /* This will break if a hash table ever grows above 64MB
+ * 2^(13+13)
+ */
+ printf("setting hash scratch to %lx\n",
+ ((vm_offset_t)th->th_hashtable) | ((vm_offset_t)th->th_size));
+ set_hash_scratchpad(((vm_offset_t)th->th_hashtable) | ((vm_offset_t)th->th_size));
}
More information about the p4-projects
mailing list