PERFORCE change 96461 for review
Kip Macy
kmacy at FreeBSD.org
Mon May 1 01:31:31 UTC 2006
http://perforce.freebsd.org/chv.cgi?CH=96461
Change 96461 by kmacy at kmacy_storage:sun4v_rwbuf on 2006/05/01 01:31:10
add bucket locking in TSB miss handler
simplify logic somewhat by using local registers
Affected files ...
.. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#44 edit
Differences ...
==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#44 (text+ko) ====
@@ -260,7 +260,14 @@
.endm
.macro insn_excptn
- tl0_gen T_RESERVED
+ GET_MMFSA_SCRATCH(%g1)
+ mov MMFSA_D_ADDR, %g2
+ ldxa [%g1 + %g2]ASI_REAL, %g3
+ sub %g0, 1, %g4
+ set trap, %g1
+ ba %xcc, tl0_trap
+ mov T_INSTRUCTION_EXCEPTION, %g2
+
.align 32
.endm
@@ -310,12 +317,14 @@
.endm
.macro tl0_align
- GET_MMFSA_SCRATCH(%g1)
+ GET_MMFSA_SCRATCH(%g1)
mov MMFSA_D_ADDR, %g3
- mov MMFSA_D_CTX, %g7
- ldxa [%g1 + %g3]ASI_REAL, %g5
+! mov MMFSA_D_CTX, %g7
+ ldxa [%g1 + %g3]ASI_REAL, %g3
+ ldxa [%g1 + %g7]ASI_REAL, %g4
+ or %g4, %g3, %g3
ba,pt %xcc, tl0_trap
- mov T_MEM_ADDRESS_NOT_ALIGNED, %g3
+ mov T_MEM_ADDRESS_NOT_ALIGNED, %g2
.align 32
.endm
@@ -330,6 +339,8 @@
.endm
.macro resumable_error
+ MAGIC_TRAP_ON
+ MAGIC_EXIT
clr %g3
sub %g0, 1, %g4
set trap, %g1
@@ -935,6 +946,7 @@
add %sp, REGOFF + SPOFF, %l7
ENTRY(user_rtt)
GET_PCPU_SCRATCH
+
! pil handling needs to be re-visited
wrpr %g0, PIL_TICK, %pil
ldx [PCPU(CURTHREAD)], %l4
@@ -964,6 +976,7 @@
ba,a %xcc, user_rtt
nop
2:
+
ld [%l7 + TF_WSTATE], %l3
!
! restore user globals and outs
@@ -1321,9 +1334,33 @@
+#define HASH_LOOKUP(addr, tag, searchtag, endlabel, matchlabel) \
+ ldda [addr]%asi, tag ; \
+ cmp tag, searchtag ; \
+ be,pn %xcc, matchlabel ; \
+ nop ; \
+ cmp tag, %g0 ; \
+ be,pn %xcc, endlabel ; \
+ nop
+#define RESTORE_TRAPWIN(pcpu, cansave, label1, label2) \
+ brz cansave, label1 ## f; \
+ nop ; \
+ mov -1, %l0 ; \
+ mov -1, %l7 ; \
+ restore ; \
+ ba,a,pt %xcc, label2 ## f ; \
+label1: ; \
+ rdpr %tl, cansave ; \
+ dec cansave ; \
+ sll cansave, RW_SHIFT, cansave ; \
+ add cansave, PC_KWBUF, cansave ; \
+ add pcpu, cansave, cansave ; \
+ RESTORE_LOCALS_ASI(cansave) ; \
+label2:
+
! %g3==trap type
! %g4==fault type (if data miss)
! %g5==fault addr
@@ -1339,216 +1376,251 @@
ldxa [%g1 + %g7]ASI_REAL, %g6 ! load in the context
rdpr %tl, %g7 ! need to use real addresses
+
+#ifdef notyet
+ rdpr %cansave, %g1
+#else
+ mov 0, %g1
+#endif
+
+ brz,pn %g1, 0f
+ nop
+ save
+0:
+
cmp %g7, 1 ! for tl > 1
bne,pn %xcc, 2f
nop
+ mov ASI_LDTD_N, %g3
+ wr %g0, ASI_N, %asi
+ GET_PCPU_SCRATCH
+
cmp %g6, %g0 ! kernel?
be,pn %xcc, 1f
nop
GET_HASH_SCRATCH_USER(%g2)
- wr %g0, ASI_LDTD_N, %asi
- ba,pt %xcc, 4f
+ GET_TSB_SCRATCH_USER(%g4)
+ brz,pn %g1, 4f
+ nop
+ ba,pt %xcc, 5f
nop
1:
GET_HASH_SCRATCH_KERNEL(%g2)
- wr %g0, ASI_LDTD_N, %asi
- ba,pt %xcc, 4f
+ GET_TSB_SCRATCH_KERNEL(%g4)
+ brz,pn %g1, 4f
+ nop
+ ba,pt %xcc, 5f
nop
2:
+ mov ASI_LDTD_REAL, %g3
+ wr %g0, ASI_REAL, %asi
+ GET_PCPU_PHYS_SCRATCH(%g4)
+
cmp %g6, %g0 ! kernel?
be,pn %xcc, 3f
nop
- GET_HASH_PHYS_SCRATCH_USER(%g7, %g2)
- wr %g0, ASI_LDTD_REAL, %asi
- ba,pt %xcc, 4f
+ GET_HASH_PHYS_SCRATCH_USER(%g4, %g2)
+ GET_TSB_SCRATCH_USER(%g4)
+ brz,pn %g1, 4f
+ nop
+ ba,pt %xcc, 5f
+ nop
+3:
+
+ GET_HASH_PHYS_SCRATCH_KERNEL(%g4, %g2)
+ GET_TSB_SCRATCH_KERNEL(%g4)
+ brnz,pt %g1, 5f
nop
-3:
- GET_HASH_PHYS_SCRATCH_KERNEL(%g7, %g2)
- wr %g0, ASI_LDTD_REAL, %asi
-4:
-
- srlx %g5, TTARGET_VA_SHIFT, %g1
- sllx %g6, TTARGET_CTX_SHIFT, %g6
- or %g6, %g1, %g6 ! %g6 == search tag
+4:
+ rdpr %tl, %g1
+ dec %g1
+ sll %g1, RW_SHIFT, %g1
+ add %g1, PC_KWBUF, %g1
+ add PCPU_REG, %g1, %g1
+ SAVE_LOCALS_ASI(%g1)
+ mov 0, %g1 ! cansave is 0
+5:
+
+ ! %g1 == %cansave
+ ! %g2 == hash scratch value
+ ! %g3 == TWDW ASI
+ ! %g4 == TSB RA
+ ! %g5 == fault addr
+ ! %g6 == context
+ srlx %g5, TTARGET_VA_SHIFT, %l0
+ sllx %g6, TTARGET_CTX_SHIFT, %l1
+ or %l0, %l1, %l2 ! %l2 == search tag
- ! %g4 == fault type %g5 == fault addr %g6 == tag
- ! XXX only handle normal miss for now (look at fault type in the future)
-
tsb_miss_compute_hash_addr:
- sethi %hi(PAGE_SIZE), %g7
- sub %g7, 1, %g7 ! %g7==PAGE_MASK
+ sethi %hi(PAGE_SIZE), %l0
+ sub %l0, 1, %l1 ! %l1==PAGE_MASK
- and %g2, %g7, %g4 ! size stored in lower 13 bits
- andn %g2, %g7, %g2 ! actual VA of hash
+ and %g2, %l1, %l3 ! size stored in lower 13 bits
+ andn %g2, %l1, %g2 ! actual VA/RA of hash
! XXX only handle 8k page miss
! calculate hash index
- srlx %g5, PAGE_SHIFT, %g1 ! absolute hash index
- sllx %g4, (PAGE_SHIFT - THE_SHIFT), %g4 ! size of hash in THEs
- sub %g4, 1, %g4 ! THE_MASK
- and %g1, %g4, %g4 ! masked hash index
- sllx %g4, THE_SHIFT, %g4 ! masked hash offset
+ srlx %g5, PAGE_SHIFT, %l4 ! absolute hash index
+ sllx %l3, (PAGE_SHIFT - THE_SHIFT), %l0 ! size of hash in THEs
+ sub %l0, 1, %l5 ! THE_MASK
+ and %l4, %l5, %l5 ! masked hash index
+ sllx %l5, THE_SHIFT, %l5 ! masked hash offset
! fetch hash entries - exit when we find what were looking for
! %g2==entry base
- add %g2, %g4, %g2 ! base + offset == entry base
+ add %g2, %l5, %g2 ! base + offset == entry base
- mov %g5, %g7 ! save fault addr
- ! entry 0
- ! %g1 == abs index %g2 == THE pointer %g3 == flags
- ! %g4 <- tag %g5 <- data
- ! %g6 == search tag %g7 == fault addr
+ mov 1, %l0
+ add %g2, 8, %g2 ! point g2 at data
+ sllx %l0, 56, %l0 ! %l0 == VTD_LOCK
+6:
+ ldxa [%g2]%asi, %l7 ! don't lock bus if bucket is locked
+ andcc %l7, %l0, %g0
+ bnz,pn %xcc, 6b
+ or %l7, %l0, %l6
+ casxa [%g2]%asi, %l7, %l6
+ cmp %l6, %l7
+ bne,pn %icc, 6b
+ nop
+
+ sub %g2, 8, %g2 ! point g2 back at tag
+
+ ! %g1 == cansave
+ ! %g2 == THE
+ ! %g3 == TWDW ASI
+ ! %g4 == TSB RA
+ ! %g5 == fault addr
+ ! %g6 == context
+ ! %g7 == PCPU_REG
+ ! %l0 == VTD_LOCK
+ ! %l1 == PAGE_MASK
+ ! %l2 == search tag
+ ! %l4 == absolute index
+
+ ! %l3 == ASI
+ ! %l5 == saved head of bucket
+ ! %l6 == tag
+ ! %l7 == data
+
+ rd %asi, %l3
+ wr %g0, %g3, %asi
+ mov %g2, %l5 ! save head of bucket
+ rdpr %tt, %g3 ! reload trap type
-tsb_miss_lookup_0:
- ldda [%g2]%asi, %g4
- cmp %g4, %g0 ! entry tag == 0 ?
- be,pn %xcc, 5f
- nop
- cmp %g4, %g6 ! entry tag == VA tag?
- be,pn %xcc, 6f
- nop
- ! entry 1
+tsb_miss_lookup_0:
+ HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f)
tsb_miss_lookup_1:
add %g2, 16, %g2
- ldda [%g2]%asi, %g4
- cmp %g4, %g0 ! entry tag == 0 ?
- be,pn %xcc, 5f
- nop
- cmp %g4, %g6 ! entry tag == search tag?
- be,pn %xcc, 6f
- nop
- ! entry 2
+ HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f)
tsb_miss_lookup_2:
add %g2, 16, %g2
- ldda [%g2]%asi, %g4
- cmp %g4, %g0 ! entry tag == 0 ?
- be,pn %xcc, 5f
- nop
- cmp %g4, %g6 ! entry tag == search tag?
- be,pn %xcc, 6f
- nop
- ! entry 3
+ HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f)
tsb_miss_lookup_3:
add %g2, 16, %g2
- ldda [%g2]%asi, %g4
- cmp %g4, %g0 ! entry tag == 0 ?
- be,pn %xcc, 5f
- nop
- cmp %g4, %g6 ! entry tag == search tag?
- be,pn %xcc, 6f
- nop
+ HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f)
tsb_miss_not_found:
-5: ! not found
+7: ! not found
! we need to jump to tl0_trap to drop us back down to tl0
! and take us to trap(...) to service the fault
+ wr %g0, %l3, %asi
+
+ ldxa [%l5 + 8]%asi, %g2
+ andn %g2, %l0, %g2
+ stxa %g2, [%l5 + 8]%asi ! unlock head of bucket
- srlx %g7, 13, %g7 ! slow painful way of masking off
- sllx %g7, 13, %g7 ! bottom bits without using a reg
- ! XXX FIXME
-
- mov %g3, %g2 ! save fault type
- srlx %g6, TTARGET_CTX_SHIFT, %g4 ! recover context
+ andn %g5, %l1, %g5 ! fault page PA
+
+ RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15)
+
+ mov %g3, %g2 ! trap type
sethi %hi(trap), %g1
- or %g4, %g7, %g3
- mov -1, %g4
+ or %g6, %g5, %g3 ! trap data
+ sub %g0, 1, %g4 ! pil info
ba %xcc, tl0_trap
or %g1, %lo(trap), %g1
-
-tsb_miss_found:
-6: !found
- ! %g1 == abs index %g2 == THE pointer %g3 == flags
- ! %g4 == tag %g5 == data %g7 == PAGE_MASK
- ! %g3 <- TSB RA %g6 <- TSB size, TTE RA %g7 == PAGE_MASK
- srlx %g6, TTARGET_CTX_SHIFT, %g6
- or %g7, %g6, %g6 ! save context and fault addr
- ! will break when ctxbits > 13
- sethi %hi(PAGE_SIZE), %g7
- sub %g7, 1, %g7 ! %g7==PAGE_MASK
-
+tsb_miss_found:
+8:
+ wr %g0, %l3, %asi
cmp %g3, T_DATA_MISS ! TSB data miss
- be,pt %xcc, 7f
- or %g5, VTD_REF, %g5 ! set referenced unconditionally
+ be,pt %xcc, 9f
+ or %l7, VTD_REF, %l7 ! set referenced unconditionally
cmp %g3, T_INSTRUCTION_MISS ! TSB instruction miss
- be,pt %xcc, 7f
+ be,pt %xcc, 9f
nop
cmp %g3, T_DATA_PROTECTION ! protection fault
bne,pn %xcc, unsupported_fault_trap ! we don't handle any other fault types currently
nop
- andcc %g5, VTD_SW_W, %g0 ! write enabled?
- bz,pn %xcc, prot_fault_trap ! write to read only page
+ andcc %l7, VTD_SW_W, %g0 ! write enabled?
+ bz,a,pn %xcc, prot_fault_trap ! write to read only page
nop
- or %g5, VTD_W, %g5 ! add modifed bit
-7:
- rdpr %tl, %g3
- dec %g3
- movrnz %g3, ASI_REAL, %g3
- movrz %g3, ASI_N, %g3
- wr %g0, %g3, %asi
+ or %l7, VTD_W, %l7 ! add modifed bit
+9:
+
+
+ andn %l7, %l0, %l7 ! remove lock bit
+
+ and %g4, %l1, %g3 ! size of TSB in pages
+ andn %g4, %l1, %l3 ! TSB real address
+
+ sllx %g3, (PAGE_SHIFT - TTE_SHIFT), %g3 ! nttes
+ subx %g3, 1, %g3 ! TSB_MASK
+ and %g3, %l4, %g3 ! masked index
+ sllx %g3, TTE_SHIFT, %g3 ! masked byte offset
+ add %g3, %l3, %g3 ! TTE RA
+
+ mov 8, %l4
- stxa %g5, [%g2 + 8]%asi ! update TTE
- andcc %g6, %g7, %g0 ! kernel context?
- bz,pn %xcc, 8f
- nop
- GET_TSB_SCRATCH_USER(%g3) ! %g3 == TSB (RA)
- ba,pt %xcc, 9f
- and %g3, %g7, %g2 ! size of TSB in pages
-8:
- GET_TSB_SCRATCH_KERNEL(%g3) ! %g3 == TSB (RA)
- and %g3, %g7, %g2 ! size of TSB in pages
-9:
- andn %g3, %g7, %g3 ! TSB real address
- sllx %g2, (PAGE_SHIFT - TTE_SHIFT), %g2 ! nttes
- subx %g2, 1, %g2 ! TSB_MASK
- and %g2, %g1, %g2 ! masked index
- sllx %g2, TTE_SHIFT, %g2 ! masked byte offset
- add %g2, %g3, %g2 ! TTE RA
- mov 8, %g7
#ifdef PMAP_DEBUG
- mov %g2, %g1
- ldda [%g2]ASI_LDTD_REAL, %g2
- cmp %g3, %g5
- bne,pt %xcc, 10f
- cmp %g2, %g4
- bne,pt %xcc, 10f
+ ldda [%g3]ASI_LDTD_REAL, %l2
+ cmp %l3, %l7
+ bne,pt %xcc, 12f
+ cmp %l2, %l6
+ bne,pt %xcc, 12f
nop
+#ifndef SIMULATOR
PUTCHAR(0x5a)
PUTCHAR(0x5a)
PUTCHAR(0x5a)
+#endif
MAGIC_TRAP_ON;MAGIC_TRAP_ON;MAGIC_EXIT ! die if all we're doing
! is storing same data
-10:
- mov %g1, %g2
+12:
#endif
- stxa %g0, [%g2 + %g7]ASI_REAL ! invalidate data
- stxa %g4, [%g2]ASI_REAL ! store tag
- stxa %g5, [%g2 + %g7]ASI_REAL ! store data
+ stxa %g0, [%g3 + %l4]ASI_REAL ! invalidate data
+ stxa %l6, [%g3]ASI_REAL ! store tag
+ stxa %l7, [%g3 + %l4]ASI_REAL ! store data
+ stxa %l7, [%g2 + 8]%asi ! update TTE
-upgrade_demap:
- MAGIC_TRAP_ON
- rdpr %tt, %g5
- cmp %g5, T_DATA_PROTECTION
+ cmp %g2, %l5
+ beq,a,pt %xcc, 17f
+ nop
+ ldxa [%l5 + 8]%asi, %l7
+ andn %l7, %l0, %l7 ! remove lock bit
+ stxa %l7, [%l5 + 8]%asi ! unlock bucket
+17:
+ membar #Sync
+ RESTORE_TRAPWIN(PCPU_REG, %g1, 13, 16)
+upgrade_demap:
+ rdpr %tt, %g3
+ cmp %g3, T_DATA_PROTECTION
beq,pn %xcc, demap_begin
nop
- MAGIC_TRAP_OFF
retry
demap_begin:
sethi %hi(PAGE_SIZE), %g1
sub %g1, 1, %g1
- and %g6, %g1, %g2
- andn %g6, %g1, %g3
- mov %o0, %g5
- mov %o1, %g6
- mov %o2, %g7
+ mov %o0, %g1
+ mov %o1, %g2
+ mov %o2, %g3
mov MAP_DTLB, %o2
- mov %g3, %o0
- mov %g2, %o1
+ mov %g5, %o0
+ mov %g6, %o1
ta MMU_UNMAP_ADDR
- mov %g5, %o0
- mov %g6, %o1
- mov %g7, %o2
- MAGIC_TRAP_OFF
+ mov %g1, %o0
+ mov %g2, %o1
+ mov %g3, %o2
retry
END(tsb_miss_handler)
@@ -1556,13 +1628,21 @@
/*
* Write to read-only page
*/
-! %g4==tag
-! %g5==TTE
-! %g6==fault addr | context
+! %g1 == cansave
+! %g4 == tag
+! %g5 == fault addr
+! %g6 == context
+! %l0 == VTD_LOCK
+! %l5 == head of bucket
+
ENTRY(prot_fault_trap)
+ ldxa [%l5 + 8]%asi, %g2
+ andn %g2, %l0, %g2 ! remove lock bit
+ stxa %g2, [%l5 + 8]%asi ! unlock head of bucket
+ RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15)
sethi %hi(trap), %g1
mov T_DATA_PROTECTION, %g2
- mov %g6, %g3
+ or %g5, %g6, %g3
sub %g0, 1, %g4
ba %xcc, tl0_trap
or %g1, %lo(trap), %g1
More information about the p4-projects
mailing list