int index;
/* This driver is only for cheetah platforms. */
- if (tlb_type != cheetah)
+ if (tlb_type != cheetah && tlb_type != cheetah_plus)
return -ENODEV;
index = probe_for_string("memory-controller", 0);
{ 0x17, 0x12, 0, "UltraSparc IIi integrated FPU"},
{ 0x17, 0x13, 0, "UltraSparc IIe integrated FPU"},
{ 0x3e, 0x14, 0, "UltraSparc III integrated FPU"},
+ { 0x3e, 0x15, 0, "UltraSparc III+ integrated FPU"},
};
#define NSPARCFPU (sizeof(linux_sparc_fpu)/sizeof(struct cpu_fp_info))
{ 0x17, 0x12, "TI UltraSparc IIi"},
{ 0x17, 0x13, "TI UltraSparc IIe"},
{ 0x3e, 0x14, "TI UltraSparc III (Cheetah)"},
+ { 0x3e, 0x15, "TI UltraSparc III+ (Cheetah+)"},
};
#define NSPARCCHIPS (sizeof(linux_sparc_chips)/sizeof(struct cpu_iu_info))
if (tlb_type == spitfire) {
prom_getproperty(scan, "upa-portid",
(char *) &thismid, sizeof(thismid));
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah ||
+ tlb_type == cheetah_plus) {
prom_getproperty(scan, "portid",
(char *) &thismid, sizeof(thismid));
}
CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
be,pn %xcc, 3f ! Yep, special processing
CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
- cmp %g5, 3 ! Last trap level?
+ cmp %g5, 4 ! Last trap level?
be,pn %xcc, longpath ! Yep, cannot risk VPTE miss
nop ! delay slot
jmpl %g2 + %lo(cheetah_deferred_trap), %g0
mov 1, %g1
+ /* Cheetah+ specific traps. These are for the new I/D cache parity
+ * error traps. The first argument to cheetah_plus_parity_handler
+ * is encoded as follows:
+ *
+ * Bit0: 0=dcache,1=icache
+ * Bit1: 0=recoverable,1=unrecoverable
+ */
+ .globl cheetah_plus_dcpe_trap_vector, cheetah_plus_dcpe_trap_vector_tl1
+cheetah_plus_dcpe_trap_vector:
+ membar #Sync
+ ba,pt %xcc, etrap
+ rd %pc, %g7
+ mov 0x0, %o0
+ call cheetah_plus_parity_error
+ add %sp, STACK_BIAS + REGWIN_SZ, %o1
+ ba,pt %xcc, rtrap
+ clr %l6
+cheetah_plus_dcpe_trap_vector_tl1:
+ membar #Sync
+ wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
+ sethi %hi(do_dcpe_tl1), %g3
+ jmpl %g3 + %lo(do_dcpe_tl1), %g0
+ nop
+ nop
+ nop
+ nop
+
+ .globl cheetah_plus_icpe_trap_vector, cheetah_plus_icpe_trap_vector_tl1
+cheetah_plus_icpe_trap_vector:
+ membar #Sync
+ ba,pt %xcc, etrap
+ rd %pc, %g7
+ mov 0x1, %o0
+ call cheetah_plus_parity_error
+ add %sp, STACK_BIAS + REGWIN_SZ, %o1
+ ba,pt %xcc, rtrap
+ clr %l6
+cheetah_plus_icpe_trap_vector_tl1:
+ membar #Sync
+ wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
+ sethi %hi(do_icpe_tl1), %g3
+ jmpl %g3 + %lo(do_icpe_tl1), %g0
+ nop
+ nop
+ nop
+ nop
+
+ /* If we take one of these traps when tl >= 1, then we
+ * jump to interrupt globals. If some trap level above us
+ * was also using interrupt globals, we cannot recover.
+ * We may use all interrupt global registers except %g6.
+ */
+ .globl do_dcpe_tl1, do_icpe_tl1
+do_dcpe_tl1:
+ rdpr %tl, %g1 ! Save original trap level
+ mov 1, %g2 ! Setup TSTATE checking loop
+ sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit
+1: wrpr %g2, %tl ! Set trap level to check
+ rdpr %tstate, %g4 ! Read TSTATE for this level
+ andcc %g4, %g3, %g0 ! Interrupt globals in use?
+ bne,a,pn %xcc, do_dcpe_tl1_fatal ! Yep, irrecoverable
+ wrpr %g1, %tl ! Restore original trap level
+ add %g2, 1, %g2 ! Next trap level
+ cmp %g2, %g1 ! Hit them all yet?
+ ble,pt %icc, 1b ! Not yet
+ nop
+ wrpr %g1, %tl ! Restore original trap level
+do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */
+ /* Reset D-cache parity */
+ sethi %hi(1 << 16), %g1 ! D-cache size
+ mov (1 << 5), %g2 ! D-cache line size
+ sub %g1, %g2, %g1 ! Move down 1 cacheline
+1: srl %g1, 14, %g3 ! Compute UTAG
+ membar #Sync
+ stxa %g3, [%g1] ASI_DCACHE_UTAG
+ membar #Sync
+ sub %g2, 8, %g3 ! 64-bit data word within line
+2: membar #Sync
+ stxa %g0, [%g1 + %g3] ASI_DCACHE_DATA
+ membar #Sync
+ subcc %g3, 8, %g3 ! Next 64-bit data word
+ bge,pt %icc, 2b
+ nop
+ subcc %g1, %g2, %g1 ! Next cacheline
+ bge,pt %icc, 1b
+ nop
+ ba,pt %xcc, dcpe_icpe_tl1_common
+ nop
+
+do_dcpe_tl1_fatal:
+ sethi %hi(1f), %g7
+ ba,pt %xcc, etraptl1
+1: or %g7, %lo(1b), %g7
+ mov 0x2, %o0
+ call cheetah_plus_parity_error
+ add %sp, STACK_BIAS + REGWIN_SZ, %o1
+ ba,pt %xcc, rtrap
+ clr %l6
+
+do_icpe_tl1:
+ rdpr %tl, %g1 ! Save original trap level
+ mov 1, %g2 ! Setup TSTATE checking loop
+ sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit
+1: wrpr %g2, %tl ! Set trap level to check
+ rdpr %tstate, %g4 ! Read TSTATE for this level
+ andcc %g4, %g3, %g0 ! Interrupt globals in use?
+ bne,a,pn %xcc, do_icpe_tl1_fatal ! Yep, irrecoverable
+ wrpr %g1, %tl ! Restore original trap level
+ add %g2, 1, %g2 ! Next trap level
+ cmp %g2, %g1 ! Hit them all yet?
+ ble,pt %icc, 1b ! Not yet
+ nop
+ wrpr %g1, %tl ! Restore original trap level
+do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */
+ /* Flush I-cache */
+ sethi %hi(1 << 15), %g1 ! I-cache size
+ mov (1 << 5), %g2 ! I-cache line size
+ sub %g1, %g2, %g1
+1: or %g1, (2 << 3), %g3
+ stxa %g0, [%g3] ASI_IC_TAG
+ membar #Sync
+ subcc %g1, %g2, %g1
+ bge,pt %icc, 1b
+ nop
+ ba,pt %xcc, dcpe_icpe_tl1_common
+ nop
+
+do_icpe_tl1_fatal:
+ sethi %hi(1f), %g7
+ ba,pt %xcc, etraptl1
+1: or %g7, %lo(1b), %g7
+ mov 0x3, %o0
+ call cheetah_plus_parity_error
+ add %sp, STACK_BIAS + REGWIN_SZ, %o1
+ ba,pt %xcc, rtrap
+ clr %l6
+
+dcpe_icpe_tl1_common:
+ /* Flush D-cache, re-enable D/I caches in DCU and finally
+ * retry the trapping instruction.
+ */
+ sethi %hi(1 << 16), %g1 ! D-cache size
+ mov (1 << 5), %g2 ! D-cache line size
+ sub %g1, %g2, %g1
+1: stxa %g0, [%g1] ASI_DCACHE_TAG
+ membar #Sync
+ subcc %g1, %g2, %g1
+ bge,pt %icc, 1b
+ nop
+ ldxa [%g0] ASI_DCU_CONTROL_REG, %g1
+ or %g1, (DCU_DC | DCU_IC), %g1
+ stxa %g1, [%g0] ASI_DCU_CONTROL_REG
+ membar #Sync
+ retry
+
/* Cheetah FECC trap handling, we get here from tl{0,1}_fecc
* in the trap table. That code has done a memory barrier
* and has disabled both the I-cache and D-cache in the DCU
or %g2, %lo(xtime), %g2
or %g1, %lo(timer_tick_compare), %g1
1: rdpr %ver, %o2
- sethi %hi(0x003e0014), %o1
+ sethi %hi(CHEETAH_ID), %o1
srlx %o2, 32, %o2
- or %o1, %lo(0x003e0014), %o1
+ or %o1, %lo(CHEETAH_ID), %o1
membar #Sync
ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %o4
cmp %o2, %o1
+ be,a,pn %xcc, 3f
+ rd %asr24, %o1
+ sethi %hi(CHEETAH_PLUS_ID), %o1
+ or %o1, %lo(CHEETAH_PLUS_ID), %o1
+ cmp %o2, %o1
bne,pt %xcc, 2f
nop
ba,pt %xcc, 3f
*/
sparc64_boot:
rdpr %ver, %g1
- sethi %hi(0x003e0014), %g5
+ sethi %hi(CHEETAH_ID), %g5
srlx %g1, 32, %g1
- or %g5, %lo(0x003e0014), %g5
+ or %g5, %lo(CHEETAH_ID), %g5
+ cmp %g1, %g5
+ be,pn %icc, cheetah_boot
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
cmp %g1, %g5
bne,pt %icc, spitfire_boot
nop
+cheetah_plus_boot:
+ /* Preserve OBP choosen DCU and DCR register settings. */
+ ba,pt %xcc, cheetah_generic_boot
+ nop
+
cheetah_boot:
mov DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1
wr %g1, %asr18
- sethi %uhi(DCU_ME | DCU_RE | /*DCU_PE |*/ DCU_HPE | DCU_SPE | DCU_SL | DCU_WE), %g5
- or %g5, %ulo(DCU_ME | DCU_RE | /*DCU_PE |*/ DCU_HPE | DCU_SPE | DCU_SL | DCU_WE), %g5
+ sethi %uhi(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5
+ or %g5, %ulo(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5
sllx %g5, 32, %g5
or %g5, DCU_DM | DCU_IM | DCU_DC | DCU_IC, %g5
stxa %g5, [%g0] ASI_DCU_CONTROL_REG
membar #Sync
+cheetah_generic_boot:
mov TSB_EXTENSION_P, %g3
stxa %g0, [%g3] ASI_DMMU
stxa %g0, [%g3] ASI_IMMU
add %l0, (1 << 3), %l0
cheetah_got_tlbentry:
+ ldxa [%l0] ASI_ITLB_DATA_ACCESS, %g0
ldxa [%l0] ASI_ITLB_DATA_ACCESS, %g1
membar #Sync
and %g1, %g3, %g1
blu,pt %xcc, 1b
add %l0, (1 << 3), %l0
+ /* On Cheetah+, have to check second DTLB. */
+ rdpr %ver, %g1
+ srlx %g1, 32, %g1
+ sethi %hi(CHEETAH_PLUS_ID), %l0
+ or %l0, %lo(CHEETAH_PLUS_ID), %l0
+ cmp %g1, %l0
+ bne,pt %icc, 9f
+ nop
+
+ set 3 << 16, %l0
+1: ldxa [%l0] ASI_DTLB_TAG_READ, %g1
+ membar #Sync
+ andn %g1, %l2, %g1
+ cmp %g1, %g3
+ blu,pn %xcc, 2f
+ cmp %g1, %g7
+ bgeu,pn %xcc, 2f
+ nop
+ stxa %g0, [%l7] ASI_DMMU
+ membar #Sync
+ stxa %g0, [%l0] ASI_DTLB_DATA_ACCESS
+ membar #Sync
+
+2: and %l0, (511 << 3), %g1
+ cmp %g1, (511 << 3)
+ blu,pt %xcc, 1b
+ add %l0, (1 << 3), %l0
+
+9:
+
/* Now lock the TTE we created into ITLB-0 and DTLB-0,
* entry 15 (and maybe 14 too).
*/
membar #Sync
rdpr %ver, %g1
- sethi %hi(0x003e0014), %g5
+ sethi %hi(CHEETAH_ID), %g5
srlx %g1, 32, %g1
- or %g5, %lo(0x003e0014), %g5
+ or %g5, %lo(CHEETAH_ID), %g5
+ cmp %g1, %g5
+ be,pn %icc, cheetah_tlb_fixup
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
cmp %g1, %g5
bne,pt %icc, spitfire_tlb_fixup
nop
cheetah_tlb_fixup:
set (0 << 16) | (15 << 3), %g7
+ ldxa [%g7] ASI_ITLB_DATA_ACCESS, %g0
ldxa [%g7] ASI_ITLB_DATA_ACCESS, %g1
andn %g1, (_PAGE_G), %g1
stxa %g1, [%g7] ASI_ITLB_DATA_ACCESS
membar #Sync
+ ldxa [%g7] ASI_DTLB_DATA_ACCESS, %g0
ldxa [%g7] ASI_DTLB_DATA_ACCESS, %g1
andn %g1, (_PAGE_G), %g1
stxa %g1, [%g7] ASI_DTLB_DATA_ACCESS
flush %g3
membar #Sync
- /* Set TLB type to cheetah. */
- mov 1, %g2
- sethi %hi(tlb_type), %g5
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
+ rdpr %ver, %g2
+ srlx %g2, 32, %g2
+ cmp %g2, %g5
+ bne,a,pt %icc, 1f
+ mov 1, %g2 /* Set TLB type to cheetah. */
+
+ mov 2, %g2 /* Set TLB type to cheetah+. */
+
+1: sethi %hi(tlb_type), %g5
stw %g2, [%g5 + %lo(tlb_type)]
/* Patch copy/page operations to cheetah optimized versions. */
nop
call cheetah_patch_pgcopyops
nop
+ call cheetah_patch_cachetlbops
+ nop
ba,pt %xcc, tlb_fixup_done
nop
or %g2, KERN_LOWBITS, %g2
rdpr %ver, %g3
- sethi %hi(0x003e0014), %g7
+ sethi %hi(CHEETAH_ID), %g7
srlx %g3, 32, %g3
- or %g7, %lo(0x003e0014), %g7
+ or %g7, %lo(CHEETAH_ID), %g7
+ cmp %g3, %g7
+ be,pn %icc, cheetah_vpte_base
+ sethi %hi(CHEETAH_PLUS_ID), %g7
+ or %g7, %lo(CHEETAH_PLUS_ID), %g7
cmp %g3, %g7
- bne,pt %icc, 1f
+ bne,pt %icc, spitfire_vpte_base
nop
+cheetah_vpte_base:
sethi %uhi(VPTE_BASE_CHEETAH), %g3
or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
ba,pt %xcc, 2f
sllx %g3, 32, %g3
-1:
+spitfire_vpte_base:
sethi %uhi(VPTE_BASE_SPITFIRE), %g3
or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3
sllx %g3, 32, %g3
not_starfire:
rdpr %ver, %g1
- sethi %hi(0x003e0014), %g5
+ sethi %hi(CHEETAH_ID), %g5
srlx %g1, 32, %g1
- or %g7, %lo(0x003e0014), %g5
+ or %g5, %lo(CHEETAH_ID), %g5
+ cmp %g1, %g5
+ be,pn %icc, is_cheetah
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
cmp %g1, %g5
bne,pt %icc, not_cheetah
nop
+is_cheetah:
ldxa [%g0] ASI_SAFARI_CONFIG, %g1
srlx %g1, 17, %g1
ba,pt %xcc, set_worklist
wr %g0, 0, %tick_cmpr
rdpr %ver, %g1
- sethi %hi(0x003e0014), %g5
+ sethi %hi(CHEETAH_ID), %g5
srlx %g1, 32, %g1
- or %g7, %lo(0x003e0014), %g5
+ or %g5, %lo(CHEETAH_ID), %g5
cmp %g1, %g5
- bne,pt %icc, 1f
+ be,pn %icc, 1f
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
+ cmp %g1, %g5
+ bne,pt %icc, 2f
nop
/* Disable STICK_INT interrupts. */
+1:
sethi %hi(0x80000000), %g1
sllx %g1, 32, %g1
wr %g1, %asr25
/* Ok, we're done setting up all the state our trap mechanims needs,
* now get back into normal globals and let the PROM know what is up.
*/
-1:
+2:
wrpr %g0, %g0, %wstate
wrpr %o1, PSTATE_IE, %pstate
if (imap == 0UL)
return;
- if (tlb_type == cheetah) {
+ if (tlb_type == cheetah || tlb_type == cheetah_plus) {
/* We set it to our Safari AID. */
__asm__ __volatile__("ldxa [%%g0] %1, %0"
: "=r" (tid)
goal_cpu = 0;
}
- if (tlb_type == cheetah) {
+ if (tlb_type == cheetah || tlb_type == cheetah_plus) {
tid = goal_cpu << 26;
tid &= IMAP_AID_SAFARI;
} else if (this_is_starfire == 0) {
{
unsigned long va;
- if (tlb_type == cheetah) {
+ if (tlb_type == cheetah || tlb_type == cheetah_plus) {
for (va = 0; va < (1 << 16); va += (1 << 5))
spitfire_put_dcache_tag(va, 0x0);
/* No need to mess with I-cache on Cheetah. */
if (tlb_type == spitfire)
tte = spitfire_get_dtlb_data(SPITFIRE_HIGHEST_LOCKED_TLBENT);
- else if (tlb_type == cheetah)
+ else if (tlb_type == cheetah || tlb_type == cheetah_plus)
tte = cheetah_get_ldtlb_data(CHEETAH_HIGHEST_LOCKED_TLBENT);
res = PROM_TRUE;
extern unsigned long xcall_flush_tlb_mm;
extern unsigned long xcall_flush_tlb_range;
extern unsigned long xcall_flush_tlb_kernel_range;
-extern unsigned long xcall_flush_tlb_all;
-extern unsigned long xcall_tlbcachesync;
-extern unsigned long xcall_flush_cache_all;
+extern unsigned long xcall_flush_tlb_all_spitfire;
+extern unsigned long xcall_flush_tlb_all_cheetah;
+extern unsigned long xcall_flush_cache_all_spitfire;
extern unsigned long xcall_report_regs;
extern unsigned long xcall_receive_signal;
extern unsigned long xcall_flush_dcache_page_cheetah;
void smp_flush_cache_all(void)
{
- smp_cross_call(&xcall_flush_cache_all, 0, 0, 0);
- __flush_cache_all();
+ /* Cheetah need do nothing. */
+ if (tlb_type == spitfire) {
+ smp_cross_call(&xcall_flush_cache_all_spitfire, 0, 0, 0);
+ __flush_cache_all();
+ }
}
void smp_flush_tlb_all(void)
{
- smp_cross_call(&xcall_flush_tlb_all, 0, 0, 0);
+ if (tlb_type == spitfire)
+ smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0);
+ else
+ smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0);
__flush_tlb_all();
}
flushw
rdpr %ver, %g1
- sethi %hi(0x003e0014), %g5
+ sethi %hi(CHEETAH_ID), %g5
srlx %g1, 32, %g1
- or %g5, %lo(0x003e0014), %g5
+ or %g5, %lo(CHEETAH_ID), %g5
+ cmp %g1, %g5
+ be,pn %icc, cheetah_startup
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
cmp %g1, %g5
bne,pt %icc, spitfire_startup
nop
+cheetah_plus_startup:
+ /* Preserve OBP choosen DCU and DCR register settings. */
+ ba,pt %xcc, cheetah_generic_startup
+ nop
+
cheetah_startup:
mov DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1
wr %g1, %asr18
stxa %g5, [%g0] ASI_DCU_CONTROL_REG
membar #Sync
+cheetah_generic_startup:
mov TSB_EXTENSION_P, %g3
stxa %g0, [%g3] ASI_DMMU
stxa %g0, [%g3] ASI_IMMU
stx %g2, [%sp + 2047 + 128 + 0x30]
rdpr %ver, %g1
- sethi %hi(0x003e0014), %g5
+ sethi %hi(CHEETAH_ID), %g5
srlx %g1, 32, %g1
- or %g5, %lo(0x003e0014), %g5
+ or %g5, %lo(CHEETAH_ID), %g5
+ cmp %g1, %g5
+ be,a,pn %icc, 1f
+ mov 15, %g2
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
cmp %g1, %g5
bne,a,pt %icc, 1f
mov 63, %g2
stx %g2, [%sp + 2047 + 128 + 0x30]
rdpr %ver, %g1
- sethi %hi(0x003e0014), %g5
+ sethi %hi(CHEETAH_ID), %g5
srlx %g1, 32, %g1
- or %g5, %lo(0x003e0014), %g5
+ or %g5, %lo(CHEETAH_ID), %g5
+ cmp %g1, %g5
+ be,a,pn %icc, 1f
+ mov 15, %g2
+ sethi %hi(CHEETAH_PLUS_ID), %g5
+ or %g5, %lo(CHEETAH_PLUS_ID), %g5
cmp %g1, %g5
bne,a,pt %icc, 1f
mov 63, %g2
or %g2, KERN_LOWBITS, %g2
rdpr %ver, %g3
- sethi %hi(0x003e0014), %g7
+ sethi %hi(CHEETAH_ID), %g7
srlx %g3, 32, %g3
- or %g7, %lo(0x003e0014), %g7
+ or %g7, %lo(CHEETAH_ID), %g7
+ cmp %g3, %g7
+ be,pn %icc, 9f
+ sethi %hi(CHEETAH_PLUS_ID), %g7
+ or %g7, %lo(CHEETAH_PLUS_ID), %g7
cmp %g3, %g7
bne,pt %icc, 1f
nop
+9:
sethi %uhi(VPTE_BASE_CHEETAH), %g3
or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
ba,pt %xcc, 2f
LSU_CONTROL_IM | LSU_CONTROL_DM),
"i" (ASI_LSU_CONTROL)
: "memory");
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
/* Flush D-cache */
for (va = 0; va < (1 << 16); va += (1 << 5)) {
__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
pci_poke_faulted = 1;
/* Why the fuck did they have to change this? */
- if (tlb_type == cheetah)
+ if (tlb_type == cheetah || tlb_type == cheetah_plus)
regs->tpc += 4;
regs->tnpc = regs->tpc + 4;
return p;
}
+extern unsigned int tl0_icpe[], tl1_icpe[];
+extern unsigned int tl0_dcpe[], tl1_dcpe[];
extern unsigned int tl0_fecc[], tl1_fecc[];
extern unsigned int tl0_cee[], tl1_cee[];
extern unsigned int tl0_iae[], tl1_iae[];
extern unsigned int tl0_dae[], tl1_dae[];
+extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
+extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
+ if (tlb_type == cheetah_plus) {
+ memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
+ memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
+ memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
+ memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
+ }
flushi(PAGE_OFFSET);
}
*
* So we must only flush the I-cache when it is disabled.
*/
+static void __cheetah_flush_icache(void)
+{
+ unsigned long i;
+
+ /* Clear the valid bits in all the tags. */
+ for (i = 0; i < (1 << 15); i += (1 << 5)) {
+ __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "r" (i | (2 << 3)), "i" (ASI_IC_TAG));
+ }
+}
+
static void cheetah_flush_icache(void)
{
- unsigned long dcu_save, i;
+ unsigned long dcu_save;
/* Save current DCU, disable I-cache. */
__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
: "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
: "g1");
- /* Clear the valid bits in all the tags. */
- for (i = 0; i < (1 << 16); i += (1 << 5)) {
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : /* no outputs */
- : "r" (i | (2 << 3)), "i" (ASI_IC_TAG));
- }
+ __cheetah_flush_icache();
/* Restore DCU register */
__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
}
}
+/* In order to make the even parity correct we must do two things.
+ * First, we clear DC_data_parity and set DC_utag to an appropriate value.
+ * Next, we clear out all 32-bytes of data for that line. Data of
+ * all-zero + tag parity value of zero == correct parity.
+ */
+static void cheetah_plus_zap_dcache_parity(void)
+{
+ unsigned long i;
+
+ for (i = 0; i < (1 << 16); i += (1 << 5)) {
+ unsigned long tag = (i >> 14);
+ unsigned long j;
+
+ __asm__ __volatile__("membar #Sync\n\t"
+ "stxa %0, [%1] %2\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "r" (tag), "r" (i),
+ "i" (ASI_DCACHE_UTAG));
+ for (j = i; j < i + (1 << 5); j += (1 << 3))
+ __asm__ __volatile__("membar #Sync\n\t"
+ "stxa %%g0, [%0] %1\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "r" (j), "i" (ASI_DCACHE_DATA));
+ }
+}
+
/* Conversion tables used to frob Cheetah AFSR syndrome values into
* something palatable to the memory controller driver get_unumber
* routine.
panic("Irrecoverable deferred error trap.\n");
}
+/* Handle a D/I cache parity error trap. TYPE is encoded as:
+ *
+ * Bit0: 0=dcache,1=icache
+ * Bit1: 0=recoverable,1=unrecoverable
+ *
+ * The hardware has disabled both the I-cache and D-cache in
+ * the %dcr register.
+ */
+void cheetah_plus_parity_error(int type, struct pt_regs *regs)
+{
+ if (type & 0x1)
+ __cheetah_flush_icache();
+ else
+ cheetah_plus_zap_dcache_parity();
+ cheetah_flush_dcache();
+
+ /* Re-enable I-cache/D-cache */
+ __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
+ "or %%g1, %1, %%g1\n\t"
+ "stxa %%g1, [%%g0] %0\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "i" (ASI_DCU_CONTROL_REG),
+ "i" (DCU_DC | DCU_IC)
+ : "g1");
+
+ if (type & 0x2) {
+ printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
+ smp_processor_id(),
+ (type & 0x1) ? 'I' : 'D',
+ regs->tpc);
+ panic("Irrecoverable Cheetah+ parity error.");
+ }
+
+ printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
+ smp_processor_id(),
+ (type & 0x1) ? 'I' : 'D',
+ regs->tpc);
+}
+
void do_fpe_common(struct pt_regs *regs)
{
if (regs->tstate & TSTATE_PRIV) {
#include <linux/config.h>
.globl sparc64_ttable_tl0, sparc64_ttable_tl1
+ .globl tl0_icpe, tl1_icpe
+ .globl tl0_dcpe, tl1_dcpe
.globl tl0_fecc, tl1_fecc
.globl tl0_cee, tl1_cee
.globl tl0_iae, tl1_iae
tl0_daprot:
#include "dtlb_prot.S"
tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */
-tl0_resv071: BTRAP(0x71) BTRAP(0x72) BTRAP(0x73) BTRAP(0x74) BTRAP(0x75)
+tl0_dcpe: BTRAP(0x71) /* D-cache Parity Error on Cheetah+ */
+tl0_icpe: BTRAP(0x72) /* I-cache Parity Error on Cheetah+ */
+tl0_resv073: BTRAP(0x73) BTRAP(0x74) BTRAP(0x75)
tl0_resv076: BTRAP(0x76) BTRAP(0x77) BTRAP(0x78) BTRAP(0x79) BTRAP(0x7a) BTRAP(0x7b)
tl0_resv07c: BTRAP(0x7c) BTRAP(0x7d) BTRAP(0x7e) BTRAP(0x7f)
tl0_s0n: SPILL_0_NORMAL
#include "dtlb_backend.S"
tl1_daprot:
#include "dtlb_prot.S"
-tl1_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */
-tl1_resc071: BTRAPTL1(0x71) BTRAPTL1(0x72) BTRAPTL1(0x73)
+tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */
+tl1_dcpe: BTRAPTL1(0x71) /* D-cache Parity Error on Cheetah+ */
+tl1_icpe: BTRAPTL1(0x72) /* I-cache Parity Error on Cheetah+ */
+tl1_resv073: BTRAPTL1(0x73)
tl1_resv074: BTRAPTL1(0x74) BTRAPTL1(0x75) BTRAPTL1(0x76) BTRAPTL1(0x77)
tl1_resv078: BTRAPTL1(0x78) BTRAPTL1(0x79) BTRAPTL1(0x7a) BTRAPTL1(0x7b)
tl1_resv07c: BTRAPTL1(0x7c) BTRAPTL1(0x7d) BTRAPTL1(0x7e) BTRAPTL1(0x7f)
ldx [%g6 + TI_FLAGS], %g3
/* Spitfire Errata #32 workaround */
- mov 0x8, %o4
+ mov PRIMARY_CONTEXT, %o4
stxa %g0, [%o4] ASI_DMMU
membar #Sync
ldxa [%o3] ASI_DTLB_TAG_READ, %o4
/* Spitfire Errata #32 workaround */
- mov 0x8, %o5
+ mov PRIMARY_CONTEXT, %o5
stxa %g0, [%o5] ASI_DMMU
membar #Sync
add %o3, (TLBTEMP_ENTSZ), %o3
/* Spitfire Errata #32 workaround */
- mov 0x8, %g5
+ mov PRIMARY_CONTEXT, %g5
stxa %g0, [%g5] ASI_DMMU
membar #Sync
ldxa [%o3] ASI_DTLB_TAG_READ, %g5
/* Spitfire Errata #32 workaround */
- mov 0x8, %g7
+ mov PRIMARY_CONTEXT, %g7
stxa %g0, [%g7] ASI_DMMU
membar #Sync
rdpr %ver, %g3
sllx %g3, 16, %g3
srlx %g3, 32 + 16, %g3
- cmp %g3, 0x14
+ cmp %g3, 0x14 ! CHEETAH_ID
+ be,pn %icc, cheetah_copy_user_page
+ cmp %g3, 0x15 ! CHEETAH_PLUS_ID
bne,pt %icc, spitfire_copy_user_page
nop
wrpr %g3, PSTATE_IE, %pstate
/* Spitfire Errata #32 workaround */
- mov 0x8, %g5
+ mov PRIMARY_CONTEXT, %g5
stxa %g0, [%g5] ASI_DMMU
membar #Sync
ldxa [%o3] ASI_DTLB_TAG_READ, %g5
/* Spitfire Errata #32 workaround */
- mov 0x8, %g7
+ mov PRIMARY_CONTEXT, %g7
stxa %g0, [%g7] ASI_DMMU
membar #Sync
retl
nop
-1: stxa %g5, [%o2] ASI_DMMU
+1:
+ stxa %g5, [%o2] ASI_DMMU
stxa %g7, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
jmpl %o7 + 0x8, %g0
{
if (tlb_type == cheetah)
seq_printf(m, "MMU Type\t: Cheetah\n");
+ else if (tlb_type == cheetah_plus)
+ seq_printf(m, "MMU Type\t: Cheetah+\n");
else if (tlb_type == spitfire)
seq_printf(m, "MMU Type\t: Spitfire\n");
else
break;
case cheetah:
+ case cheetah_plus:
phys_page = cheetah_get_litlb_data(sparc64_highest_locked_tlbent());
break;
};
"i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
"i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
: "memory");
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
/* Lock this into i/d tlb-0 entry 11 */
__asm__ __volatile__(
"stxa %%g0, [%2] %3\n\t"
spitfire_put_dtlb_data(i, 0x0UL);
}
}
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
for (i = 0; i < 512; i++) {
- unsigned long tag = cheetah_get_dtlb_tag(i);
+ unsigned long tag = cheetah_get_dtlb_tag(i, 2);
if ((tag & ~PAGE_MASK) == 0 &&
(tag & PAGE_MASK) >= prom_reserved_base) {
"membar #Sync"
: /* no outputs */
: "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
- cheetah_put_dtlb_data(i, 0x0UL);
+ cheetah_put_dtlb_data(i, 0x0UL, 2);
+ }
+
+ if (tlb_type != cheetah_plus)
+ continue;
+
+ tag = cheetah_get_dtlb_tag(i, 3);
+
+ if ((tag & ~PAGE_MASK) == 0 &&
+ (tag & PAGE_MASK) >= prom_reserved_base) {
+ __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
+ "membar #Sync"
+ : /* no outputs */
+ : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
+ cheetah_put_dtlb_data(i, 0x0UL, 3);
}
}
} else {
if (tlb_type == spitfire)
spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
prom_dtlb[i].tlb_data);
- else if (tlb_type == cheetah)
+ else if (tlb_type == cheetah || tlb_type == cheetah_plus)
cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
prom_dtlb[i].tlb_data);
}
if (tlb_type == spitfire)
spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
prom_itlb[i].tlb_data);
- else if (tlb_type == cheetah)
+ else if (tlb_type == cheetah || tlb_type == cheetah_plus)
cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
prom_itlb[i].tlb_data);
}
break;
}
}
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
int high = CHEETAH_HIGHEST_LOCKED_TLBENT - bigkernel;
for (i = 0; i < high; i++) {
if (tlb_type == spitfire)
spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
prom_dtlb[i].tlb_data);
- else if (tlb_type == cheetah)
+ else if (tlb_type == cheetah || tlb_type == cheetah_plus)
cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
prom_dtlb[i].tlb_data);
}
spitfire_put_itlb_data(i, 0x0UL);
}
}
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
cheetah_flush_dtlb_all();
cheetah_flush_itlb_all();
}
slot+2,
spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
}
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
printk ("Contents of itlb0:\n");
for (slot = 0; slot < 16; slot+=2) {
printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
slot+2,
spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
}
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
printk ("Contents of dtlb0:\n");
for (slot = 0; slot < 16; slot+=2) {
printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
for (slot = 0; slot < 512; slot+=2) {
printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
slot,
- cheetah_get_dtlb_tag(slot), cheetah_get_dtlb_data(slot),
+ cheetah_get_dtlb_tag(slot, 2), cheetah_get_dtlb_data(slot, 2),
slot+1,
- cheetah_get_dtlb_tag(slot+1), cheetah_get_dtlb_data(slot+1));
+ cheetah_get_dtlb_tag(slot+1, 2), cheetah_get_dtlb_data(slot+1, 2));
+ }
+ if (tlb_type == cheetah_plus) {
+ printk ("Contents of dtlb3:\n");
+ for (slot = 0; slot < 512; slot+=2) {
+ printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
+ slot,
+ cheetah_get_dtlb_tag(slot, 3), cheetah_get_dtlb_data(slot, 3),
+ slot+1,
+ cheetah_get_dtlb_tag(slot+1, 3), cheetah_get_dtlb_data(slot+1, 3));
+ }
}
}
}
"i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3)
: "memory");
}
- } else if (tlb_type == cheetah) {
+ } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
__asm__ __volatile__(
" stxa %1, [%0] %3\n"
" stxa %2, [%5] %4\n"
initpages << (PAGE_SHIFT-10),
PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
- if (tlb_type == cheetah)
+ if (tlb_type == cheetah || tlb_type == cheetah_plus)
cheetah_ecache_flush_init();
}
#include <asm/head.h>
#include <asm/thread_info.h>
- /* Basically, all this madness has to do with the
- * fact that Cheetah does not support IMMU flushes
- * out of the secondary context. Someone needs to
- * throw a south lake birthday party for the folks
+ /* Basically, most of the Spitfire vs. Cheetah madness
+ * has to do with the fact that Cheetah does not support
+ * IMMU flushes out of the secondary context. Someone needs
+ * to throw a south lake birthday party for the folks
* in Microelectronics who refused to fix this shit.
*/
-#define BRANCH_IF_CHEETAH(tmp1, tmp2, label) \
- rdpr %ver, %tmp1; \
- sethi %hi(0x003e0014), %tmp2; \
- srlx %tmp1, 32, %tmp1; \
- or %tmp2, %lo(0x003e0014), %tmp2; \
- cmp %tmp1, %tmp2; \
- be,pn %icc, label; \
- nop; \
- nop;
/* This file is meant to be read efficiently by the CPU, not humans.
* Staraj sie tego nikomu nie pierdolnac...
.align 32
.globl __flush_tlb_page, __flush_tlb_mm, __flush_tlb_range
__flush_tlb_page: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=page&PAGE_MASK, %o2=SECONDARY_CONTEXT */
-/*IC1*/ BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_page)
-__spitfire_flush_tlb_page:
-/*IC2*/ ldxa [%o2] ASI_DMMU, %g2
+ ldxa [%o2] ASI_DMMU, %g2
cmp %g2, %o0
bne,pn %icc, __spitfire_flush_tlb_page_slow
or %o1, 0x10, %g3
stxa %g0, [%g3] ASI_IMMU_DEMAP
retl
flush %g6
-__cheetah_flush_tlb_page:
-/*IC3*/ rdpr %pstate, %g5
- andn %g5, PSTATE_IE, %g2
- wrpr %g2, 0x0, %pstate
- wrpr %g0, 1, %tl
- mov PRIMARY_CONTEXT, %o2
- ldxa [%o2] ASI_DMMU, %g2
- stxa %o0, [%o2] ASI_DMMU
- stxa %g0, [%o1] ASI_DMMU_DEMAP
-/*IC4*/ stxa %g0, [%o1] ASI_IMMU_DEMAP
- stxa %g2, [%o2] ASI_DMMU
- flush %g6
- wrpr %g0, 0, %tl
- retl
- wrpr %g5, 0x0, %pstate
nop
nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+
__flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
-/*IC5*/ BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_mm)
-__spitfire_flush_tlb_mm:
-/*IC6*/ ldxa [%o1] ASI_DMMU, %g2
+ ldxa [%o1] ASI_DMMU, %g2
cmp %g2, %o0
bne,pn %icc, __spitfire_flush_tlb_mm_slow
mov 0x50, %g3
stxa %g0, [%g3] ASI_IMMU_DEMAP
retl
flush %g6
-__cheetah_flush_tlb_mm:
-/*IC7*/ rdpr %pstate, %g5
- andn %g5, PSTATE_IE, %g2
- wrpr %g2, 0x0, %pstate
- wrpr %g0, 1, %tl
- mov PRIMARY_CONTEXT, %o2
- mov 0x40, %g3
- ldxa [%o2] ASI_DMMU, %g2
- stxa %o0, [%o2] ASI_DMMU
-/*IC8*/ stxa %g0, [%g3] ASI_DMMU_DEMAP
- stxa %g0, [%g3] ASI_IMMU_DEMAP
- stxa %g2, [%o2] ASI_DMMU
- flush %g6
- wrpr %g0, 0, %tl
- retl
- wrpr %g5, 0x0, %pstate
nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+
__flush_tlb_range: /* %o0=(ctx&TAG_CONTEXT_BITS), %o1=start&PAGE_MASK, %o2=SECONDARY_CONTEXT,
* %o3=end&PAGE_MASK, %o4=PAGE_SIZE, %o5=(end - start)
*/
-/*IC9*/ BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_range)
-__spitfire_flush_tlb_range:
#define TLB_MAGIC 207 /* Students, do you know how I calculated this? -DaveM */
-/*IC10*/cmp %o5, %o4
+ cmp %o5, %o4
bleu,pt %xcc, __flush_tlb_page
srlx %o5, PAGE_SHIFT, %g5
cmp %g5, TLB_MAGIC
ldxa [%o2] ASI_DMMU, %g2
cmp %g2, %o0
__spitfire_flush_tlb_range_page_by_page:
-/*IC11*/bne,pn %icc, __spitfire_flush_tlb_range_pbp_slow
+ bne,pn %icc, __spitfire_flush_tlb_range_pbp_slow
sub %o5, %o4, %o5
1: stxa %g0, [%g5 + %o5] ASI_DMMU_DEMAP
stxa %g0, [%g5 + %o5] ASI_IMMU_DEMAP
retl
flush %g6
__spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
-/*IC12*/rdpr %pstate, %g1
+ rdpr %pstate, %g1
wrpr %g1, PSTATE_IE, %pstate
mov TLB_TAG_ACCESS, %g3
- /* XXX Spitfire dependency... */
mov ((SPITFIRE_HIGHEST_LOCKED_TLBENT-1) << 3), %g2
/* Spitfire Errata #32 workaround. */
and %o4, TAG_CONTEXT_BITS, %o5
cmp %o5, %o0
bne,pt %icc, 2f
-/*IC13*/ andn %o4, TAG_CONTEXT_BITS, %o4
+ andn %o4, TAG_CONTEXT_BITS, %o4
cmp %o4, %o1
blu,pt %xcc, 2f
cmp %o4, %o3
2: ldxa [%g2] ASI_DTLB_TAG_READ, %o4
and %o4, TAG_CONTEXT_BITS, %o5
cmp %o5, %o0
-/*IC14*/andn %o4, TAG_CONTEXT_BITS, %o4
+ andn %o4, TAG_CONTEXT_BITS, %o4
bne,pt %icc, 3f
cmp %o4, %o1
blu,pt %xcc, 3f
blu,pn %xcc, 5f
nop
3: brnz,pt %g2, 1b
-/*IC15*/ sub %g2, (1 << 3), %g2
+ sub %g2, (1 << 3), %g2
retl
wrpr %g1, 0x0, %pstate
4: stxa %g0, [%g3] ASI_IMMU
nop
5: stxa %g0, [%g3] ASI_DMMU
-/*IC16*/stxa %g0, [%g2] ASI_DTLB_DATA_ACCESS
+ stxa %g0, [%g2] ASI_DTLB_DATA_ACCESS
flush %g6
/* Spitfire Errata #32 workaround. */
nop
.align 32
-__cheetah_flush_tlb_range:
- cmp %o5, %o4
- bleu,pt %xcc, __cheetah_flush_tlb_page
- nop
-/*IC17*/rdpr %pstate, %g5
- andn %g5, PSTATE_IE, %g2
- wrpr %g2, 0x0, %pstate
- wrpr %g0, 1, %tl
- mov PRIMARY_CONTEXT, %o2
- sub %o5, %o4, %o5
- ldxa [%o2] ASI_DMMU, %g2
- stxa %o0, [%o2] ASI_DMMU
-
-/*IC18*/
-1: stxa %g0, [%o1 + %o5] ASI_DMMU_DEMAP
- stxa %g0, [%o1 + %o5] ASI_IMMU_DEMAP
- membar #Sync
- brnz,pt %o5, 1b
- sub %o5, %o4, %o5
-
- stxa %g2, [%o2] ASI_DMMU
- flush %g6
- wrpr %g0, 0, %tl
- retl
-/*IC19*/ wrpr %g5, 0x0, %pstate
-
- .align 32
.globl __flush_tlb_kernel_range
__flush_tlb_kernel_range: /* %o0=start, %o1=end */
cmp %o0, %o1
stxa %g0, [%g3] ASI_IMMU_DEMAP
flush %g6
stxa %g2, [%o1] ASI_DMMU
-/*IC18*/flush %g6
+ flush %g6
retl
wrpr %g1, 0, %pstate
stxa %o0, [%o2] ASI_DMMU
stxa %g0, [%g3] ASI_DMMU_DEMAP
stxa %g0, [%g3] ASI_IMMU_DEMAP
-/*IC20*/flush %g6
+ flush %g6
stxa %g2, [%o2] ASI_DMMU
flush %g6
retl
rdpr %pstate, %g1
wrpr %g1, PSTATE_IE, %pstate
stxa %o0, [%o2] ASI_DMMU
-/*IC21*/
+
2: stxa %g0, [%g5 + %o5] ASI_DMMU_DEMAP
stxa %g0, [%g5 + %o5] ASI_IMMU_DEMAP
brnz,pt %o5, 2b
stxa %g2, [%o2] ASI_DMMU
flush %g6
retl
-/*IC22*/ wrpr %g1, 0x0, %pstate
+ wrpr %g1, 0x0, %pstate
/*
* The following code flushes one page_size worth.
flush %g6
ba,a,pt %xcc, 3b
- .align 64
- .globl __flush_dcache_page
-__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
- sethi %uhi(PAGE_OFFSET), %g1
- sllx %g1, 32, %g1
- sub %o0, %g1, %o0
-
- rdpr %ver, %g1
- sethi %hi(0x003e0014), %g2
- srlx %g1, 32, %g1
- or %g2, %lo(0x003e0014), %g2
- cmp %g1, %g2
- bne,pt %icc, flush_dcpage_spitfire
- nop
-
-flush_dcpage_cheetah:
- sethi %hi(PAGE_SIZE), %o4
-1: subcc %o4, (1 << 5), %o4
- stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
- bne,pt %icc, 1b
- nop
- membar #Sync
- /* I-cache flush never needed on Cheetah, see callers. */
- retl
- nop
-
#if (PAGE_SHIFT == 13)
#define DTAG_MASK 0x3
#elif (PAGE_SHIFT == 16)
#define DTAG_MASK 0x3ff
#endif
-flush_dcpage_spitfire:
+ .align 64
+ .globl __flush_dcache_page
+__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
+ sethi %uhi(PAGE_OFFSET), %g1
+ sllx %g1, 32, %g1
+ sub %o0, %g1, %o0
clr %o4
srlx %o0, 11, %o0
sethi %hi(1 << 14), %o2
or %o5, %o0, %o5
ba,a,pt %xcc, __prefill_itlb
+ /* Cheetah specific versions, patched at boot time. */
+__cheetah_flush_tlb_page: /* 14 insns */
+ rdpr %pstate, %g5
+ andn %g5, PSTATE_IE, %g2
+ wrpr %g2, 0x0, %pstate
+ wrpr %g0, 1, %tl
+ mov PRIMARY_CONTEXT, %o2
+ ldxa [%o2] ASI_DMMU, %g2
+ stxa %o0, [%o2] ASI_DMMU
+ stxa %g0, [%o1] ASI_DMMU_DEMAP
+ stxa %g0, [%o1] ASI_IMMU_DEMAP
+ stxa %g2, [%o2] ASI_DMMU
+ flush %g6
+ wrpr %g0, 0, %tl
+ retl
+ wrpr %g5, 0x0, %pstate
+
+__cheetah_flush_tlb_mm: /* 15 insns */
+ rdpr %pstate, %g5
+ andn %g5, PSTATE_IE, %g2
+ wrpr %g2, 0x0, %pstate
+ wrpr %g0, 1, %tl
+ mov PRIMARY_CONTEXT, %o2
+ mov 0x40, %g3
+ ldxa [%o2] ASI_DMMU, %g2
+ stxa %o0, [%o2] ASI_DMMU
+ stxa %g0, [%g3] ASI_DMMU_DEMAP
+ stxa %g0, [%g3] ASI_IMMU_DEMAP
+ stxa %g2, [%o2] ASI_DMMU
+ flush %g6
+ wrpr %g0, 0, %tl
+ retl
+ wrpr %g5, 0x0, %pstate
+
+__cheetah_flush_tlb_range: /* 20 insns */
+ cmp %o5, %o4
+ blu,pt %xcc, 9f
+ rdpr %pstate, %g5
+ andn %g5, PSTATE_IE, %g2
+ wrpr %g2, 0x0, %pstate
+ wrpr %g0, 1, %tl
+ mov PRIMARY_CONTEXT, %o2
+ sub %o5, %o4, %o5
+ ldxa [%o2] ASI_DMMU, %g2
+ stxa %o0, [%o2] ASI_DMMU
+1: stxa %g0, [%o1 + %o5] ASI_DMMU_DEMAP
+ stxa %g0, [%o1 + %o5] ASI_IMMU_DEMAP
+ membar #Sync
+ brnz,pt %o5, 1b
+ sub %o5, %o4, %o5
+ stxa %g2, [%o2] ASI_DMMU
+ flush %g6
+ wrpr %g0, 0, %tl
+9: retl
+ wrpr %g5, 0x0, %pstate
+
+flush_dcpage_cheetah: /* 9 insns */
+ sethi %uhi(PAGE_OFFSET), %g1
+ sllx %g1, 32, %g1
+ sub %o0, %g1, %o0
+ sethi %hi(PAGE_SIZE), %o4
+1: subcc %o4, (1 << 5), %o4
+ stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
+ bne,pt %icc, 1b
+ nop
+ membar #Sync
+ retl /* I-cache flush never needed on Cheetah, see callers. */
+ nop
+
+cheetah_patch_one:
+1: lduw [%o1], %g1
+ stw %g1, [%o0]
+ flush %o0
+ subcc %o2, 1, %o2
+ add %o1, 4, %o1
+ bne,pt %icc, 1b
+ add %o0, 4, %o0
+ retl
+ nop
+
+ .globl cheetah_patch_cachetlbops
+cheetah_patch_cachetlbops:
+ save %sp, -128, %sp
+
+ sethi %hi(__flush_tlb_page), %o0
+ or %o0, %lo(__flush_tlb_page), %o0
+ sethi %hi(__cheetah_flush_tlb_page), %o1
+ or %o1, %lo(__cheetah_flush_tlb_page), %o1
+ call cheetah_patch_one
+ mov 14, %o2
+
+ sethi %hi(__flush_tlb_mm), %o0
+ or %o0, %lo(__flush_tlb_mm), %o0
+ sethi %hi(__cheetah_flush_tlb_mm), %o1
+ or %o1, %lo(__cheetah_flush_tlb_mm), %o1
+ call cheetah_patch_one
+ mov 15, %o2
+
+ sethi %hi(__flush_tlb_range), %o0
+ or %o0, %lo(__flush_tlb_range), %o0
+ sethi %hi(__cheetah_flush_tlb_range), %o1
+ or %o1, %lo(__cheetah_flush_tlb_range), %o1
+ call cheetah_patch_one
+ mov 20, %o2
+
+ sethi %hi(__flush_dcache_page), %o0
+ or %o0, %lo(__flush_dcache_page), %o0
+ sethi %hi(flush_dcpage_cheetah), %o1
+ or %o1, %lo(flush_dcpage_cheetah), %o1
+ call cheetah_patch_one
+ mov 9, %o2
+
+ ret
+ restore
+
#ifdef CONFIG_SMP
/* These are all called by the slaves of a cross call, at
* trap level 1, with interrupts fully disabled.
.text
/* These two are not performance critical... */
- .globl xcall_flush_tlb_all
-xcall_flush_tlb_all:
- BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_tlb_all)
-__spitfire_xcall_flush_tlb_all:
+ .globl xcall_flush_tlb_all_spitfire
+xcall_flush_tlb_all_spitfire:
/* Spitfire Errata #32 workaround. */
sethi %hi(errata32_hwbug), %g4
stx %g0, [%g4 + %lo(errata32_hwbug)]
flush %g6
retry
-__cheetah_xcall_flush_tlb_all:
+ .globl xcall_flush_tlb_all_cheetah
+xcall_flush_tlb_all_cheetah:
mov 0x80, %g2
stxa %g0, [%g2] ASI_DMMU_DEMAP
stxa %g0, [%g2] ASI_IMMU_DEMAP
retry
- .globl xcall_flush_cache_all
-xcall_flush_cache_all:
- BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_cache_all)
-__spitfire_xcall_flush_cache_all:
+ .globl xcall_flush_cache_all_spitfire
+xcall_flush_cache_all_spitfire:
sethi %hi(16383), %g2
or %g2, %lo(16383), %g2
clr %g3
flush %g6
retry
- /* Cheetah's caches are fully coherent in the sense that
- * caches are flushed here. We need to verify this and
- * really just not even send out the xcall at the top level.
- */
-__cheetah_xcall_flush_cache_all:
- retry
-
/* These just get rescheduled to PIL vectors. */
.globl xcall_call_function
xcall_call_function:
#define ASI_WCACHE_DATA 0x39 /* (III) WCache data RAM diag */
#define ASI_WCACHE_TAG 0x3a /* (III) WCache tag RAM diag */
#define ASI_WCACHE_SNOOP_TAG 0x3b /* (III) WCache snoop tag RAM diag */
+#define ASI_SRAM_FAST_INIT 0x40 /* (III+) Fast SRAM init */
#define ASI_DCACHE_INVALIDATE 0x42 /* (III) DCache Invalidate diag */
#define ASI_DCACHE_UTAG 0x43 /* (III) DCache uTag diag */
#define ASI_DCACHE_SNOOP_TAG 0x44 /* (III) DCache snoop tag RAM diag */
#ifndef _SPARC64_DCR_H
#define _SPARC64_DCR_H
-/* UltraSparc-III Dispatch Control Register, ASR 0x12 */
+/* UltraSparc-III/III+ Dispatch Control Register, ASR 0x12 */
+#define DCR_DPE 0x0000000000001000 /* III+: D$ Parity Error Enable */
#define DCR_OBS 0x0000000000000fc0 /* Observability Bus Controls */
#define DCR_BPE 0x0000000000000020 /* Branch Predict Enable */
#define DCR_RPE 0x0000000000000010 /* Return Address Prediction Enable */
#define DCR_SI 0x0000000000000008 /* Single Instruction Disable */
+#define DCR_IPE 0x0000000000000004 /* III+: I$ Parity Error Enable */
#define DCR_IFPOE 0x0000000000000002 /* IRQ FP Operation Enable */
#define DCR_MS 0x0000000000000001 /* Multi-Scalar dispatch */
#define ELF_HWCAP ((HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | \
HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV | \
HWCAP_SPARC_V9) | \
- ((tlb_type == cheetah) ? HWCAP_SPARC_ULTRA3 : 0))
+ ((tlb_type == cheetah || tlb_type == cheetah_plus) ? \
+ HWCAP_SPARC_ULTRA3 : 0))
/* This yields a string that ld.so will use to load implementation
specific libraries for optimization. This is more specific in
#define PTREGS_OFF (STACK_BIAS + REGWIN_SZ)
+#define CHEETAH_ID 0x003e0014
+#define CHEETAH_PLUS_ID 0x003e0015
+
#endif /* !(_SPARC64_HEAD_H) */
extern __inline__ int hard_smp_processor_id(void)
{
- if (tlb_type == cheetah) {
+ if (tlb_type == cheetah || tlb_type == cheetah_plus) {
unsigned long safari_config;
__asm__ __volatile__("ldxa [%%g0] %1, %0"
: "=r" (safari_config)
#define TSB_EXTENSION_P 0x0000000000000048 /* Ultra-III and later */
#define TSB_EXTENSION_S 0x0000000000000050 /* Ultra-III and later, D-TLB only */
#define TSB_EXTENSION_N 0x0000000000000058 /* Ultra-III and later */
+#define TLB_TAG_ACCESS_EXT 0x0000000000000060 /* Ultra-III+ and later */
/* These registers only exist as one entity, and are accessed
* via ASI_DMMU only.
enum ultra_tlb_layout {
spitfire = 0,
- cheetah = 1
+ cheetah = 1,
+ cheetah_plus = 2,
};
extern enum ultra_tlb_layout tlb_type;
-#define SPARC64_USE_STICK (tlb_type == cheetah)
+#define SPARC64_USE_STICK (tlb_type != spitfire)
#define CHEETAH_HIGHEST_LOCKED_TLBENT (16 - 1)
"i" (ASI_ITLB_DATA_ACCESS));
}
-extern __inline__ unsigned long cheetah_get_dtlb_data(int entry)
+extern __inline__ unsigned long cheetah_get_dtlb_data(int entry, int tlb)
{
unsigned long data;
__asm__ __volatile__("ldxa [%1] %2, %%g0\n\t"
"ldxa [%1] %2, %0"
: "=r" (data)
- : "r" ((2 << 16) | (entry << 3)), "i" (ASI_DTLB_DATA_ACCESS));
+ : "r" ((tlb << 16) | (entry << 3)), "i" (ASI_DTLB_DATA_ACCESS));
return data;
}
-extern __inline__ unsigned long cheetah_get_dtlb_tag(int entry)
+extern __inline__ unsigned long cheetah_get_dtlb_tag(int entry, int tlb)
{
unsigned long tag;
__asm__ __volatile__("ldxa [%1] %2, %0"
: "=r" (tag)
- : "r" ((2 << 16) | (entry << 3)), "i" (ASI_DTLB_TAG_READ));
+ : "r" ((tlb << 16) | (entry << 3)), "i" (ASI_DTLB_TAG_READ));
return tag;
}
-extern __inline__ void cheetah_put_dtlb_data(int entry, unsigned long data)
+extern __inline__ void cheetah_put_dtlb_data(int entry, unsigned long data, int tlb)
{
__asm__ __volatile__("stxa %0, [%1] %2\n\t"
"membar #Sync"
: /* No outputs */
: "r" (data),
- "r" ((2 << 16) | (entry << 3)),
+ "r" ((tlb << 16) | (entry << 3)),
"i" (ASI_DTLB_DATA_ACCESS));
}
projects / history.git / commitdiff