*/
.globl DataAccess_common
DataAccess_common:
+BEGIN_FTR_SECTION
mfspr r22,DAR
srdi r22,r22,60
cmpi 0,r22,0xc
/* Segment fault on a bolted segment. Go off and map that segment. */
beq- .do_stab_bolted
+END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
stab_bolted_user_return:
EXCEPTION_PROLOG_COMMON
ld r3,_DSISR(r1)
rlwinm r4,r3,32-23,29,29 /* DSISR_STORE -> _PAGE_RW */
ld r3,_DAR(r1) /* into the hash table */
+BEGIN_FTR_SECTION
beq+ 2f /* If so handle it */
li r4,0x300 /* Trap number */
bl .do_stab_SI
b 1f
+END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
2: li r5,0x300
bl .do_hash_page_DSI /* Try to handle as hpte fault */
EXCEPTION_PROLOG_COMMON
ld r3,_DAR(r1)
li r4,0x380 /* Exception vector */
- bl .ste_allocate
+ bl .slb_allocate
or. r3,r3,r3 /* Check return code */
beq fast_exception_return /* Return if we succeeded */
addi r3,r1,STACK_FRAME_OVERHEAD
InstructionAccess_common:
EXCEPTION_PROLOG_COMMON
+BEGIN_FTR_SECTION
andis. r0,r23,0x0020 /* no ste found? */
beq+ 2f
mr r3,r22 /* SRR0 at interrupt */
li r4,0x400 /* Trap number */
bl .do_stab_SI
b 1f
+END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
2: mr r3,r22
li r5,0x400
EXCEPTION_PROLOG_COMMON
mr r3,r22 /* SRR0 = NIA */
li r4,0x480 /* Exception vector */
- bl .ste_allocate
+ bl .slb_allocate
or. r3,r3,r3 /* Check return code */
beq+ fast_exception_return /* Return if we succeeded */
* r20 - r23, SRR0 and SRR1 are saved in the exception frame.
* We assume we aren't going to take any exceptions during this procedure.
*/
+/* XXX note fix masking in get_kernel_vsid to match */
_GLOBAL(do_slb_bolted)
- stw r23,EX_CCR(r21) /* save CR in exc. frame */
+ stw r23,EX_CCR(r21) /* save CR in exc. frame */
- /* (((ea >> 28) & 0x1fff) << 15) | (ea >> 60) */
- mfspr r21,DAR
- rldicl r20,r21,36,32 /* Permits a full 32b of ESID */
- rldicr r20,r20,15,48
- rldicl r21,r21,4,60
- or r20,r20,r21
-
- li r21,9 /* VSID_RANDOMIZER */
- sldi r21,r21,32
- oris r21,r21,58231
- ori r21,r21,39831
-
- mulld r20,r20,r21
- clrldi r20,r20,28 /* r20 = vsid */
-
- /* Search the SLB for a free entry */
- li r22,1
-1:
- slbmfee r23,r22
- rldicl r23,r23,37,63
- cmpwi r23,0
- beq 4f /* Found an invalid entry */
-
- addi r22,r22,1
- cmpldi r22,64
- blt 1b
+ /*
+ * We take the next entry, round robin. Previously we tried
+ * to find a free slot first but that took too long. Unfortunately
+ * we dont have any LRU information to help us choose a slot.
+ */
- /* No free entry - just take the next entry, round-robin */
- /* XXX we should get the number of SLB entries from the naca */
+ /* r20 = paca */
+ /* use a cpu feature mask if we ever change our slb size */
SLB_NUM_ENTRIES = 64
-2: mfspr r21,SPRG3
- ld r22,PACASTABRR(r21)
- addi r23,r22,1
- cmpdi r23,SLB_NUM_ENTRIES
- blt 3f
- li r23,1
-3: std r23,PACASTABRR(r21)
+1: ld r22,PACASTABRR(r20)
+ addi r21,r22,1
+ cmpdi r21,SLB_NUM_ENTRIES
+ blt+ 2f
+ li r21,1 /* dont touch bolted slot 0 */
+2: std r21,PACASTABRR(r20)
- /* r20 = vsid, r22 = entry */
+ /* r20 = paca, r22 = entry */
/*
* Never cast out the segment for our kernel stack. Since we
* which gets invalidated due to a tlbie from another cpu at a
* non recoverable point (after setting srr0/1) - Anton
*/
- slbmfee r23,r22
- srdi r23,r23,28
+ slbmfee r21,r22
+ srdi r21,r21,27
/*
* This is incorrect (r1 is not the kernel stack) if we entered
* from userspace but there is no critical window from userspace
* so this should be OK. Also if we cast out the userspace stack
* segment while in userspace we will fault it straight back in.
*/
- srdi r21,r1,28
- cmpd r21,r23
- beq- 2b
-
- /* Put together the vsid portion of the entry. */
-4: li r21,0
- rldimi r21,r20,12,0
- ori r20,r21,1024
- ori r20,r20,128 /* set class bit for kernel region */
-#ifndef CONFIG_PPC_ISERIES
- ori r20,r20,256 /* map kernel region with large ptes */
-#endif
+ srdi r23,r1,27
+ ori r23,r23,1
+ cmpd r23,r21
+ beq- 1b
+
+ /* r20 = paca, r22 = entry */
+
+ /* (((ea >> 28) & 0x1fff) << 15) | (ea >> 60) */
+ mfspr r21,DAR
+ rldicl r23,r21,36,51
+ sldi r23,r23,15
+ srdi r21,r21,60
+ or r23,r23,r21
+
+ /* VSID_RANDOMIZER */
+ li r21,9
+ sldi r21,r21,32
+ oris r21,r21,58231
+ ori r21,r21,39831
+
+ /* vsid = (ordinal * VSID_RANDOMIZER) & VSID_MASK */
+ mulld r23,r23,r21
+ clrldi r23,r23,28
+
+ /* r20 = paca, r22 = entry, r23 = vsid */
+
+ /* Put together slb word1 */
+ sldi r23,r23,12
+
+BEGIN_FTR_SECTION
+ /* set kp and c bits */
+ ori r23,r23,0x480
+END_FTR_SECTION_IFCLR(CPU_FTR_16M_PAGE)
+BEGIN_FTR_SECTION
+ /* set kp, l and c bits */
+ ori r23,r23,0x580
+END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE)
+
+ /* r20 = paca, r22 = entry, r23 = slb word1 */
+
+ /* Put together slb word0 */
+ mfspr r21,DAR
+ rldicr r21,r21,0,35 /* get the new esid */
+ oris r21,r21,2048 /* set valid bit */
+ rldimi r21,r22,0,52 /* insert entry */
- /* Put together the esid portion of the entry. */
- mfspr r21,DAR /* Get the new esid */
- rldicl r21,r21,36,28 /* Permits a full 36b of ESID */
- li r23,0
- rldimi r23,r21,28,0 /* Insert esid */
- oris r21,r23,2048 /* valid bit */
- rldimi r21,r22,0,52 /* Insert entry */
+ /* r20 = paca, r21 = slb word0, r23 = slb word1 */
/*
* No need for an isync before or after this slbmte. The exception
* we enter with and the rfid we exit with are context synchronizing .
*/
- slbmte r20,r21
+ slbmte r23,r21
/* All done -- return from exception. */
- mfsprg r20,3 /* Load the PACA pointer */
- ld r21,PACAEXCSP(r20) /* Get the exception frame pointer */
- addi r21,r21,EXC_FRAME_SIZE
+ ld r21,PACAEXCSP(r20) /* Get the exception frame pointer */
+ addi r21,r21,EXC_FRAME_SIZE
lwz r23,EX_CCR(r21) /* get saved CR */
/* note that this is almost identical to maskable_exception_exit */
- mtcr r23 /* restore CR */
+
+ /*
+ * Until everyone updates binutils hardwire the POWER4 optimised
+ * single field mtcrf
+ */
+#if 0
+ .machine push
+ .machine "power4"
+ mtcrf 0x80,r23
+ .machine pop
+#else
+ .long 0x7ef80120
+#endif
mfmsr r22
li r23, MSR_RI
ld r22,EX_SRR0(r21) /* Get SRR0 from exc. frame */
ld r23,EX_SRR1(r21) /* Get SRR1 from exc. frame */
mtspr SRR0,r22
- mtspr SRR1,r23
+ mtspr SRR1,r23
ld r22,EX_R22(r21) /* restore r22 and r23 */
ld r23,EX_R23(r21)
- mfspr r20,SPRG2
+ ld r20,EX_R20(r21)
mfspr r21,SPRG1
rfid
* 2 of the License, or (at your option) any later version.
*/
-/* XXX Note: Changes for bolted region have not been merged - Anton */
-
#include <linux/config.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
}
}
+/* Both the segment table and SLB code uses the following cache */
+#define NR_STAB_CACHE_ENTRIES 8
+DEFINE_PER_CPU(long, stab_cache_ptr);
+DEFINE_PER_CPU(long, stab_cache[NR_STAB_CACHE_ENTRIES]);
+
+/*
+ * Segment table stuff
+ */
+
/*
* Create a segment table entry for the given esid/vsid pair.
*/
/*
* Could not find empty entry, pick one with a round robin selection.
- * Search all entries in the two groups. Note that the first time
- * we get here, we start with entry 1 so the initializer
- * can be common with the SLB castout code.
+ * Search all entries in the two groups.
*/
-
- /* This assumes we never castout when initializing the stab. */
- PMC_SW_PROCESSOR(stab_capacity_castouts);
-
castout_entry = get_paca()->xStab_data.next_round_robin;
for (i = 0; i < 16; i++) {
if (castout_entry < 8) {
/* Modify the old entry to the new value. */
/* Force previous translations to complete. DRENG */
- asm volatile("isync" : : : "memory" );
+ asm volatile("isync" : : : "memory");
castout_ste->dw0.dw0.v = 0;
- asm volatile("sync" : : : "memory" ); /* Order update */
+ asm volatile("sync" : : : "memory"); /* Order update */
castout_ste->dw1.dw1.vsid = vsid;
old_esid = castout_ste->dw0.dw0.esid;
castout_ste->dw0.dw0.esid = esid;
castout_ste->dw0.dw0.kp = 1;
- asm volatile("eieio" : : : "memory" ); /* Order update */
+ asm volatile("eieio" : : : "memory"); /* Order update */
castout_ste->dw0.dw0.v = 1;
asm volatile("slbie %0" : : "r" (old_esid << SID_SHIFT));
/* Ensure completion of slbie */
- asm volatile("sync" : : : "memory" );
+ asm volatile("sync" : : : "memory");
return (global_entry | (castout_entry & 0x7));
}
+static inline void __ste_allocate(unsigned long esid, unsigned long vsid,
+ mm_context_t context)
+{
+ unsigned char stab_entry;
+ unsigned long *offset;
+ int region_id = REGION_ID(esid << SID_SHIFT);
+
+ stab_entry = make_ste(get_paca()->xStab_data.virt, esid, vsid);
+
+ if (region_id != USER_REGION_ID)
+ return;
+
+ offset = &__get_cpu_var(stab_cache_ptr);
+ if (*offset < NR_STAB_CACHE_ENTRIES) {
+ __get_cpu_var(stab_cache[*offset]) = stab_entry;
+ }
+ (*offset)++;
+}
+
+/*
+ * Allocate a segment table entry for the given ea.
+ */
+int ste_allocate(unsigned long ea)
+{
+ unsigned long vsid, esid;
+ mm_context_t context;
+
+ /* Check for invalid effective addresses. */
+ if (!IS_VALID_EA(ea))
+ return 1;
+
+ /* Kernel or user address? */
+ if (REGION_ID(ea) >= KERNEL_REGION_ID) {
+ vsid = get_kernel_vsid(ea);
+ context = REGION_ID(ea);
+ } else {
+ if (!current->mm)
+ return 1;
+
+ context = current->mm->context;
+ vsid = get_vsid(context, ea);
+ }
+
+ esid = GET_ESID(ea);
+ __ste_allocate(esid, vsid, context);
+ /* Order update */
+ asm volatile("sync":::"memory");
+
+ return 0;
+}
+
+/*
+ * preload some userspace segments into the segment table.
+ */
+static void preload_stab(struct task_struct *tsk, struct mm_struct *mm)
+{
+ unsigned long pc = KSTK_EIP(tsk);
+ unsigned long stack = KSTK_ESP(tsk);
+ unsigned long unmapped_base;
+ unsigned long pc_esid = GET_ESID(pc);
+ unsigned long stack_esid = GET_ESID(stack);
+ unsigned long unmapped_base_esid;
+ unsigned long vsid;
+
+ if (test_tsk_thread_flag(tsk, TIF_32BIT))
+ unmapped_base = TASK_UNMAPPED_BASE_USER32;
+ else
+ unmapped_base = TASK_UNMAPPED_BASE_USER64;
+
+ unmapped_base_esid = GET_ESID(unmapped_base);
+
+ if (!IS_VALID_EA(pc) || (REGION_ID(pc) >= KERNEL_REGION_ID))
+ return;
+ vsid = get_vsid(mm->context, pc);
+ __ste_allocate(pc_esid, vsid, mm->context);
+
+ if (pc_esid == stack_esid)
+ return;
+
+ if (!IS_VALID_EA(stack) || (REGION_ID(stack) >= KERNEL_REGION_ID))
+ return;
+ vsid = get_vsid(mm->context, stack);
+ __ste_allocate(stack_esid, vsid, mm->context);
+
+ if (pc_esid == unmapped_base_esid || stack_esid == unmapped_base_esid)
+ return;
+
+ if (!IS_VALID_EA(unmapped_base) ||
+ (REGION_ID(unmapped_base) >= KERNEL_REGION_ID))
+ return;
+ vsid = get_vsid(mm->context, unmapped_base);
+ __ste_allocate(unmapped_base_esid, vsid, mm->context);
+
+ /* Order update */
+ asm volatile("sync" : : : "memory");
+}
+
+/* Flush all user entries from the segment table of the current processor. */
+void flush_stab(struct task_struct *tsk, struct mm_struct *mm)
+{
+ STE *stab = (STE *) get_paca()->xStab_data.virt;
+ STE *ste;
+ unsigned long *offset = &__get_cpu_var(stab_cache_ptr);
+
+ /* Force previous translations to complete. DRENG */
+ asm volatile("isync" : : : "memory");
+
+ if (*offset <= NR_STAB_CACHE_ENTRIES) {
+ int i;
+
+ for (i = 0; i < *offset; i++) {
+ ste = stab + __get_cpu_var(stab_cache[i]);
+ ste->dw0.dw0.v = 0;
+ }
+
+ asm volatile("sync; slbia; sync":::"memory");
+ } else {
+ unsigned long entry;
+
+ /* Invalidate all entries. */
+ ste = stab;
+
+ /* Never flush the first entry. */
+ ste += 1;
+ for (entry = 1;
+ entry < (PAGE_SIZE / sizeof(STE));
+ entry++, ste++) {
+ unsigned long ea;
+ ea = ste->dw0.dw0.esid << SID_SHIFT;
+ if (ea < KERNELBASE) {
+ ste->dw0.dw0.v = 0;
+ }
+ }
+
+ asm volatile("sync; slbia; sync":::"memory");
+ }
+
+ *offset = 0;
+
+ preload_stab(tsk, mm);
+}
+
+/*
+ * SLB stuff
+ */
+
/*
* Create a segment buffer entry for the given esid/vsid pair.
*
} vsid_data;
/*
- * Find an empty entry, if one exists. Must start at 0 because
- * we use this code to load SLB entry 0 at boot.
- */
- for (entry = 0; entry < naca->slb_size; entry++) {
- asm volatile("slbmfee %0,%1"
- : "=r" (esid_data) : "r" (entry));
- if (!esid_data.data.v)
- goto write_entry;
- }
-
- /*
- * Could not find empty entry, pick one with a round robin selection.
+ * We take the next entry, round robin. Previously we tried
+ * to find a free slot first but that took too long. Unfortunately
+ * we dont have any LRU information to help us choose a slot.
*/
- PMC_SW_PROCESSOR(stab_capacity_castouts);
-
/*
* Never cast out the segment for our kernel stack. Since we
* dont invalidate the ERAT we could have a valid translation
if (castout_entry >= naca->slb_size)
castout_entry = 1;
asm volatile("slbmfee %0,%1" : "=r" (esid_data) : "r" (entry));
- } while (esid_data.data.esid == GET_ESID((unsigned long)_get_SP()));
+ } while (esid_data.data.v &&
+ esid_data.data.esid == GET_ESID((unsigned long)_get_SP()));
get_paca()->xStab_data.next_round_robin = castout_entry;
/* slbie not needed as the previous mapping is still valid. */
-write_entry:
/*
* Write the new SLB entry.
*/
asm volatile("slbmte %0,%1" : : "r" (vsid_data), "r" (esid_data));
}
-static inline void __ste_allocate(unsigned long esid, unsigned long vsid,
- int kernel_segment, mm_context_t context)
+static inline void __slb_allocate(unsigned long esid, unsigned long vsid,
+ mm_context_t context)
{
- if (cur_cpu_spec->cpu_features & CPU_FTR_SLB) {
- int large = 0;
+ int large = 0;
+ int region_id = REGION_ID(esid << SID_SHIFT);
+ unsigned long *offset;
-#ifndef CONFIG_PPC_ISERIES
- if (REGION_ID(esid << SID_SHIFT) == KERNEL_REGION_ID)
+ if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) {
+ if (region_id == KERNEL_REGION_ID)
large = 1;
- else if (REGION_ID(esid << SID_SHIFT) == USER_REGION_ID)
+ else if (region_id == USER_REGION_ID)
large = in_hugepage_area(context, esid << SID_SHIFT);
-#endif
- make_slbe(esid, vsid, large, kernel_segment);
- } else {
- unsigned char top_entry, stab_entry, *segments;
-
- stab_entry = make_ste(get_paca()->xStab_data.virt, esid, vsid);
- PMC_SW_PROCESSOR_A(stab_entry_use, stab_entry & 0xf);
-
- segments = get_paca()->xSegments;
- top_entry = get_paca()->stab_cache_pointer;
- if (!kernel_segment && top_entry < STAB_CACHE_SIZE) {
- segments[top_entry] = stab_entry;
- if (top_entry == STAB_CACHE_SIZE)
- top_entry = 0xff;
- top_entry++;
- get_paca()->stab_cache_pointer = top_entry;
- }
}
+
+ make_slbe(esid, vsid, large, region_id != USER_REGION_ID);
+
+ if (region_id != USER_REGION_ID)
+ return;
+
+ offset = &__get_cpu_var(stab_cache_ptr);
+ if (*offset < NR_STAB_CACHE_ENTRIES) {
+ __get_cpu_var(stab_cache[*offset]) = esid;
+ }
+ (*offset)++;
}
/*
* Allocate a segment table entry for the given ea.
*/
-int ste_allocate(unsigned long ea)
+int slb_allocate(unsigned long ea)
{
unsigned long vsid, esid;
- int kernel_segment = 0;
mm_context_t context;
- PMC_SW_PROCESSOR(stab_faults);
-
/* Check for invalid effective addresses. */
- if (!IS_VALID_EA(ea))
+ if (unlikely(!IS_VALID_EA(ea)))
return 1;
/* Kernel or user address? */
if (REGION_ID(ea) >= KERNEL_REGION_ID) {
- kernel_segment = 1;
- vsid = get_kernel_vsid(ea);
context = REGION_ID(ea);
+ vsid = get_kernel_vsid(ea);
} else {
- if (! current->mm)
+ if (unlikely(!current->mm))
return 1;
context = current->mm->context;
-
vsid = get_vsid(context, ea);
}
esid = GET_ESID(ea);
- __ste_allocate(esid, vsid, kernel_segment, context);
- if (!(cur_cpu_spec->cpu_features & CPU_FTR_SLB)) {
- /* Order update */
- asm volatile("sync":::"memory");
- }
+ __slb_allocate(esid, vsid, context);
return 0;
}
-unsigned long ppc64_preload_all_segments;
-unsigned long ppc64_stab_preload = 1;
-#define STAB_PRESSURE 0
-#define USE_SLBIE_ON_STAB 0
-
/*
- * preload all 16 segments for a 32 bit process and the PC and SP segments
- * for a 64 bit process.
+ * preload some userspace segments into the SLB.
*/
-static void preload_stab(struct task_struct *tsk, struct mm_struct *mm)
+static void preload_slb(struct task_struct *tsk, struct mm_struct *mm)
{
- if (ppc64_preload_all_segments &&
- test_tsk_thread_flag(tsk, TIF_32BIT)) {
- unsigned long esid, vsid;
-
- for (esid = 0; esid < 16; esid++) {
- unsigned long ea = esid << SID_SHIFT;
- vsid = get_vsid(mm->context, ea);
- __ste_allocate(esid, vsid, 0, mm->context);
- }
- } else {
- unsigned long pc = KSTK_EIP(tsk);
- unsigned long stack = KSTK_ESP(tsk);
- unsigned long pc_segment = pc & ~SID_MASK;
- unsigned long stack_segment = stack & ~SID_MASK;
- unsigned long vsid;
-
- if (pc) {
- if (!IS_VALID_EA(pc) ||
- (REGION_ID(pc) >= KERNEL_REGION_ID))
- return;
- vsid = get_vsid(mm->context, pc);
- __ste_allocate(GET_ESID(pc), vsid, 0, mm->context);
- }
-
- if (stack && (pc_segment != stack_segment)) {
- if (!IS_VALID_EA(stack) ||
- (REGION_ID(stack) >= KERNEL_REGION_ID))
- return;
- vsid = get_vsid(mm->context, stack);
- __ste_allocate(GET_ESID(stack), vsid, 0, mm->context);
- }
- }
-
- if (!(cur_cpu_spec->cpu_features & CPU_FTR_SLB)) {
- /* Order update */
- asm volatile("sync" : : : "memory");
- }
+ unsigned long pc = KSTK_EIP(tsk);
+ unsigned long stack = KSTK_ESP(tsk);
+ unsigned long unmapped_base;
+ unsigned long pc_esid = GET_ESID(pc);
+ unsigned long stack_esid = GET_ESID(stack);
+ unsigned long unmapped_base_esid;
+ unsigned long vsid;
+
+ if (test_tsk_thread_flag(tsk, TIF_32BIT))
+ unmapped_base = TASK_UNMAPPED_BASE_USER32;
+ else
+ unmapped_base = TASK_UNMAPPED_BASE_USER64;
+
+ unmapped_base_esid = GET_ESID(unmapped_base);
+
+ if (!IS_VALID_EA(pc) || (REGION_ID(pc) >= KERNEL_REGION_ID))
+ return;
+ vsid = get_vsid(mm->context, pc);
+ __slb_allocate(pc_esid, vsid, mm->context);
+
+ if (pc_esid == stack_esid)
+ return;
+
+ if (!IS_VALID_EA(stack) || (REGION_ID(stack) >= KERNEL_REGION_ID))
+ return;
+ vsid = get_vsid(mm->context, stack);
+ __slb_allocate(stack_esid, vsid, mm->context);
+
+ if (pc_esid == unmapped_base_esid || stack_esid == unmapped_base_esid)
+ return;
+
+ if (!IS_VALID_EA(unmapped_base) ||
+ (REGION_ID(unmapped_base) >= KERNEL_REGION_ID))
+ return;
+ vsid = get_vsid(mm->context, unmapped_base);
+ __slb_allocate(unmapped_base_esid, vsid, mm->context);
}
/* Flush all user entries from the segment table of the current processor. */
-void flush_stab(struct task_struct *tsk, struct mm_struct *mm)
+void flush_slb(struct task_struct *tsk, struct mm_struct *mm)
{
- if (cur_cpu_spec->cpu_features & CPU_FTR_SLB) {
- /*
- * XXX disable 32bit slb invalidate optimisation until we fix
- * the issue where a 32bit app execed out of a 64bit app can
- * cause segments above 4GB not to be flushed - Anton
- */
- if (0 && !STAB_PRESSURE && test_thread_flag(TIF_32BIT)) {
- union {
- unsigned long word0;
- slb_dword0 data;
- } esid_data;
- unsigned long esid;
-
- asm volatile("isync" : : : "memory");
- for (esid = 0; esid < 16; esid++) {
- esid_data.word0 = 0;
- esid_data.data.esid = esid;
- asm volatile("slbie %0" : : "r" (esid_data));
- }
- asm volatile("isync" : : : "memory");
- } else {
- asm volatile("isync; slbia; isync":::"memory");
- }
+ unsigned long *offset = &__get_cpu_var(stab_cache_ptr);
- PMC_SW_PROCESSOR(stab_invalidations);
- } else {
- STE *stab = (STE *) get_paca()->xStab_data.virt;
- STE *ste;
- unsigned long flags;
+ if (*offset <= NR_STAB_CACHE_ENTRIES) {
+ int i;
+ union {
+ unsigned long word0;
+ slb_dword0 data;
+ } esid_data;
- /* Force previous translations to complete. DRENG */
asm volatile("isync" : : : "memory");
-
- local_irq_save(flags);
- if (get_paca()->stab_cache_pointer != 0xff && !STAB_PRESSURE) {
- int i;
- unsigned char *segments = get_paca()->xSegments;
-
- for (i = 0; i < get_paca()->stab_cache_pointer; i++) {
- ste = stab + segments[i];
- ste->dw0.dw0.v = 0;
- PMC_SW_PROCESSOR(stab_invalidations);
- }
-
-#if USE_SLBIE_ON_STAB
- asm volatile("sync":::"memory");
- for (i = 0; i < get_paca()->stab_cache_pointer; i++) {
- ste = stab + segments[i];
- asm volatile("slbie %0" : :
- "r" (ste->dw0.dw0.esid << SID_SHIFT));
- }
- asm volatile("sync":::"memory");
-#else
- asm volatile("sync; slbia; sync":::"memory");
-#endif
-
- } else {
- unsigned long entry;
-
- /* Invalidate all entries. */
- ste = stab;
-
- /* Never flush the first entry. */
- ste += 1;
- for (entry = 1;
- entry < (PAGE_SIZE / sizeof(STE));
- entry++, ste++) {
- unsigned long ea;
- ea = ste->dw0.dw0.esid << SID_SHIFT;
- if (STAB_PRESSURE || ea < KERNELBASE) {
- ste->dw0.dw0.v = 0;
- PMC_SW_PROCESSOR(stab_invalidations);
- }
- }
-
- asm volatile("sync; slbia; sync":::"memory");
+ for (i = 0; i < *offset; i++) {
+ esid_data.word0 = 0;
+ esid_data.data.esid = __get_cpu_var(stab_cache[i]);
+ asm volatile("slbie %0" : : "r" (esid_data));
}
-
- get_paca()->stab_cache_pointer = 0;
- local_irq_restore(flags);
+ asm volatile("isync" : : : "memory");
+ } else {
+ asm volatile("isync; slbia; isync" : : : "memory");
}
- if (ppc64_stab_preload)
- preload_stab(tsk, mm);
+ *offset = 0;
+
+ preload_slb(tsk, mm);
}
projects / history.git / commitdiff