2 * PowerPC64 Segment Translation Support.
4 * Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com
5 * Copyright (c) 2001 Dave Engebretsen
7 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version
12 * 2 of the License, or (at your option) any later version.
15 #include <linux/config.h>
16 #include <asm/pgtable.h>
18 #include <asm/mmu_context.h>
21 #include <asm/cputable.h>
23 static int make_ste(unsigned long stab, unsigned long esid, unsigned long vsid);
24 static void make_slbe(unsigned long esid, unsigned long vsid, int large,
27 static inline void slb_add_bolted(void)
29 #ifndef CONFIG_PPC_ISERIES
30 unsigned long esid = GET_ESID(VMALLOCBASE);
31 unsigned long vsid = get_kernel_vsid(VMALLOCBASE);
33 WARN_ON(!irqs_disabled());
36 * Bolt in the first vmalloc segment. Since modules end
37 * up there it gets hit very heavily.
39 get_paca()->stab_next_rr = 1;
40 make_slbe(esid, vsid, 0, 1);
45 * Build an entry for the base kernel segment and put it into
46 * the segment table or SLB. All other segment table or SLB
47 * entries are faulted in.
49 void stab_initialize(unsigned long stab)
51 unsigned long esid, vsid;
52 int seg0_largepages = 0;
54 esid = GET_ESID(KERNELBASE);
55 vsid = get_kernel_vsid(esid << SID_SHIFT);
57 if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE)
60 if (cur_cpu_spec->cpu_features & CPU_FTR_SLB) {
61 /* Invalidate the entire SLB & all the ERATS */
62 #ifdef CONFIG_PPC_ISERIES
63 asm volatile("isync; slbia; isync":::"memory");
65 asm volatile("isync":::"memory");
66 asm volatile("slbmte %0,%0"::"r" (0) : "memory");
67 asm volatile("isync; slbia; isync":::"memory");
68 get_paca()->stab_next_rr = 0;
69 make_slbe(esid, vsid, seg0_largepages, 1);
70 asm volatile("isync":::"memory");
75 asm volatile("isync; slbia; isync":::"memory");
76 make_ste(stab, esid, vsid);
79 asm volatile("sync":::"memory");
83 /* Both the segment table and SLB code uses the following cache */
84 #define NR_STAB_CACHE_ENTRIES 8
85 DEFINE_PER_CPU(long, stab_cache_ptr);
86 DEFINE_PER_CPU(long, stab_cache[NR_STAB_CACHE_ENTRIES]);
93 * Create a segment table entry for the given esid/vsid pair.
95 static int make_ste(unsigned long stab, unsigned long esid, unsigned long vsid)
97 unsigned long entry, group, old_esid, castout_entry, i;
98 unsigned int global_entry;
99 STE *ste, *castout_ste;
100 unsigned long kernel_segment = (REGION_ID(esid << SID_SHIFT) !=
103 /* Search the primary group first. */
104 global_entry = (esid & 0x1f) << 3;
105 ste = (STE *)(stab | ((esid & 0x1f) << 7));
107 /* Find an empty entry, if one exists. */
108 for (group = 0; group < 2; group++) {
109 for (entry = 0; entry < 8; entry++, ste++) {
110 if (!(ste->dw0.dw0.v)) {
113 ste->dw1.dw1.vsid = vsid;
114 ste->dw0.dw0.esid = esid;
118 asm volatile("eieio":::"memory");
120 return (global_entry | entry);
123 /* Now search the secondary group. */
124 global_entry = ((~esid) & 0x1f) << 3;
125 ste = (STE *)(stab | (((~esid) & 0x1f) << 7));
129 * Could not find empty entry, pick one with a round robin selection.
130 * Search all entries in the two groups.
132 castout_entry = get_paca()->stab_next_rr;
133 for (i = 0; i < 16; i++) {
134 if (castout_entry < 8) {
135 global_entry = (esid & 0x1f) << 3;
136 ste = (STE *)(stab | ((esid & 0x1f) << 7));
137 castout_ste = ste + castout_entry;
139 global_entry = ((~esid) & 0x1f) << 3;
140 ste = (STE *)(stab | (((~esid) & 0x1f) << 7));
141 castout_ste = ste + (castout_entry - 8);
144 /* Dont cast out the first kernel segment */
145 if (castout_ste->dw0.dw0.esid != GET_ESID(KERNELBASE))
148 castout_entry = (castout_entry + 1) & 0xf;
151 get_paca()->stab_next_rr = (castout_entry + 1) & 0xf;
153 /* Modify the old entry to the new value. */
155 /* Force previous translations to complete. DRENG */
156 asm volatile("isync" : : : "memory");
158 castout_ste->dw0.dw0.v = 0;
159 asm volatile("sync" : : : "memory"); /* Order update */
161 castout_ste->dw0.dword0 = 0;
162 castout_ste->dw1.dword1 = 0;
163 castout_ste->dw1.dw1.vsid = vsid;
164 old_esid = castout_ste->dw0.dw0.esid;
165 castout_ste->dw0.dw0.esid = esid;
166 castout_ste->dw0.dw0.kp = 1;
168 castout_ste->dw0.dw0.ks = 1;
169 asm volatile("eieio" : : : "memory"); /* Order update */
170 castout_ste->dw0.dw0.v = 1;
171 asm volatile("slbie %0" : : "r" (old_esid << SID_SHIFT));
172 /* Ensure completion of slbie */
173 asm volatile("sync" : : : "memory");
175 return (global_entry | (castout_entry & 0x7));
178 static inline void __ste_allocate(unsigned long esid, unsigned long vsid)
180 unsigned char stab_entry;
181 unsigned long offset;
182 int region_id = REGION_ID(esid << SID_SHIFT);
184 stab_entry = make_ste(get_paca()->stab_addr, esid, vsid);
186 if (region_id != USER_REGION_ID)
189 offset = __get_cpu_var(stab_cache_ptr);
190 if (offset < NR_STAB_CACHE_ENTRIES)
191 __get_cpu_var(stab_cache[offset++]) = stab_entry;
193 offset = NR_STAB_CACHE_ENTRIES+1;
194 __get_cpu_var(stab_cache_ptr) = offset;
198 * Allocate a segment table entry for the given ea.
200 int ste_allocate(unsigned long ea)
202 unsigned long vsid, esid;
203 mm_context_t context;
205 /* Check for invalid effective addresses. */
206 if (!IS_VALID_EA(ea))
209 /* Kernel or user address? */
210 if (REGION_ID(ea) >= KERNEL_REGION_ID) {
211 vsid = get_kernel_vsid(ea);
212 context = KERNEL_CONTEXT(ea);
217 context = current->mm->context;
218 vsid = get_vsid(context.id, ea);
222 __ste_allocate(esid, vsid);
224 asm volatile("sync":::"memory");
230 * preload some userspace segments into the segment table.
232 static void preload_stab(struct task_struct *tsk, struct mm_struct *mm)
234 unsigned long pc = KSTK_EIP(tsk);
235 unsigned long stack = KSTK_ESP(tsk);
236 unsigned long unmapped_base;
237 unsigned long pc_esid = GET_ESID(pc);
238 unsigned long stack_esid = GET_ESID(stack);
239 unsigned long unmapped_base_esid;
242 if (test_tsk_thread_flag(tsk, TIF_32BIT))
243 unmapped_base = TASK_UNMAPPED_BASE_USER32;
245 unmapped_base = TASK_UNMAPPED_BASE_USER64;
247 unmapped_base_esid = GET_ESID(unmapped_base);
249 if (!IS_VALID_EA(pc) || (REGION_ID(pc) >= KERNEL_REGION_ID))
251 vsid = get_vsid(mm->context.id, pc);
252 __ste_allocate(pc_esid, vsid);
254 if (pc_esid == stack_esid)
257 if (!IS_VALID_EA(stack) || (REGION_ID(stack) >= KERNEL_REGION_ID))
259 vsid = get_vsid(mm->context.id, stack);
260 __ste_allocate(stack_esid, vsid);
262 if (pc_esid == unmapped_base_esid || stack_esid == unmapped_base_esid)
265 if (!IS_VALID_EA(unmapped_base) ||
266 (REGION_ID(unmapped_base) >= KERNEL_REGION_ID))
268 vsid = get_vsid(mm->context.id, unmapped_base);
269 __ste_allocate(unmapped_base_esid, vsid);
272 asm volatile("sync" : : : "memory");
275 /* Flush all user entries from the segment table of the current processor. */
276 void flush_stab(struct task_struct *tsk, struct mm_struct *mm)
278 STE *stab = (STE *) get_paca()->stab_addr;
280 unsigned long offset = __get_cpu_var(stab_cache_ptr);
282 /* Force previous translations to complete. DRENG */
283 asm volatile("isync" : : : "memory");
285 if (offset <= NR_STAB_CACHE_ENTRIES) {
288 for (i = 0; i < offset; i++) {
289 ste = stab + __get_cpu_var(stab_cache[i]);
295 /* Invalidate all entries. */
298 /* Never flush the first entry. */
301 entry < (PAGE_SIZE / sizeof(STE));
304 ea = ste->dw0.dw0.esid << SID_SHIFT;
305 if (ea < KERNELBASE) {
311 asm volatile("sync; slbia; sync":::"memory");
313 __get_cpu_var(stab_cache_ptr) = 0;
315 preload_stab(tsk, mm);
323 * Create a segment buffer entry for the given esid/vsid pair.
325 * NOTE: A context syncronising instruction is required before and after
326 * this, in the common case we use exception entry and rfid.
328 static void make_slbe(unsigned long esid, unsigned long vsid, int large,
331 unsigned long entry, castout_entry;
340 struct paca_struct *lpaca = get_paca();
343 * We take the next entry, round robin. Previously we tried
344 * to find a free slot first but that took too long. Unfortunately
345 * we dont have any LRU information to help us choose a slot.
349 * Never cast out the segment for our kernel stack. Since we
350 * dont invalidate the ERAT we could have a valid translation
351 * for the kernel stack during the first part of exception exit
352 * which gets invalidated due to a tlbie from another cpu at a
353 * non recoverable point (after setting srr0/1) - Anton
355 * paca Ksave is always valid (even when on the interrupt stack)
358 castout_entry = lpaca->stab_next_rr;
360 entry = castout_entry;
363 * We bolt in the first kernel segment and the first
366 if (castout_entry >= SLB_NUM_ENTRIES)
368 asm volatile("slbmfee %0,%1" : "=r" (esid_data) : "r" (entry));
369 } while (esid_data.data.v &&
370 esid_data.data.esid == GET_ESID(lpaca->kstack));
372 lpaca->stab_next_rr = castout_entry;
374 /* slbie not needed as the previous mapping is still valid. */
377 * Write the new SLB entry.
380 vsid_data.data.vsid = vsid;
381 vsid_data.data.kp = 1;
383 vsid_data.data.l = 1;
385 vsid_data.data.c = 1;
387 vsid_data.data.ks = 1;
390 esid_data.data.esid = esid;
391 esid_data.data.v = 1;
392 esid_data.data.index = entry;
395 * No need for an isync before or after this slbmte. The exception
396 * we enter with and the rfid we exit with are context synchronizing.
398 asm volatile("slbmte %0,%1" : : "r" (vsid_data), "r" (esid_data));
401 static inline void __slb_allocate(unsigned long esid, unsigned long vsid,
402 mm_context_t context)
405 int region_id = REGION_ID(esid << SID_SHIFT);
406 unsigned long offset;
408 if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) {
409 if (region_id == KERNEL_REGION_ID)
411 else if (region_id == USER_REGION_ID)
412 large = in_hugepage_area(context, esid << SID_SHIFT);
415 make_slbe(esid, vsid, large, region_id != USER_REGION_ID);
417 if (region_id != USER_REGION_ID)
420 offset = __get_cpu_var(stab_cache_ptr);
421 if (offset < NR_STAB_CACHE_ENTRIES)
422 __get_cpu_var(stab_cache[offset++]) = esid;
424 offset = NR_STAB_CACHE_ENTRIES+1;
425 __get_cpu_var(stab_cache_ptr) = offset;
429 * Allocate a segment table entry for the given ea.
431 int slb_allocate(unsigned long ea)
433 unsigned long vsid, esid;
434 mm_context_t context;
436 /* Check for invalid effective addresses. */
437 if (unlikely(!IS_VALID_EA(ea)))
440 /* Kernel or user address? */
441 if (REGION_ID(ea) >= KERNEL_REGION_ID) {
442 context = KERNEL_CONTEXT(ea);
443 vsid = get_kernel_vsid(ea);
445 if (unlikely(!current->mm))
448 context = current->mm->context;
449 vsid = get_vsid(context.id, ea);
453 #ifndef CONFIG_PPC_ISERIES
454 BUG_ON((esid << SID_SHIFT) == VMALLOCBASE);
456 __slb_allocate(esid, vsid, context);
462 * preload some userspace segments into the SLB.
464 static void preload_slb(struct task_struct *tsk, struct mm_struct *mm)
466 unsigned long pc = KSTK_EIP(tsk);
467 unsigned long stack = KSTK_ESP(tsk);
468 unsigned long unmapped_base;
469 unsigned long pc_esid = GET_ESID(pc);
470 unsigned long stack_esid = GET_ESID(stack);
471 unsigned long unmapped_base_esid;
474 if (test_tsk_thread_flag(tsk, TIF_32BIT))
475 unmapped_base = TASK_UNMAPPED_BASE_USER32;
477 unmapped_base = TASK_UNMAPPED_BASE_USER64;
479 unmapped_base_esid = GET_ESID(unmapped_base);
481 if (!IS_VALID_EA(pc) || (REGION_ID(pc) >= KERNEL_REGION_ID))
483 vsid = get_vsid(mm->context.id, pc);
484 __slb_allocate(pc_esid, vsid, mm->context);
486 if (pc_esid == stack_esid)
489 if (!IS_VALID_EA(stack) || (REGION_ID(stack) >= KERNEL_REGION_ID))
491 vsid = get_vsid(mm->context.id, stack);
492 __slb_allocate(stack_esid, vsid, mm->context);
494 if (pc_esid == unmapped_base_esid || stack_esid == unmapped_base_esid)
497 if (!IS_VALID_EA(unmapped_base) ||
498 (REGION_ID(unmapped_base) >= KERNEL_REGION_ID))
500 vsid = get_vsid(mm->context.id, unmapped_base);
501 __slb_allocate(unmapped_base_esid, vsid, mm->context);
504 /* Flush all user entries from the segment table of the current processor. */
505 void flush_slb(struct task_struct *tsk, struct mm_struct *mm)
507 unsigned long offset = __get_cpu_var(stab_cache_ptr);
513 if (offset <= NR_STAB_CACHE_ENTRIES) {
515 asm volatile("isync" : : : "memory");
516 for (i = 0; i < offset; i++) {
518 esid_data.data.esid = __get_cpu_var(stab_cache[i]);
519 BUG_ON(esid_data.data.esid == GET_ESID(VMALLOCBASE));
520 asm volatile("slbie %0" : : "r" (esid_data));
522 asm volatile("isync" : : : "memory");
524 asm volatile("isync; slbia; isync" : : : "memory");
528 /* Workaround POWER5 < DD2.1 issue */
529 if (offset == 1 || offset > NR_STAB_CACHE_ENTRIES) {
531 * flush segment in EEH region, we dont normally access
532 * addresses in this region.
535 esid_data.data.esid = EEH_REGION_ID;
536 asm volatile("slbie %0" : : "r" (esid_data));
539 __get_cpu_var(stab_cache_ptr) = 0;
541 preload_slb(tsk, mm);