- __asm__ __volatile__("wrpr %0, 0, %%pstate"
- : : "r" (pstate));
-}
-
-void inherit_locked_prom_mappings(int save_p)
-{
- int i;
- int dtlb_seen = 0;
- int itlb_seen = 0;
-
- /* Fucking losing PROM has more mappings in the TLB, but
- * it (conveniently) fails to mention any of these in the
- * translations property. The only ones that matter are
- * the locked PROM tlb entries, so we impose the following
- * irrecovable rule on the PROM, it is allowed 8 locked
- * entries in the ITLB and 8 in the DTLB.
- *
- * Supposedly the upper 16GB of the address space is
- * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
- * SOMEWHERE!!!!!!!!!!!!!!!!! Furthermore the entire interface
- * used between the client program and the firmware on sun5
- * systems to coordinate mmu mappings is also COMPLETELY
- * UNDOCUMENTED!!!!!! Thanks S(t)un!
- */
- if (save_p) {
- for (i = 0; i < 16; i++) {
- prom_itlb[i].tlb_ent = -1;
- prom_dtlb[i].tlb_ent = -1;
- }
- }
- if (tlb_type == spitfire) {
- int high = SPITFIRE_HIGHEST_LOCKED_TLBENT - bigkernel;
- for (i = 0; i < high; i++) {
- unsigned long data;
-
- /* Spitfire Errata #32 workaround */
- /* NOTE: Always runs on spitfire, so no cheetah+
- * page size encodings.
- */
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "flush %%g6"
- : /* No outputs */
- : "r" (0),
- "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
-
- data = spitfire_get_dtlb_data(i);
- if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
- unsigned long tag;
-
- /* Spitfire Errata #32 workaround */
- /* NOTE: Always runs on spitfire, so no
- * cheetah+ page size encodings.
- */
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "flush %%g6"
- : /* No outputs */
- : "r" (0),
- "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
-
- tag = spitfire_get_dtlb_tag(i);
- if (save_p) {
- prom_dtlb[dtlb_seen].tlb_ent = i;
- prom_dtlb[dtlb_seen].tlb_tag = tag;
- prom_dtlb[dtlb_seen].tlb_data = data;
- }
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
- spitfire_put_dtlb_data(i, 0x0UL);
-
- dtlb_seen++;
- if (dtlb_seen > 15)
- break;
- }
- }
-
- for (i = 0; i < high; i++) {
- unsigned long data;
-
- /* Spitfire Errata #32 workaround */
- /* NOTE: Always runs on spitfire, so no
- * cheetah+ page size encodings.
- */
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "flush %%g6"
- : /* No outputs */
- : "r" (0),
- "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
-
- data = spitfire_get_itlb_data(i);
- if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
- unsigned long tag;
-
- /* Spitfire Errata #32 workaround */
- /* NOTE: Always runs on spitfire, so no
- * cheetah+ page size encodings.
- */
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "flush %%g6"
- : /* No outputs */
- : "r" (0),
- "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
-
- tag = spitfire_get_itlb_tag(i);
- if (save_p) {
- prom_itlb[itlb_seen].tlb_ent = i;
- prom_itlb[itlb_seen].tlb_tag = tag;
- prom_itlb[itlb_seen].tlb_data = data;
- }
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
- spitfire_put_itlb_data(i, 0x0UL);
-
- itlb_seen++;
- if (itlb_seen > 15)
- break;
- }
- }
- } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
- int high = CHEETAH_HIGHEST_LOCKED_TLBENT - bigkernel;
-
- for (i = 0; i < high; i++) {
- unsigned long data;
-
- data = cheetah_get_ldtlb_data(i);
- if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
- unsigned long tag;
-
- tag = cheetah_get_ldtlb_tag(i);
- if (save_p) {
- prom_dtlb[dtlb_seen].tlb_ent = i;
- prom_dtlb[dtlb_seen].tlb_tag = tag;
- prom_dtlb[dtlb_seen].tlb_data = data;
- }
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
- cheetah_put_ldtlb_data(i, 0x0UL);
-
- dtlb_seen++;
- if (dtlb_seen > 15)
- break;
- }
- }
-
- for (i = 0; i < high; i++) {
- unsigned long data;
-
- data = cheetah_get_litlb_data(i);
- if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
- unsigned long tag;
-
- tag = cheetah_get_litlb_tag(i);
- if (save_p) {
- prom_itlb[itlb_seen].tlb_ent = i;
- prom_itlb[itlb_seen].tlb_tag = tag;
- prom_itlb[itlb_seen].tlb_data = data;
- }
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
- cheetah_put_litlb_data(i, 0x0UL);
-
- itlb_seen++;
- if (itlb_seen > 15)
- break;
- }
- }
- } else {
- /* Implement me :-) */
- BUG();
- }
- if (save_p)
- prom_ditlb_set = 1;
-}
-
-/* Give PROM back his world, done during reboots... */
-void prom_reload_locked(void)
-{
- int i;
-
- for (i = 0; i < 16; i++) {
- if (prom_dtlb[i].tlb_ent != -1) {
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "membar #Sync"
- : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
- "i" (ASI_DMMU));
- if (tlb_type == spitfire)
- spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
- prom_dtlb[i].tlb_data);
- else if (tlb_type == cheetah || tlb_type == cheetah_plus)
- cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
- prom_dtlb[i].tlb_data);
- }
-
- if (prom_itlb[i].tlb_ent != -1) {
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "membar #Sync"
- : : "r" (prom_itlb[i].tlb_tag),
- "r" (TLB_TAG_ACCESS),
- "i" (ASI_IMMU));
- if (tlb_type == spitfire)
- spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
- prom_itlb[i].tlb_data);
- else
- cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
- prom_itlb[i].tlb_data);
- }
- }
-}
-
-#ifdef DCACHE_ALIASING_POSSIBLE
-void __flush_dcache_range(unsigned long start, unsigned long end)
-{
- unsigned long va;
-
- if (tlb_type == spitfire) {
- int n = 0;
-
- for (va = start; va < end; va += 32) {
- spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
- if (++n >= 512)
- break;
- }
- } else {
- start = __pa(start);
- end = __pa(end);
- for (va = start; va < end; va += 32)
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : /* no outputs */
- : "r" (va),
- "i" (ASI_DCACHE_INVALIDATE));
- }
-}
-#endif /* DCACHE_ALIASING_POSSIBLE */
-
-/* If not locked, zap it. */
-void __flush_tlb_all(void)
-{
- unsigned long pstate;
- int i;
-
- __asm__ __volatile__("flushw\n\t"
- "rdpr %%pstate, %0\n\t"
- "wrpr %0, %1, %%pstate"
- : "=r" (pstate)
- : "i" (PSTATE_IE));
- if (tlb_type == spitfire) {
- for (i = 0; i < 64; i++) {
- /* Spitfire Errata #32 workaround */
- /* NOTE: Always runs on spitfire, so no
- * cheetah+ page size encodings.
- */
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "flush %%g6"
- : /* No outputs */
- : "r" (0),
- "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
-
- if (!(spitfire_get_dtlb_data(i) & _PAGE_L)) {
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : /* no outputs */
- : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
- spitfire_put_dtlb_data(i, 0x0UL);
- }
-
- /* Spitfire Errata #32 workaround */
- /* NOTE: Always runs on spitfire, so no
- * cheetah+ page size encodings.
- */
- __asm__ __volatile__("stxa %0, [%1] %2\n\t"
- "flush %%g6"
- : /* No outputs */
- : "r" (0),
- "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
-
- if (!(spitfire_get_itlb_data(i) & _PAGE_L)) {
- __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
- "membar #Sync"
- : /* no outputs */
- : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
- spitfire_put_itlb_data(i, 0x0UL);
- }
- }
- } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
- cheetah_flush_dtlb_all();
- cheetah_flush_itlb_all();
- }
- __asm__ __volatile__("wrpr %0, 0, %%pstate"
- : : "r" (pstate));
-}
-
-/* Caller does TLB context flushing on local CPU if necessary.
- * The caller also ensures that CTX_VALID(mm->context) is false.
- *
- * We must be careful about boundary cases so that we never
- * let the user have CTX 0 (nucleus) or we ever use a CTX
- * version of zero (and thus NO_CONTEXT would not be caught
- * by version mis-match tests in mmu_context.h).
- */
-void get_new_mmu_context(struct mm_struct *mm)
-{
- unsigned long ctx, new_ctx;
- unsigned long orig_pgsz_bits;
-
-
- spin_lock(&ctx_alloc_lock);
- orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
- ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
- new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
- if (new_ctx >= (1 << CTX_NR_BITS)) {
- new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
- if (new_ctx >= ctx) {
- int i;
- new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
- CTX_FIRST_VERSION;
- if (new_ctx == 1)
- new_ctx = CTX_FIRST_VERSION;
-
- /* Don't call memset, for 16 entries that's just
- * plain silly...
- */
- mmu_context_bmap[0] = 3;
- mmu_context_bmap[1] = 0;
- mmu_context_bmap[2] = 0;
- mmu_context_bmap[3] = 0;
- for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
- mmu_context_bmap[i + 0] = 0;
- mmu_context_bmap[i + 1] = 0;
- mmu_context_bmap[i + 2] = 0;
- mmu_context_bmap[i + 3] = 0;
- }
- goto out;
- }
- }
- mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
- new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
-out:
- tlb_context_cache = new_ctx;
- mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
- spin_unlock(&ctx_alloc_lock);
-}
-
-#ifndef CONFIG_SMP
-struct pgtable_cache_struct pgt_quicklists;
-#endif
-
-/* OK, we have to color these pages. The page tables are accessed
- * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
- * code, as well as by PAGE_OFFSET range direct-mapped addresses by
- * other parts of the kernel. By coloring, we make sure that the tlbmiss
- * fast handlers do not get data from old/garbage dcache lines that
- * correspond to an old/stale virtual address (user/kernel) that
- * previously mapped the pagetable page while accessing vpte range
- * addresses. The idea is that if the vpte color and PAGE_OFFSET range
- * color is the same, then when the kernel initializes the pagetable
- * using the later address range, accesses with the first address
- * range will see the newly initialized data rather than the garbage.
- */
-#ifdef DCACHE_ALIASING_POSSIBLE
-#define DC_ALIAS_SHIFT 1
-#else
-#define DC_ALIAS_SHIFT 0
-#endif
-pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
-{
- struct page *page;
- unsigned long color;
-
- {
- pte_t *ptep = pte_alloc_one_fast(mm, address);
-
- if (ptep)
- return ptep;
- }
-
- color = VPTE_COLOR(address);
- page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, DC_ALIAS_SHIFT);
- if (page) {
- unsigned long *to_free;
- unsigned long paddr;
- pte_t *pte;
-
-#ifdef DCACHE_ALIASING_POSSIBLE
- set_page_count(page, 1);
- ClearPageCompound(page);
-
- set_page_count((page + 1), 1);
- ClearPageCompound(page + 1);
-#endif
- paddr = (unsigned long) page_address(page);
- memset((char *)paddr, 0, (PAGE_SIZE << DC_ALIAS_SHIFT));
-
- if (!color) {
- pte = (pte_t *) paddr;
- to_free = (unsigned long *) (paddr + PAGE_SIZE);
- } else {
- pte = (pte_t *) (paddr + PAGE_SIZE);
- to_free = (unsigned long *) paddr;
- }
-
-#ifdef DCACHE_ALIASING_POSSIBLE
- /* Now free the other one up, adjust cache size. */
- preempt_disable();
- *to_free = (unsigned long) pte_quicklist[color ^ 0x1];
- pte_quicklist[color ^ 0x1] = to_free;
- pgtable_cache_size++;
- preempt_enable();
-#endif
-
- return pte;
- }
- return NULL;