X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fsparc64%2Fmm%2Fultra.S;h=d70b60a3bbccb5340cf517dcf64657eba1927456;hb=16c70f8c1b54b61c3b951b6fb220df250fe09b32;hp=16410ba9555cf58f784d339bbac184d47b8d2e24;hpb=5273a3df6485dc2ad6aa7ddd441b9a21970f003b;p=linux-2.6.git diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S index 16410ba95..d70b60a3b 100644 --- a/arch/sparc64/mm/ultra.S +++ b/arch/sparc64/mm/ultra.S @@ -4,15 +4,17 @@ * Copyright (C) 1997, 2000 David S. Miller (davem@redhat.com) */ -#include #include #include #include #include #include +#include #include #include #include +#include +#include /* Basically, most of the Spitfire vs. Cheetah madness * has to do with the fact that Cheetah does not support @@ -26,34 +28,20 @@ */ .text .align 32 - .globl __flush_tlb_page, __flush_tlb_mm, __flush_tlb_range -__flush_tlb_page: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=page&PAGE_MASK, %o2=SECONDARY_CONTEXT */ - ldxa [%o2] ASI_DMMU, %g2 - cmp %g2, %o0 - bne,pn %icc, __spitfire_flush_tlb_page_slow - or %o1, 0x10, %g3 - stxa %g0, [%g3] ASI_DMMU_DEMAP - stxa %g0, [%g3] ASI_IMMU_DEMAP - retl - flush %g6 - nop - nop - nop - nop - nop - nop - nop - nop - -__flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ + .globl __flush_tlb_mm +__flush_tlb_mm: /* 18 insns */ + /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ ldxa [%o1] ASI_DMMU, %g2 cmp %g2, %o0 bne,pn %icc, __spitfire_flush_tlb_mm_slow mov 0x50, %g3 stxa %g0, [%g3] ASI_DMMU_DEMAP stxa %g0, [%g3] ASI_IMMU_DEMAP + sethi %hi(KERNBASE), %g3 + flush %g3 retl - flush %g6 + nop + nop nop nop nop @@ -63,88 +51,42 @@ __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ nop nop -__flush_tlb_range: /* %o0=(ctx&TAG_CONTEXT_BITS), %o1=start&PAGE_MASK, %o2=SECONDARY_CONTEXT, - * %o3=end&PAGE_MASK, %o4=PAGE_SIZE, %o5=(end - start) - */ -#define TLB_MAGIC 207 /* Students, do you know how I calculated this? -DaveM */ - cmp %o5, %o4 - bleu,pt %xcc, __flush_tlb_page - srlx %o5, PAGE_SHIFT, %g5 - cmp %g5, TLB_MAGIC - bgeu,pn %icc, __spitfire_flush_tlb_range_constant_time - or %o1, 0x10, %g5 - ldxa [%o2] ASI_DMMU, %g2 - cmp %g2, %o0 -__spitfire_flush_tlb_range_page_by_page: - bne,pn %icc, __spitfire_flush_tlb_range_pbp_slow - sub %o5, %o4, %o5 -1: stxa %g0, [%g5 + %o5] ASI_DMMU_DEMAP - stxa %g0, [%g5 + %o5] ASI_IMMU_DEMAP - brnz,pt %o5, 1b - sub %o5, %o4, %o5 - retl - flush %g6 -__spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */ - rdpr %pstate, %g1 - wrpr %g1, PSTATE_IE, %pstate - mov TLB_TAG_ACCESS, %g3 - mov ((SPITFIRE_HIGHEST_LOCKED_TLBENT-1) << 3), %g2 - - /* Spitfire Errata #32 workaround. */ - mov 0x8, %o4 - stxa %g0, [%o4] ASI_DMMU - flush %g6 - -1: ldxa [%g2] ASI_ITLB_TAG_READ, %o4 - and %o4, TAG_CONTEXT_BITS, %o5 - cmp %o5, %o0 - bne,pt %icc, 2f - andn %o4, TAG_CONTEXT_BITS, %o4 - cmp %o4, %o1 - blu,pt %xcc, 2f - cmp %o4, %o3 - blu,pn %xcc, 4f -2: ldxa [%g2] ASI_DTLB_TAG_READ, %o4 - and %o4, TAG_CONTEXT_BITS, %o5 - cmp %o5, %o0 - andn %o4, TAG_CONTEXT_BITS, %o4 - bne,pt %icc, 3f - cmp %o4, %o1 - blu,pt %xcc, 3f - cmp %o4, %o3 - blu,pn %xcc, 5f + .align 32 + .globl __flush_tlb_pending +__flush_tlb_pending: /* 26 insns */ + /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ + rdpr %pstate, %g7 + sllx %o1, 3, %o1 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, %pstate + mov SECONDARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + stxa %o0, [%o4] ASI_DMMU +1: sub %o1, (1 << 3), %o1 + ldx [%o2 + %o1], %o3 + andcc %o3, 1, %g0 + andn %o3, 1, %o3 + be,pn %icc, 2f + or %o3, 0x10, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +2: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + brnz,pt %o1, 1b nop -3: brnz,pt %g2, 1b - sub %g2, (1 << 3), %g2 + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 retl - wrpr %g1, 0x0, %pstate -4: stxa %g0, [%g3] ASI_IMMU - stxa %g0, [%g2] ASI_ITLB_DATA_ACCESS - flush %g6 - - /* Spitfire Errata #32 workaround. */ - mov 0x8, %o4 - stxa %g0, [%o4] ASI_DMMU - flush %g6 - - ba,pt %xcc, 2b - nop - -5: stxa %g0, [%g3] ASI_DMMU - stxa %g0, [%g2] ASI_DTLB_DATA_ACCESS - flush %g6 - - /* Spitfire Errata #32 workaround. */ - mov 0x8, %o4 - stxa %g0, [%o4] ASI_DMMU - flush %g6 - - ba,pt %xcc, 3b - nop + wrpr %g7, 0x0, %pstate + nop + nop + nop + nop .align 32 .globl __flush_tlb_kernel_range -__flush_tlb_kernel_range: /* %o0=start, %o1=end */ +__flush_tlb_kernel_range: /* 16 insns */ + /* %o0=start, %o1=end */ cmp %o0, %o1 be,pn %xcc, 2f sethi %hi(PAGE_SIZE), %o4 @@ -156,8 +98,11 @@ __flush_tlb_kernel_range: /* %o0=start, %o1=end */ membar #Sync brnz,pt %o3, 1b sub %o3, %o4, %o3 -2: retl - flush %g6 +2: sethi %hi(KERNBASE), %o3 + flush %o3 + retl + nop + nop __spitfire_flush_tlb_mm_slow: rdpr %pstate, %g1 @@ -167,37 +112,11 @@ __spitfire_flush_tlb_mm_slow: stxa %g0, [%g3] ASI_IMMU_DEMAP flush %g6 stxa %g2, [%o1] ASI_DMMU - flush %g6 + sethi %hi(KERNBASE), %o1 + flush %o1 retl wrpr %g1, 0, %pstate -__spitfire_flush_tlb_page_slow: - rdpr %pstate, %g1 - wrpr %g1, PSTATE_IE, %pstate - stxa %o0, [%o2] ASI_DMMU - stxa %g0, [%g3] ASI_DMMU_DEMAP - stxa %g0, [%g3] ASI_IMMU_DEMAP - flush %g6 - stxa %g2, [%o2] ASI_DMMU - flush %g6 - retl - wrpr %g1, 0, %pstate - -__spitfire_flush_tlb_range_pbp_slow: - rdpr %pstate, %g1 - wrpr %g1, PSTATE_IE, %pstate - stxa %o0, [%o2] ASI_DMMU - -2: stxa %g0, [%g5 + %o5] ASI_DMMU_DEMAP - stxa %g0, [%g5 + %o5] ASI_IMMU_DEMAP - brnz,pt %o5, 2b - sub %o5, %o4, %o5 - flush %g6 - stxa %g2, [%o2] ASI_DMMU - flush %g6 - retl - wrpr %g1, 0x0, %pstate - /* * The following code flushes one page_size worth. */ @@ -208,103 +127,54 @@ __spitfire_flush_tlb_range_pbp_slow: #else #error unsupported PAGE_SIZE #endif + .section .kprobes.text, "ax" .align 32 .globl __flush_icache_page __flush_icache_page: /* %o0 = phys_page */ - sethi %hi(1 << 13), %o2 ! IC_set bit - mov 1, %g1 - srlx %o0, 5, %o0 - clr %o1 ! IC_addr - sllx %g1, 36, %g1 - ldda [%o1] ASI_IC_TAG, %o4 - sub %g1, 1, %g2 - or %o0, %g1, %o0 ! VALID+phys-addr comparitor - - sllx %g2, 1, %g2 - andn %g2, ITAG_MASK, %g2 ! IC_tag mask - nop - nop - nop - nop - nop - nop - -1: addx %g0, %g0, %g0 - ldda [%o1 + %o2] ASI_IC_TAG, %g4 - addx %g0, %g0, %g0 - and %o5, %g2, %g3 - cmp %g3, %o0 - add %o1, 0x20, %o1 - ldda [%o1] ASI_IC_TAG, %o4 - be,pn %xcc, iflush1 - -2: nop - and %g5, %g2, %g5 - cmp %g5, %o0 - be,pn %xcc, iflush2 -3: cmp %o1, %o2 - bne,pt %xcc, 1b - addx %g0, %g0, %g0 - nop - + membar #StoreStore + srlx %o0, PAGE_SHIFT, %o0 + sethi %uhi(PAGE_OFFSET), %g1 + sllx %o0, PAGE_SHIFT, %o0 + sethi %hi(PAGE_SIZE), %g2 + sllx %g1, 32, %g1 + add %o0, %g1, %o0 +1: subcc %g2, 32, %g2 + bne,pt %icc, 1b + flush %o0 + %g2 retl - ldx [%g6 + TI_TASK], %g4 + nop -iflush1:sub %o1, 0x20, %g3 - stxa %g0, [%g3] ASI_IC_TAG - flush %g6 - ba,a,pt %xcc, 2b -iflush2:sub %o1, 0x20, %g3 - stxa %g0, [%o1 + %o2] ASI_IC_TAG - flush %g6 - ba,a,pt %xcc, 3b +#ifdef DCACHE_ALIASING_POSSIBLE -#if (PAGE_SHIFT == 13) -#define DTAG_MASK 0x3 -#elif (PAGE_SHIFT == 16) -#define DTAG_MASK 0x1f -#elif (PAGE_SHIFT == 19) -#define DTAG_MASK 0xff -#elif (PAGE_SHIFT == 22) -#define DTAG_MASK 0x3ff +#if (PAGE_SHIFT != 13) +#error only page shift of 13 is supported by dcache flush #endif +#define DTAG_MASK 0x3 + + /* This routine is Spitfire specific so the hardcoded + * D-cache size and line-size are OK. + */ .align 64 .globl __flush_dcache_page __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ sethi %uhi(PAGE_OFFSET), %g1 sllx %g1, 32, %g1 - sub %o0, %g1, %o0 - clr %o4 - srlx %o0, 11, %o0 - sethi %hi(1 << 14), %o2 -1: ldxa [%o4] ASI_DCACHE_TAG, %o3 ! LSU Group - add %o4, (1 << 5), %o4 ! IEU0 - ldxa [%o4] ASI_DCACHE_TAG, %g1 ! LSU Group - add %o4, (1 << 5), %o4 ! IEU0 - ldxa [%o4] ASI_DCACHE_TAG, %g2 ! LSU Group o3 available - add %o4, (1 << 5), %o4 ! IEU0 - andn %o3, DTAG_MASK, %o3 ! IEU1 - ldxa [%o4] ASI_DCACHE_TAG, %g3 ! LSU Group - add %o4, (1 << 5), %o4 ! IEU0 - andn %g1, DTAG_MASK, %g1 ! IEU1 - cmp %o0, %o3 ! IEU1 Group - be,a,pn %xcc, dflush1 ! CTI - sub %o4, (4 << 5), %o4 ! IEU0 (Group) - cmp %o0, %g1 ! IEU1 Group - andn %g2, DTAG_MASK, %g2 ! IEU0 - be,a,pn %xcc, dflush2 ! CTI - sub %o4, (3 << 5), %o4 ! IEU0 (Group) - cmp %o0, %g2 ! IEU1 Group - andn %g3, DTAG_MASK, %g3 ! IEU0 - be,a,pn %xcc, dflush3 ! CTI - sub %o4, (2 << 5), %o4 ! IEU0 (Group) - cmp %o0, %g3 ! IEU1 Group - be,a,pn %xcc, dflush4 ! CTI - sub %o4, (1 << 5), %o4 ! IEU0 -2: cmp %o4, %o2 ! IEU1 Group - bne,pt %xcc, 1b ! CTI - nop ! IEU0 + sub %o0, %g1, %o0 ! physical address + srlx %o0, 11, %o0 ! make D-cache TAG + sethi %hi(1 << 14), %o2 ! D-cache size + sub %o2, (1 << 5), %o2 ! D-cache line size +1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG + andcc %o3, DTAG_MASK, %g0 ! Valid? + be,pn %xcc, 2f ! Nope, branch + andn %o3, DTAG_MASK, %o3 ! Clear valid bits + cmp %o3, %o0 ! TAG match? + bne,pt %xcc, 2f ! Nope, branch + nop + stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG + membar #Sync +2: brnz,pt %o2, 1b + sub %o2, (1 << 5), %o2 ! D-cache line size /* The I-cache does not snoop local stores so we * better flush that too when necessary. @@ -314,104 +184,64 @@ __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ retl nop -dflush1:stxa %g0, [%o4] ASI_DCACHE_TAG - add %o4, (1 << 5), %o4 -dflush2:stxa %g0, [%o4] ASI_DCACHE_TAG - add %o4, (1 << 5), %o4 -dflush3:stxa %g0, [%o4] ASI_DCACHE_TAG - add %o4, (1 << 5), %o4 -dflush4:stxa %g0, [%o4] ASI_DCACHE_TAG - add %o4, (1 << 5), %o4 - membar #Sync - ba,pt %xcc, 2b - nop +#endif /* DCACHE_ALIASING_POSSIBLE */ - .align 32 -__prefill_dtlb: - rdpr %pstate, %g7 - wrpr %g7, PSTATE_IE, %pstate - mov TLB_TAG_ACCESS, %g1 - stxa %o5, [%g1] ASI_DMMU - stxa %o2, [%g0] ASI_DTLB_DATA_IN - flush %g6 - retl - wrpr %g7, %pstate -__prefill_itlb: - rdpr %pstate, %g7 - wrpr %g7, PSTATE_IE, %pstate - mov TLB_TAG_ACCESS, %g1 - stxa %o5, [%g1] ASI_IMMU - stxa %o2, [%g0] ASI_ITLB_DATA_IN - flush %g6 - retl - wrpr %g7, %pstate - - .globl __update_mmu_cache -__update_mmu_cache: /* %o0=hw_context, %o1=address, %o2=pte, %o3=fault_code */ - srlx %o1, PAGE_SHIFT, %o1 - andcc %o3, FAULT_CODE_DTLB, %g0 - sllx %o1, PAGE_SHIFT, %o5 - bne,pt %xcc, __prefill_dtlb - or %o5, %o0, %o5 - ba,a,pt %xcc, __prefill_itlb - - /* Cheetah specific versions, patched at boot time. */ -__cheetah_flush_tlb_page: /* 14 insns */ - rdpr %pstate, %g5 - andn %g5, PSTATE_IE, %g2 - wrpr %g2, 0x0, %pstate - wrpr %g0, 1, %tl - mov PRIMARY_CONTEXT, %o2 - ldxa [%o2] ASI_DMMU, %g2 - stxa %o0, [%o2] ASI_DMMU - stxa %g0, [%o1] ASI_DMMU_DEMAP - stxa %g0, [%o1] ASI_IMMU_DEMAP - stxa %g2, [%o2] ASI_DMMU - flush %g6 - wrpr %g0, 0, %tl - retl - wrpr %g5, 0x0, %pstate + .previous -__cheetah_flush_tlb_mm: /* 15 insns */ - rdpr %pstate, %g5 - andn %g5, PSTATE_IE, %g2 + /* Cheetah specific versions, patched at boot time. */ +__cheetah_flush_tlb_mm: /* 19 insns */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 wrpr %g2, 0x0, %pstate wrpr %g0, 1, %tl mov PRIMARY_CONTEXT, %o2 mov 0x40, %g3 ldxa [%o2] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 + sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 + or %o0, %o1, %o0 /* Preserve nucleus page size fields */ stxa %o0, [%o2] ASI_DMMU stxa %g0, [%g3] ASI_DMMU_DEMAP stxa %g0, [%g3] ASI_IMMU_DEMAP stxa %g2, [%o2] ASI_DMMU - flush %g6 + sethi %hi(KERNBASE), %o2 + flush %o2 wrpr %g0, 0, %tl retl - wrpr %g5, 0x0, %pstate + wrpr %g7, 0x0, %pstate -__cheetah_flush_tlb_range: /* 20 insns */ - cmp %o5, %o4 - blu,pt %xcc, 9f - rdpr %pstate, %g5 - andn %g5, PSTATE_IE, %g2 +__cheetah_flush_tlb_pending: /* 27 insns */ + /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ + rdpr %pstate, %g7 + sllx %o1, 3, %o1 + andn %g7, PSTATE_IE, %g2 wrpr %g2, 0x0, %pstate wrpr %g0, 1, %tl - mov PRIMARY_CONTEXT, %o2 - sub %o5, %o4, %o5 - ldxa [%o2] ASI_DMMU, %g2 - stxa %o0, [%o2] ASI_DMMU -1: stxa %g0, [%o1 + %o5] ASI_DMMU_DEMAP - stxa %g0, [%o1 + %o5] ASI_IMMU_DEMAP + mov PRIMARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 + sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 + or %o0, %o3, %o0 /* Preserve nucleus page size fields */ + stxa %o0, [%o4] ASI_DMMU +1: sub %o1, (1 << 3), %o1 + ldx [%o2 + %o1], %o3 + andcc %o3, 1, %g0 + be,pn %icc, 2f + andn %o3, 1, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +2: stxa %g0, [%o3] ASI_DMMU_DEMAP membar #Sync - brnz,pt %o5, 1b - sub %o5, %o4, %o5 - stxa %g2, [%o2] ASI_DMMU - flush %g6 + brnz,pt %o1, 1b + nop + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 wrpr %g0, 0, %tl -9: retl - wrpr %g5, 0x0, %pstate + retl + wrpr %g7, 0x0, %pstate -flush_dcpage_cheetah: /* 11 insns */ +#ifdef DCACHE_ALIASING_POSSIBLE +__cheetah_flush_dcache_page: /* 11 insns */ sethi %uhi(PAGE_OFFSET), %g1 sllx %g1, 32, %g1 sub %o0, %g1, %o0 @@ -423,8 +253,78 @@ flush_dcpage_cheetah: /* 11 insns */ nop retl /* I-cache flush never needed on Cheetah, see callers. */ nop +#endif /* DCACHE_ALIASING_POSSIBLE */ + + /* Hypervisor specific versions, patched at boot time. */ +__hypervisor_tlb_tl0_error: + save %sp, -192, %sp + mov %i0, %o0 + call hypervisor_tlbop_error + mov %i1, %o1 + ret + restore + +__hypervisor_flush_tlb_mm: /* 10 insns */ + mov %o0, %o2 /* ARG2: mmu context */ + mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, __hypervisor_tlb_tl0_error + mov HV_FAST_MMU_DEMAP_CTX, %o1 + retl + nop + +__hypervisor_flush_tlb_pending: /* 16 insns */ + /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ + sllx %o1, 3, %g1 + mov %o2, %g2 + mov %o0, %g3 +1: sub %g1, (1 << 3), %g1 + ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ + mov %g3, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + srlx %o0, PAGE_SHIFT, %o0 + sllx %o0, PAGE_SHIFT, %o0 + ta HV_MMU_UNMAP_ADDR_TRAP + brnz,pn %o0, __hypervisor_tlb_tl0_error + mov HV_MMU_UNMAP_ADDR_TRAP, %o1 + brnz,pt %g1, 1b + nop + retl + nop + +__hypervisor_flush_tlb_kernel_range: /* 16 insns */ + /* %o0=start, %o1=end */ + cmp %o0, %o1 + be,pn %xcc, 2f + sethi %hi(PAGE_SIZE), %g3 + mov %o0, %g1 + sub %o1, %g1, %g2 + sub %g2, %g3, %g2 +1: add %g1, %g2, %o0 /* ARG0: virtual address */ + mov 0, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + ta HV_MMU_UNMAP_ADDR_TRAP + brnz,pn %o0, __hypervisor_tlb_tl0_error + mov HV_MMU_UNMAP_ADDR_TRAP, %o1 + brnz,pt %g2, 1b + sub %g2, %g3, %g2 +2: retl + nop + +#ifdef DCACHE_ALIASING_POSSIBLE + /* XXX Niagara and friends have an 8K cache, so no aliasing is + * XXX possible, but nothing explicit in the Hypervisor API + * XXX guarantees this. + */ +__hypervisor_flush_dcache_page: /* 2 insns */ + retl + nop +#endif -cheetah_patch_one: +tlb_patch_one: 1: lduw [%o1], %g1 stw %g1, [%o0] flush %o0 @@ -439,33 +339,28 @@ cheetah_patch_one: cheetah_patch_cachetlbops: save %sp, -128, %sp - sethi %hi(__flush_tlb_page), %o0 - or %o0, %lo(__flush_tlb_page), %o0 - sethi %hi(__cheetah_flush_tlb_page), %o1 - or %o1, %lo(__cheetah_flush_tlb_page), %o1 - call cheetah_patch_one - mov 14, %o2 - sethi %hi(__flush_tlb_mm), %o0 or %o0, %lo(__flush_tlb_mm), %o0 sethi %hi(__cheetah_flush_tlb_mm), %o1 or %o1, %lo(__cheetah_flush_tlb_mm), %o1 - call cheetah_patch_one - mov 15, %o2 + call tlb_patch_one + mov 19, %o2 - sethi %hi(__flush_tlb_range), %o0 - or %o0, %lo(__flush_tlb_range), %o0 - sethi %hi(__cheetah_flush_tlb_range), %o1 - or %o1, %lo(__cheetah_flush_tlb_range), %o1 - call cheetah_patch_one - mov 20, %o2 + sethi %hi(__flush_tlb_pending), %o0 + or %o0, %lo(__flush_tlb_pending), %o0 + sethi %hi(__cheetah_flush_tlb_pending), %o1 + or %o1, %lo(__cheetah_flush_tlb_pending), %o1 + call tlb_patch_one + mov 27, %o2 +#ifdef DCACHE_ALIASING_POSSIBLE sethi %hi(__flush_dcache_page), %o0 or %o0, %lo(__flush_dcache_page), %o0 - sethi %hi(flush_dcpage_cheetah), %o1 - or %o1, %lo(flush_dcpage_cheetah), %o1 - call cheetah_patch_one + sethi %hi(__cheetah_flush_dcache_page), %o1 + or %o1, %lo(__cheetah_flush_dcache_page), %o1 + call tlb_patch_one mov 11, %o2 +#endif /* DCACHE_ALIASING_POSSIBLE */ ret restore @@ -479,66 +374,64 @@ cheetah_patch_cachetlbops: * %g1 address arg 1 (tlb page and range flushes) * %g7 address arg 2 (tlb range flush only) * - * %g6 ivector table, don't touch - * %g2 scratch 1 - * %g3 scratch 2 - * %g4 scratch 3 - * - * TODO: Make xcall TLB range flushes use the tricks above... -DaveM + * %g6 scratch 1 + * %g2 scratch 2 + * %g3 scratch 3 + * %g4 scratch 4 */ .align 32 - .globl xcall_flush_tlb_page, xcall_flush_tlb_mm, xcall_flush_tlb_range -xcall_flush_tlb_page: + .globl xcall_flush_tlb_mm +xcall_flush_tlb_mm: /* 21 insns */ mov PRIMARY_CONTEXT, %g2 ldxa [%g2] ASI_DMMU, %g3 + srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 + sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 + or %g5, %g4, %g5 /* Preserve nucleus page size fields */ stxa %g5, [%g2] ASI_DMMU - stxa %g0, [%g1] ASI_DMMU_DEMAP - stxa %g0, [%g1] ASI_IMMU_DEMAP - stxa %g3, [%g2] ASI_DMMU - retry - nop - -xcall_flush_tlb_mm: - mov PRIMARY_CONTEXT, %g2 mov 0x40, %g4 - ldxa [%g2] ASI_DMMU, %g3 - stxa %g5, [%g2] ASI_DMMU stxa %g0, [%g4] ASI_DMMU_DEMAP stxa %g0, [%g4] ASI_IMMU_DEMAP stxa %g3, [%g2] ASI_DMMU retry - -xcall_flush_tlb_range: - sethi %hi(PAGE_SIZE - 1), %g2 - or %g2, %lo(PAGE_SIZE - 1), %g2 - andn %g1, %g2, %g1 - andn %g7, %g2, %g7 - sub %g7, %g1, %g3 - add %g2, 1, %g2 - srlx %g3, PAGE_SHIFT, %g4 - cmp %g4, 96 - - bgu,pn %icc, xcall_flush_tlb_mm - mov PRIMARY_CONTEXT, %g4 - ldxa [%g4] ASI_DMMU, %g7 - sub %g3, %g2, %g3 - stxa %g5, [%g4] ASI_DMMU + nop + nop + nop + nop + nop + nop + nop nop nop nop -1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP - stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP + .globl xcall_flush_tlb_pending +xcall_flush_tlb_pending: /* 21 insns */ + /* %g5=context, %g1=nr, %g7=vaddrs[] */ + sllx %g1, 3, %g1 + mov PRIMARY_CONTEXT, %g4 + ldxa [%g4] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 + sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 + or %g5, %g4, %g5 + mov PRIMARY_CONTEXT, %g4 + stxa %g5, [%g4] ASI_DMMU +1: sub %g1, (1 << 3), %g1 + ldx [%g7 + %g1], %g5 + andcc %g5, 0x1, %g0 + be,pn %icc, 2f + + andn %g5, 0x1, %g5 + stxa %g0, [%g5] ASI_IMMU_DEMAP +2: stxa %g0, [%g5] ASI_DMMU_DEMAP membar #Sync - brnz,pt %g3, 1b - sub %g3, %g2, %g3 - stxa %g7, [%g4] ASI_DMMU + brnz,pt %g1, 1b + nop + stxa %g2, [%g4] ASI_DMMU retry nop - nop .globl xcall_flush_tlb_kernel_range -xcall_flush_tlb_kernel_range: +xcall_flush_tlb_kernel_range: /* 25 insns */ sethi %hi(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2 andn %g1, %g2, %g1 @@ -556,14 +449,29 @@ xcall_flush_tlb_kernel_range: nop nop nop + nop + nop + nop + nop + nop + nop + nop + nop /* This runs in a very controlled environment, so we do * not need to worry about BH races etc. */ .globl xcall_sync_tick xcall_sync_tick: - rdpr %pstate, %g2 + +661: rdpr %pstate, %g2 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + rdpr %pil, %g2 wrpr %g0, 15, %pil sethi %hi(109f), %g7 @@ -586,8 +494,15 @@ xcall_sync_tick: */ .globl xcall_report_regs xcall_report_regs: - rdpr %pstate, %g2 + +661: rdpr %pstate, %g2 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + rdpr %pil, %g2 wrpr %g0, 15, %pil sethi %hi(109f), %g7 @@ -600,6 +515,7 @@ xcall_report_regs: b rtrap_xcall ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 +#ifdef DCACHE_ALIASING_POSSIBLE .align 32 .globl xcall_flush_dcache_page_cheetah xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ @@ -611,12 +527,13 @@ xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ nop retry nop +#endif /* DCACHE_ALIASING_POSSIBLE */ .globl xcall_flush_dcache_page_spitfire xcall_flush_dcache_page_spitfire: /* %g1 == physical page address %g7 == kernel page virtual address %g5 == (page->mapping != NULL) */ -#if (L1DCACHE_SIZE > PAGE_SIZE) +#ifdef DCACHE_ALIASING_POSSIBLE srlx %g1, (13 - 2), %g1 ! Form tag comparitor sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K sub %g3, (1 << 5), %g3 ! D$ linesize == 32 @@ -635,7 +552,7 @@ xcall_flush_dcache_page_spitfire: /* %g1 == physical page address sub %g3, (1 << 5), %g3 brz,pn %g5, 2f -#endif /* L1DCACHE_SIZE > PAGE_SIZE */ +#endif /* DCACHE_ALIASING_POSSIBLE */ sethi %hi(PAGE_SIZE), %g3 1: flush %g7 @@ -647,78 +564,96 @@ xcall_flush_dcache_page_spitfire: /* %g1 == physical page address nop nop - .globl xcall_promstop -xcall_promstop: - rdpr %pstate, %g2 - wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate - rdpr %pil, %g2 - wrpr %g0, 15, %pil - sethi %hi(109f), %g7 - b,pt %xcc, etrap_irq -109: or %g7, %lo(109b), %g7 - flushw - call prom_stopself - nop - /* We should not return, just spin if we do... */ -1: b,a,pt %xcc, 1b - nop - - .data - -errata32_hwbug: - .xword 0 - - .text - - /* These two are not performance critical... */ - .globl xcall_flush_tlb_all_spitfire -xcall_flush_tlb_all_spitfire: - /* Spitfire Errata #32 workaround. */ - sethi %hi(errata32_hwbug), %g4 - stx %g0, [%g4 + %lo(errata32_hwbug)] - - clr %g2 - clr %g3 -1: ldxa [%g3] ASI_DTLB_DATA_ACCESS, %g4 - and %g4, _PAGE_L, %g5 - brnz,pn %g5, 2f - mov TLB_TAG_ACCESS, %g7 - - stxa %g0, [%g7] ASI_DMMU - membar #Sync - stxa %g0, [%g3] ASI_DTLB_DATA_ACCESS + /* %g5: error + * %g6: tlb op + */ +__hypervisor_tlb_xcall_error: + mov %g5, %g4 + mov %g6, %g5 + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o0 + call hypervisor_tlbop_error_xcall + mov %l5, %o1 + ba,a,pt %xcc, rtrap_clr_l6 + + .globl __hypervisor_xcall_flush_tlb_mm +__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ + /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ + mov %o0, %g2 + mov %o1, %g3 + mov %o2, %g4 + mov %o3, %g1 + mov %o5, %g7 + clr %o0 /* ARG0: CPU lists unimplemented */ + clr %o1 /* ARG1: CPU lists unimplemented */ + mov %g5, %o2 /* ARG2: mmu context */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + mov HV_FAST_MMU_DEMAP_CTX, %g6 + brnz,pn %o0, __hypervisor_tlb_xcall_error + mov %o0, %g5 + mov %g2, %o0 + mov %g3, %o1 + mov %g4, %o2 + mov %g1, %o3 + mov %g7, %o5 membar #Sync + retry - /* Spitfire Errata #32 workaround. */ - sethi %hi(errata32_hwbug), %g4 - stx %g0, [%g4 + %lo(errata32_hwbug)] - -2: ldxa [%g3] ASI_ITLB_DATA_ACCESS, %g4 - and %g4, _PAGE_L, %g5 - brnz,pn %g5, 2f - mov TLB_TAG_ACCESS, %g7 - - stxa %g0, [%g7] ASI_IMMU - membar #Sync - stxa %g0, [%g3] ASI_ITLB_DATA_ACCESS + .globl __hypervisor_xcall_flush_tlb_pending +__hypervisor_xcall_flush_tlb_pending: /* 21 insns */ + /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */ + sllx %g1, 3, %g1 + mov %o0, %g2 + mov %o1, %g3 + mov %o2, %g4 +1: sub %g1, (1 << 3), %g1 + ldx [%g7 + %g1], %o0 /* ARG0: virtual address */ + mov %g5, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + srlx %o0, PAGE_SHIFT, %o0 + sllx %o0, PAGE_SHIFT, %o0 + ta HV_MMU_UNMAP_ADDR_TRAP + mov HV_MMU_UNMAP_ADDR_TRAP, %g6 + brnz,a,pn %o0, __hypervisor_tlb_xcall_error + mov %o0, %g5 + brnz,pt %g1, 1b + nop + mov %g2, %o0 + mov %g3, %o1 + mov %g4, %o2 membar #Sync - - /* Spitfire Errata #32 workaround. */ - sethi %hi(errata32_hwbug), %g4 - stx %g0, [%g4 + %lo(errata32_hwbug)] - -2: add %g2, 1, %g2 - cmp %g2, SPITFIRE_HIGHEST_LOCKED_TLBENT - ble,pt %icc, 1b - sll %g2, 3, %g3 - flush %g6 retry - .globl xcall_flush_tlb_all_cheetah -xcall_flush_tlb_all_cheetah: - mov 0x80, %g2 - stxa %g0, [%g2] ASI_DMMU_DEMAP - stxa %g0, [%g2] ASI_IMMU_DEMAP + .globl __hypervisor_xcall_flush_tlb_kernel_range +__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ + /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + add %g2, 1, %g2 + sub %g3, %g2, %g3 + mov %o0, %g2 + mov %o1, %g4 + mov %o2, %g7 +1: add %g1, %g3, %o0 /* ARG0: virtual address */ + mov 0, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + ta HV_MMU_UNMAP_ADDR_TRAP + mov HV_MMU_UNMAP_ADDR_TRAP, %g6 + brnz,pn %o0, __hypervisor_tlb_xcall_error + mov %o0, %g5 + sethi %hi(PAGE_SIZE), %o2 + brnz,pt %g3, 1b + sub %g3, %o2, %g3 + mov %g2, %o0 + mov %g4, %o1 + mov %g7, %o2 + membar #Sync retry /* These just get rescheduled to PIL vectors. */ @@ -737,4 +672,70 @@ xcall_capture: wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint retry + .globl xcall_new_mmu_context_version +xcall_new_mmu_context_version: + wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint + retry + #endif /* CONFIG_SMP */ + + + .globl hypervisor_patch_cachetlbops +hypervisor_patch_cachetlbops: + save %sp, -128, %sp + + sethi %hi(__flush_tlb_mm), %o0 + or %o0, %lo(__flush_tlb_mm), %o0 + sethi %hi(__hypervisor_flush_tlb_mm), %o1 + or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 + call tlb_patch_one + mov 10, %o2 + + sethi %hi(__flush_tlb_pending), %o0 + or %o0, %lo(__flush_tlb_pending), %o0 + sethi %hi(__hypervisor_flush_tlb_pending), %o1 + or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 + call tlb_patch_one + mov 16, %o2 + + sethi %hi(__flush_tlb_kernel_range), %o0 + or %o0, %lo(__flush_tlb_kernel_range), %o0 + sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 + or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 16, %o2 + +#ifdef DCACHE_ALIASING_POSSIBLE + sethi %hi(__flush_dcache_page), %o0 + or %o0, %lo(__flush_dcache_page), %o0 + sethi %hi(__hypervisor_flush_dcache_page), %o1 + or %o1, %lo(__hypervisor_flush_dcache_page), %o1 + call tlb_patch_one + mov 2, %o2 +#endif /* DCACHE_ALIASING_POSSIBLE */ + +#ifdef CONFIG_SMP + sethi %hi(xcall_flush_tlb_mm), %o0 + or %o0, %lo(xcall_flush_tlb_mm), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 + call tlb_patch_one + mov 21, %o2 + + sethi %hi(xcall_flush_tlb_pending), %o0 + or %o0, %lo(xcall_flush_tlb_pending), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1 + call tlb_patch_one + mov 21, %o2 + + sethi %hi(xcall_flush_tlb_kernel_range), %o0 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 25, %o2 +#endif /* CONFIG_SMP */ + + ret + restore