X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fsparc64%2Fmm%2Fultra.S;h=16410ba9555cf58f784d339bbac184d47b8d2e24;hb=052710fcf296f48df03c0e248eface251a7ab22c;hp=af8205edfbd0fe80b234fb8a2203a49f39751541;hpb=a91482bdcc2e0f6035702e46f1b99043a0893346;p=linux-2.6.git diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S index af8205edf..16410ba95 100644 --- a/arch/sparc64/mm/ultra.S +++ b/arch/sparc64/mm/ultra.S @@ -26,7 +26,25 @@ */ .text .align 32 - .globl __flush_tlb_mm + .globl __flush_tlb_page, __flush_tlb_mm, __flush_tlb_range +__flush_tlb_page: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=page&PAGE_MASK, %o2=SECONDARY_CONTEXT */ + ldxa [%o2] ASI_DMMU, %g2 + cmp %g2, %o0 + bne,pn %icc, __spitfire_flush_tlb_page_slow + or %o1, 0x10, %g3 + stxa %g0, [%g3] ASI_DMMU_DEMAP + stxa %g0, [%g3] ASI_IMMU_DEMAP + retl + flush %g6 + nop + nop + nop + nop + nop + nop + nop + nop + __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ ldxa [%o1] ASI_DMMU, %g2 cmp %g2, %o0 @@ -45,32 +63,84 @@ __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ nop nop - .align 32 - .globl __flush_tlb_pending -__flush_tlb_pending: - /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ - rdpr %pstate, %g5 - sllx %o1, 3, %o1 - andn %g5, PSTATE_IE, %g2 - wrpr %g2, %pstate - mov SECONDARY_CONTEXT, %o4 - ldxa [%o4] ASI_DMMU, %g2 - stxa %o0, [%o4] ASI_DMMU -1: sub %o1, (1 << 3), %o1 - ldx [%o2 + %o1], %o3 - andcc %o3, 1, %g0 - andn %o3, 1, %o3 - be,pn %icc, 2f - or %o3, 0x10, %o3 - stxa %g0, [%o3] ASI_IMMU_DEMAP -2: stxa %g0, [%o3] ASI_DMMU_DEMAP - membar #Sync - brnz,pt %o1, 1b - nop - stxa %g2, [%o4] ASI_DMMU +__flush_tlb_range: /* %o0=(ctx&TAG_CONTEXT_BITS), %o1=start&PAGE_MASK, %o2=SECONDARY_CONTEXT, + * %o3=end&PAGE_MASK, %o4=PAGE_SIZE, %o5=(end - start) + */ +#define TLB_MAGIC 207 /* Students, do you know how I calculated this? -DaveM */ + cmp %o5, %o4 + bleu,pt %xcc, __flush_tlb_page + srlx %o5, PAGE_SHIFT, %g5 + cmp %g5, TLB_MAGIC + bgeu,pn %icc, __spitfire_flush_tlb_range_constant_time + or %o1, 0x10, %g5 + ldxa [%o2] ASI_DMMU, %g2 + cmp %g2, %o0 +__spitfire_flush_tlb_range_page_by_page: + bne,pn %icc, __spitfire_flush_tlb_range_pbp_slow + sub %o5, %o4, %o5 +1: stxa %g0, [%g5 + %o5] ASI_DMMU_DEMAP + stxa %g0, [%g5 + %o5] ASI_IMMU_DEMAP + brnz,pt %o5, 1b + sub %o5, %o4, %o5 + retl + flush %g6 +__spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */ + rdpr %pstate, %g1 + wrpr %g1, PSTATE_IE, %pstate + mov TLB_TAG_ACCESS, %g3 + mov ((SPITFIRE_HIGHEST_LOCKED_TLBENT-1) << 3), %g2 + + /* Spitfire Errata #32 workaround. */ + mov 0x8, %o4 + stxa %g0, [%o4] ASI_DMMU flush %g6 + +1: ldxa [%g2] ASI_ITLB_TAG_READ, %o4 + and %o4, TAG_CONTEXT_BITS, %o5 + cmp %o5, %o0 + bne,pt %icc, 2f + andn %o4, TAG_CONTEXT_BITS, %o4 + cmp %o4, %o1 + blu,pt %xcc, 2f + cmp %o4, %o3 + blu,pn %xcc, 4f +2: ldxa [%g2] ASI_DTLB_TAG_READ, %o4 + and %o4, TAG_CONTEXT_BITS, %o5 + cmp %o5, %o0 + andn %o4, TAG_CONTEXT_BITS, %o4 + bne,pt %icc, 3f + cmp %o4, %o1 + blu,pt %xcc, 3f + cmp %o4, %o3 + blu,pn %xcc, 5f + nop +3: brnz,pt %g2, 1b + sub %g2, (1 << 3), %g2 retl - wrpr %g5, 0x0, %pstate + wrpr %g1, 0x0, %pstate +4: stxa %g0, [%g3] ASI_IMMU + stxa %g0, [%g2] ASI_ITLB_DATA_ACCESS + flush %g6 + + /* Spitfire Errata #32 workaround. */ + mov 0x8, %o4 + stxa %g0, [%o4] ASI_DMMU + flush %g6 + + ba,pt %xcc, 2b + nop + +5: stxa %g0, [%g3] ASI_DMMU + stxa %g0, [%g2] ASI_DTLB_DATA_ACCESS + flush %g6 + + /* Spitfire Errata #32 workaround. */ + mov 0x8, %o4 + stxa %g0, [%o4] ASI_DMMU + flush %g6 + + ba,pt %xcc, 3b + nop .align 32 .globl __flush_tlb_kernel_range @@ -101,6 +171,33 @@ __spitfire_flush_tlb_mm_slow: retl wrpr %g1, 0, %pstate +__spitfire_flush_tlb_page_slow: + rdpr %pstate, %g1 + wrpr %g1, PSTATE_IE, %pstate + stxa %o0, [%o2] ASI_DMMU + stxa %g0, [%g3] ASI_DMMU_DEMAP + stxa %g0, [%g3] ASI_IMMU_DEMAP + flush %g6 + stxa %g2, [%o2] ASI_DMMU + flush %g6 + retl + wrpr %g1, 0, %pstate + +__spitfire_flush_tlb_range_pbp_slow: + rdpr %pstate, %g1 + wrpr %g1, PSTATE_IE, %pstate + stxa %o0, [%o2] ASI_DMMU + +2: stxa %g0, [%g5 + %o5] ASI_DMMU_DEMAP + stxa %g0, [%g5 + %o5] ASI_IMMU_DEMAP + brnz,pt %o5, 2b + sub %o5, %o4, %o5 + flush %g6 + stxa %g2, [%o2] ASI_DMMU + flush %g6 + retl + wrpr %g1, 0x0, %pstate + /* * The following code flushes one page_size worth. */ @@ -259,6 +356,22 @@ __update_mmu_cache: /* %o0=hw_context, %o1=address, %o2=pte, %o3=fault_code */ ba,a,pt %xcc, __prefill_itlb /* Cheetah specific versions, patched at boot time. */ +__cheetah_flush_tlb_page: /* 14 insns */ + rdpr %pstate, %g5 + andn %g5, PSTATE_IE, %g2 + wrpr %g2, 0x0, %pstate + wrpr %g0, 1, %tl + mov PRIMARY_CONTEXT, %o2 + ldxa [%o2] ASI_DMMU, %g2 + stxa %o0, [%o2] ASI_DMMU + stxa %g0, [%o1] ASI_DMMU_DEMAP + stxa %g0, [%o1] ASI_IMMU_DEMAP + stxa %g2, [%o2] ASI_DMMU + flush %g6 + wrpr %g0, 0, %tl + retl + wrpr %g5, 0x0, %pstate + __cheetah_flush_tlb_mm: /* 15 insns */ rdpr %pstate, %g5 andn %g5, PSTATE_IE, %g2 @@ -276,29 +389,26 @@ __cheetah_flush_tlb_mm: /* 15 insns */ retl wrpr %g5, 0x0, %pstate -__cheetah_flush_tlb_pending: /* 22 insns */ - /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ - rdpr %pstate, %g5 - sllx %o1, 3, %o1 +__cheetah_flush_tlb_range: /* 20 insns */ + cmp %o5, %o4 + blu,pt %xcc, 9f + rdpr %pstate, %g5 andn %g5, PSTATE_IE, %g2 wrpr %g2, 0x0, %pstate wrpr %g0, 1, %tl - mov PRIMARY_CONTEXT, %o4 - ldxa [%o4] ASI_DMMU, %g2 - stxa %o0, [%o4] ASI_DMMU -1: sub %o1, (1 << 3), %o1 - ldx [%o2 + %o1], %o3 - andcc %o3, 1, %g0 - be,pn %icc, 2f - andn %o3, 1, %o3 - stxa %g0, [%o3] ASI_IMMU_DEMAP -2: stxa %g0, [%o3] ASI_DMMU_DEMAP - brnz,pt %o1, 1b - membar #Sync - stxa %g2, [%o4] ASI_DMMU + mov PRIMARY_CONTEXT, %o2 + sub %o5, %o4, %o5 + ldxa [%o2] ASI_DMMU, %g2 + stxa %o0, [%o2] ASI_DMMU +1: stxa %g0, [%o1 + %o5] ASI_DMMU_DEMAP + stxa %g0, [%o1 + %o5] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %o5, 1b + sub %o5, %o4, %o5 + stxa %g2, [%o2] ASI_DMMU flush %g6 wrpr %g0, 0, %tl - retl +9: retl wrpr %g5, 0x0, %pstate flush_dcpage_cheetah: /* 11 insns */ @@ -329,6 +439,13 @@ cheetah_patch_one: cheetah_patch_cachetlbops: save %sp, -128, %sp + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__cheetah_flush_tlb_page), %o1 + or %o1, %lo(__cheetah_flush_tlb_page), %o1 + call cheetah_patch_one + mov 14, %o2 + sethi %hi(__flush_tlb_mm), %o0 or %o0, %lo(__flush_tlb_mm), %o0 sethi %hi(__cheetah_flush_tlb_mm), %o1 @@ -336,12 +453,12 @@ cheetah_patch_cachetlbops: call cheetah_patch_one mov 15, %o2 - sethi %hi(__flush_tlb_pending), %o0 - or %o0, %lo(__flush_tlb_pending), %o0 - sethi %hi(__cheetah_flush_tlb_pending), %o1 - or %o1, %lo(__cheetah_flush_tlb_pending), %o1 + sethi %hi(__flush_tlb_range), %o0 + or %o0, %lo(__flush_tlb_range), %o0 + sethi %hi(__cheetah_flush_tlb_range), %o1 + or %o1, %lo(__cheetah_flush_tlb_range), %o1 call cheetah_patch_one - mov 22, %o2 + mov 20, %o2 sethi %hi(__flush_dcache_page), %o0 or %o0, %lo(__flush_dcache_page), %o0 @@ -370,7 +487,17 @@ cheetah_patch_cachetlbops: * TODO: Make xcall TLB range flushes use the tricks above... -DaveM */ .align 32 - .globl xcall_flush_tlb_mm + .globl xcall_flush_tlb_page, xcall_flush_tlb_mm, xcall_flush_tlb_range +xcall_flush_tlb_page: + mov PRIMARY_CONTEXT, %g2 + ldxa [%g2] ASI_DMMU, %g3 + stxa %g5, [%g2] ASI_DMMU + stxa %g0, [%g1] ASI_DMMU_DEMAP + stxa %g0, [%g1] ASI_IMMU_DEMAP + stxa %g3, [%g2] ASI_DMMU + retry + nop + xcall_flush_tlb_mm: mov PRIMARY_CONTEXT, %g2 mov 0x40, %g4 @@ -381,26 +508,34 @@ xcall_flush_tlb_mm: stxa %g3, [%g2] ASI_DMMU retry - .globl xcall_flush_tlb_pending -xcall_flush_tlb_pending: - /* %g5=context, %g1=nr, %g7=vaddrs[] */ - sllx %g1, 3, %g1 - mov PRIMARY_CONTEXT, %g4 - ldxa [%g4] ASI_DMMU, %g2 +xcall_flush_tlb_range: + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + add %g2, 1, %g2 + srlx %g3, PAGE_SHIFT, %g4 + cmp %g4, 96 + + bgu,pn %icc, xcall_flush_tlb_mm + mov PRIMARY_CONTEXT, %g4 + ldxa [%g4] ASI_DMMU, %g7 + sub %g3, %g2, %g3 stxa %g5, [%g4] ASI_DMMU -1: sub %g1, (1 << 3), %g1 - ldx [%g7 + %g1], %g5 - andcc %g5, 0x1, %g0 - be,pn %icc, 2f - - andn %g5, 0x1, %g5 - stxa %g0, [%g5] ASI_IMMU_DEMAP -2: stxa %g0, [%g5] ASI_DMMU_DEMAP + nop + nop + nop + +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP membar #Sync - brnz,pt %g1, 1b - nop - stxa %g2, [%g4] ASI_DMMU + brnz,pt %g3, 1b + sub %g3, %g2, %g3 + stxa %g7, [%g4] ASI_DMMU retry + nop + nop .globl xcall_flush_tlb_kernel_range xcall_flush_tlb_kernel_range: @@ -420,6 +555,7 @@ xcall_flush_tlb_kernel_range: retry nop nop + nop /* This runs in a very controlled environment, so we do * not need to worry about BH races etc.