X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fmips%2Fmm%2Ftlbex.c;h=54507be2ab5bb07f967f23e9c8305ab5f2ddf48a;hb=43bc926fffd92024b46cafaf7350d669ba9ca884;hp=87e229f4d3d5c354b5ce3d8acf5290f8a00c23bf;hpb=cee37fe97739d85991964371c1f3a745c00dd236;p=linux-2.6.git diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 87e229f4d..54507be2a 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -6,6 +6,17 @@ * Synthesize TLB refill handlers at runtime. * * Copyright (C) 2004,2005 by Thiemo Seufer + * Copyright (C) 2005 Maciej W. Rozycki + * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) + * + * ... and the days got worse and worse and now you see + * I've gone completly out of my mind. + * + * They're coming to take me a away haha + * they're coming to take me a away hoho hihi haha + * to the funny farm where code is beautiful all the time ... + * + * (Condolences to Napoleon XIV) */ #include @@ -67,6 +78,7 @@ enum fields BIMM = 0x040, JIMM = 0x080, FUNC = 0x100, + SET = 0x200 }; #define OP_MASK 0x2f @@ -85,13 +97,15 @@ enum fields #define JIMM_SH 0 #define FUNC_MASK 0x2f #define FUNC_SH 0 +#define SET_MASK 0x7 +#define SET_SH 0 enum opcode { insn_invalid, insn_addu, insn_addiu, insn_and, insn_andi, insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl, insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0, - insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32, + insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld, insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0, insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll, @@ -128,13 +142,12 @@ static __initdata struct insn insn_table[] = { { insn_bne, M(bne_op,0,0,0,0,0), RS | RT | BIMM }, { insn_daddiu, M(daddiu_op,0,0,0,0,0), RS | RT | SIMM }, { insn_daddu, M(spec_op,0,0,0,0,daddu_op), RS | RT | RD }, - { insn_dmfc0, M(cop0_op,dmfc_op,0,0,0,0), RT | RD }, - { insn_dmtc0, M(cop0_op,dmtc_op,0,0,0,0), RT | RD }, + { insn_dmfc0, M(cop0_op,dmfc_op,0,0,0,0), RT | RD | SET}, + { insn_dmtc0, M(cop0_op,dmtc_op,0,0,0,0), RT | RD | SET}, { insn_dsll, M(spec_op,0,0,0,0,dsll_op), RT | RD | RE }, { insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE }, { insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE }, { insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE }, - { insn_dsrl32, M(spec_op,0,0,0,0,dsrl32_op), RT | RD | RE }, { insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD }, { insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 }, { insn_j, M(j_op,0,0,0,0,0), JIMM }, @@ -145,8 +158,8 @@ static __initdata struct insn insn_table[] = { { insn_lld, M(lld_op,0,0,0,0,0), RS | RT | SIMM }, { insn_lui, M(lui_op,0,0,0,0,0), RT | SIMM }, { insn_lw, M(lw_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_mfc0, M(cop0_op,mfc_op,0,0,0,0), RT | RD }, - { insn_mtc0, M(cop0_op,mtc_op,0,0,0,0), RT | RD }, + { insn_mfc0, M(cop0_op,mfc_op,0,0,0,0), RT | RD | SET}, + { insn_mtc0, M(cop0_op,mtc_op,0,0,0,0), RT | RD | SET}, { insn_ori, M(ori_op,0,0,0,0,0), RS | RT | UIMM }, { insn_rfe, M(cop0_op,cop_op,0,0,0,rfe_op), 0 }, { insn_sc, M(sc_op,0,0,0,0,0), RS | RT | SIMM }, @@ -242,6 +255,14 @@ static __init u32 build_func(u32 arg) return arg & FUNC_MASK; } +static __init u32 build_set(u32 arg) +{ + if (arg & ~SET_MASK) + printk(KERN_WARNING "TLB synthesizer field overflow\n"); + + return arg & SET_MASK; +} + /* * The order of opcode arguments is implicitly left to right, * starting with RS and ending with FUNC or IMM. @@ -273,6 +294,7 @@ static void __init build_insn(u32 **buf, enum opcode opc, ...) if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32)); if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32)); if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32)); + if (ip->fields & SET) op |= build_set(va_arg(ap, u32)); va_end(ap); **buf = op; @@ -280,69 +302,69 @@ static void __init build_insn(u32 **buf, enum opcode opc, ...) } #define I_u1u2u3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ unsigned int b, unsigned int c) \ { \ build_insn(buf, insn##op, a, b, c); \ } #define I_u2u1u3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ unsigned int b, unsigned int c) \ { \ build_insn(buf, insn##op, b, a, c); \ } #define I_u3u1u2(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ unsigned int b, unsigned int c) \ { \ build_insn(buf, insn##op, b, c, a); \ } #define I_u1u2s3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ unsigned int b, signed int c) \ { \ build_insn(buf, insn##op, a, b, c); \ } #define I_u2s3u1(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ signed int b, unsigned int c) \ { \ build_insn(buf, insn##op, c, a, b); \ } #define I_u2u1s3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ unsigned int b, signed int c) \ { \ build_insn(buf, insn##op, b, a, c); \ } #define I_u1u2(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ unsigned int b) \ { \ build_insn(buf, insn##op, a, b); \ } #define I_u1s2(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ + static inline void __init i##op(u32 **buf, unsigned int a, \ signed int b) \ { \ build_insn(buf, insn##op, a, b); \ } #define I_u1(op) \ - static inline void i##op(u32 **buf, unsigned int a) \ + static inline void __init i##op(u32 **buf, unsigned int a) \ { \ build_insn(buf, insn##op, a); \ } #define I_0(op) \ - static inline void i##op(u32 **buf) \ + static inline void __init i##op(u32 **buf) \ { \ build_insn(buf, insn##op); \ } @@ -358,15 +380,14 @@ I_u1s2(_bgezl); I_u1s2(_bltz); I_u1s2(_bltzl); I_u1u2s3(_bne); -I_u1u2(_dmfc0); -I_u1u2(_dmtc0); +I_u1u2u3(_dmfc0); +I_u1u2u3(_dmtc0); I_u2u1s3(_daddiu); I_u3u1u2(_daddu); I_u2u1u3(_dsll); I_u2u1u3(_dsll32); I_u2u1u3(_dsra); I_u2u1u3(_dsrl); -I_u2u1u3(_dsrl32); I_u3u1u2(_dsubu); I_0(_eret); I_u1(_j); @@ -377,8 +398,8 @@ I_u2s3u1(_ll); I_u2s3u1(_lld); I_u1s2(_lui); I_u2s3u1(_lw); -I_u1u2(_mfc0); -I_u1u2(_mtc0); +I_u1u2u3(_mfc0); +I_u1u2u3(_mtc0); I_u2u1u3(_ori); I_0(_rfe); I_u2s3u1(_sc); @@ -412,7 +433,6 @@ enum label_id { label_nopage_tlbm, label_smp_pgtable_change, label_r3000_write_probe_fail, - label_r3000_write_probe_ok }; struct label { @@ -445,17 +465,16 @@ L_LA(_nopage_tlbs) L_LA(_nopage_tlbm) L_LA(_smp_pgtable_change) L_LA(_r3000_write_probe_fail) -L_LA(_r3000_write_probe_ok) /* convenience macros for instructions */ -#ifdef CONFIG_MIPS64 +#ifdef CONFIG_64BIT # define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off) # define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off) # define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh) # define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh) # define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh) -# define i_MFC0(buf, rt, rd) i_dmfc0(buf, rt, rd) -# define i_MTC0(buf, rt, rd) i_dmtc0(buf, rt, rd) +# define i_MFC0(buf, rt, rd...) i_dmfc0(buf, rt, rd) +# define i_MTC0(buf, rt, rd...) i_dmtc0(buf, rt, rd) # define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val) # define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd) # define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd) @@ -467,8 +486,8 @@ L_LA(_r3000_write_probe_ok) # define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh) # define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh) # define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh) -# define i_MFC0(buf, rt, rd) i_mfc0(buf, rt, rd) -# define i_MTC0(buf, rt, rd) i_mtc0(buf, rt, rd) +# define i_MFC0(buf, rt, rd...) i_mfc0(buf, rt, rd) +# define i_MTC0(buf, rt, rd...) i_mtc0(buf, rt, rd) # define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val) # define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd) # define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd) @@ -486,11 +505,11 @@ L_LA(_r3000_write_probe_ok) #define i_ssnop(buf) i_sll(buf, 0, 0, 1) #define i_ehb(buf) i_sll(buf, 0, 0, 3) -#ifdef CONFIG_MIPS64 +#ifdef CONFIG_64BIT static __init int __attribute__((unused)) in_compat_space_p(long addr) { /* Is this address in 32bit compat space? */ - return (((addr) & 0xffffffff00000000) == 0xffffffff00000000); + return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L); } static __init int __attribute__((unused)) rel_highest(long val) @@ -516,7 +535,7 @@ static __init int rel_lo(long val) static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr) { -#if CONFIG_MIPS64 +#ifdef CONFIG_64BIT if (!in_compat_space_p(addr)) { i_lui(buf, rs, rel_highest(addr)); if (rel_higher(addr)) @@ -626,42 +645,42 @@ static __init int __attribute__((unused)) insn_has_bdelay(struct reloc *rel, } /* convenience functions for labeled branches */ -static void __attribute__((unused)) il_bltz(u32 **p, struct reloc **r, - unsigned int reg, enum label_id l) +static void __init __attribute__((unused)) + il_bltz(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) { r_mips_pc16(r, *p, l); i_bltz(p, reg, 0); } -static void __attribute__((unused)) il_b(u32 **p, struct reloc **r, +static void __init __attribute__((unused)) il_b(u32 **p, struct reloc **r, enum label_id l) { r_mips_pc16(r, *p, l); i_b(p, 0); } -static void il_beqz(u32 **p, struct reloc **r, unsigned int reg, +static void __init il_beqz(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) { r_mips_pc16(r, *p, l); i_beqz(p, reg, 0); } -static void __attribute__((unused)) +static void __init __attribute__((unused)) il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) { r_mips_pc16(r, *p, l); i_beqzl(p, reg, 0); } -static void il_bnez(u32 **p, struct reloc **r, unsigned int reg, +static void __init il_bnez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) { r_mips_pc16(r, *p, l); i_bnez(p, reg, 0); } -static void il_bgezl(u32 **p, struct reloc **r, unsigned int reg, +static void __init il_bgezl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) { r_mips_pc16(r, *p, l); @@ -673,16 +692,17 @@ static void il_bgezl(u32 **p, struct reloc **r, unsigned int reg, #define K1 27 /* Some CP0 registers */ -#define C0_INDEX 0 -#define C0_ENTRYLO0 2 -#define C0_ENTRYLO1 3 -#define C0_CONTEXT 4 -#define C0_BADVADDR 8 -#define C0_ENTRYHI 10 -#define C0_EPC 14 -#define C0_XCONTEXT 20 - -#ifdef CONFIG_MIPS64 +#define C0_INDEX 0, 0 +#define C0_ENTRYLO0 2, 0 +#define C0_TCBIND 2, 2 +#define C0_ENTRYLO1 3, 0 +#define C0_CONTEXT 4, 0 +#define C0_BADVADDR 8, 0 +#define C0_ENTRYHI 10, 0 +#define C0_EPC 14, 0 +#define C0_XCONTEXT 20, 0 + +#ifdef CONFIG_64BIT # define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT) #else # define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT) @@ -734,7 +754,7 @@ static void __init build_r3000_tlb_refill_handler(void) if (p > tlb_handler + 32) panic("TLB refill handler space exceeded"); - printk("Synthesized TLB handler (%u instructions).\n", + printk("Synthesized TLB refill handler (%u instructions).\n", (unsigned int)(p - tlb_handler)); #ifdef DEBUG_TLB { @@ -745,8 +765,7 @@ static void __init build_r3000_tlb_refill_handler(void) } #endif - memcpy((void *)CAC_BASE, tlb_handler, 0x80); - flush_icache_range(CAC_BASE, CAC_BASE + 0x80); + memcpy((void *)ebase, tlb_handler, 0x80); } /* @@ -783,6 +802,8 @@ static __initdata u32 final_handler[64]; static __init void __attribute__((unused)) build_tlb_probe_entry(u32 **p) { switch (current_cpu_data.cputype) { + /* Found by experiment: R4600 v2.0 needs this, too. */ + case CPU_R4600: case CPU_R5000: case CPU_R5000A: case CPU_NEVADA: @@ -834,20 +855,30 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, case CPU_R4700: case CPU_R5000: case CPU_R5000A: + i_nop(p); + tlbw(p); + i_nop(p); + break; + + case CPU_R4300: case CPU_5KC: case CPU_TX49XX: case CPU_AU1000: case CPU_AU1100: case CPU_AU1500: case CPU_AU1550: + case CPU_AU1200: + case CPU_PR4450: i_nop(p); tlbw(p); break; case CPU_R10000: case CPU_R12000: + case CPU_R14000: case CPU_4KC: case CPU_SB1: + case CPU_SB1A: case CPU_4KSC: case CPU_20KC: case CPU_25KF: @@ -875,6 +906,8 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, case CPU_4KEC: case CPU_24K: + case CPU_34K: + case CPU_74K: i_ehb(p); tlbw(p); break; @@ -911,6 +944,7 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, case CPU_VR4131: case CPU_VR4133: + case CPU_R5432: i_nop(p); i_nop(p); tlbw(p); @@ -923,7 +957,7 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, } } -#ifdef CONFIG_MIPS64 +#ifdef CONFIG_64BIT /* * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pmd entry. @@ -942,34 +976,24 @@ build_get_pmde64(u32 **p, struct label **l, struct reloc **r, /* No i_nop needed here, since the next insn doesn't touch TMP. */ #ifdef CONFIG_SMP +# ifdef CONFIG_MIPS_MT_SMTC + /* + * SMTC uses TCBind value as "CPU" index + */ + i_mfc0(p, ptr, C0_TCBIND); + i_dsrl(p, ptr, ptr, 19); +# else /* - * 64 bit SMP has the lower part of &pgd_current[smp_processor_id()] + * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 * stored in CONTEXT. */ - if (in_compat_space_p(pgdc)) { - i_dmfc0(p, ptr, C0_CONTEXT); - i_dsra(p, ptr, ptr, 23); - i_ld(p, ptr, 0, ptr); - } else { -#ifdef CONFIG_BUILD_ELF64 - i_dmfc0(p, ptr, C0_CONTEXT); - i_dsrl(p, ptr, ptr, 23); - i_dsll(p, ptr, ptr, 3); - i_LA_mostly(p, tmp, pgdc); - i_daddu(p, ptr, ptr, tmp); - i_dmfc0(p, tmp, C0_BADVADDR); - i_ld(p, ptr, rel_lo(pgdc), ptr); -#else - i_dmfc0(p, ptr, C0_CONTEXT); - i_lui(p, tmp, rel_highest(pgdc)); - i_dsll(p, ptr, ptr, 9); - i_daddiu(p, tmp, tmp, rel_higher(pgdc)); - i_dsrl32(p, ptr, ptr, 0); - i_and(p, ptr, ptr, tmp); - i_dmfc0(p, tmp, C0_BADVADDR); - i_ld(p, ptr, 0, ptr); + i_dmfc0(p, ptr, C0_CONTEXT); + i_dsrl(p, ptr, ptr, 23); #endif - } + i_LA_mostly(p, tmp, pgdc); + i_daddu(p, ptr, ptr, tmp); + i_dmfc0(p, tmp, C0_BADVADDR); + i_ld(p, ptr, rel_lo(pgdc), ptr); #else i_LA_mostly(p, ptr, pgdc); i_ld(p, ptr, rel_lo(pgdc), ptr); @@ -1010,7 +1034,7 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r, } } -#else /* !CONFIG_MIPS64 */ +#else /* !CONFIG_64BIT */ /* * TMP and PTR are scratch. @@ -1023,10 +1047,21 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ #ifdef CONFIG_SMP +#ifdef CONFIG_MIPS_MT_SMTC + /* + * SMTC uses TCBind value as "CPU" index + */ + i_mfc0(p, ptr, C0_TCBIND); + i_LA_mostly(p, tmp, pgdc); + i_srl(p, ptr, ptr, 19); +#else + /* + * smp_processor_id() << 3 is stored in CONTEXT. + */ i_mfc0(p, ptr, C0_CONTEXT); i_LA_mostly(p, tmp, pgdc); i_srl(p, ptr, ptr, 23); - i_sll(p, ptr, ptr, 2); +#endif i_addu(p, ptr, tmp, ptr); #else i_LA_mostly(p, ptr, pgdc); @@ -1038,7 +1073,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ } -#endif /* !CONFIG_MIPS64 */ +#endif /* !CONFIG_64BIT */ static __init void build_adjust_context(u32 **p, unsigned int ctx) { @@ -1159,7 +1194,7 @@ static void __init build_r4000_tlb_refill_handler(void) /* No need for i_nop */ } -#ifdef CONFIG_MIPS64 +#ifdef CONFIG_64BIT build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ #else build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ @@ -1171,7 +1206,7 @@ static void __init build_r4000_tlb_refill_handler(void) l_leave(&l, p); i_eret(&p); /* return from trap */ -#ifdef CONFIG_MIPS64 +#ifdef CONFIG_64BIT build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); #endif @@ -1182,7 +1217,7 @@ static void __init build_r4000_tlb_refill_handler(void) * need three, with the the second nop'ed and the third being * unused. */ -#ifdef CONFIG_MIPS32 +#ifdef CONFIG_32BIT if ((p - tlb_handler) > 64) panic("TLB refill handler space exceeded"); #else @@ -1195,12 +1230,12 @@ static void __init build_r4000_tlb_refill_handler(void) /* * Now fold the handler in the TLB refill handler space. */ -#ifdef CONFIG_MIPS32 +#ifdef CONFIG_32BIT f = final_handler; /* Simplest case, just copy the handler. */ copy_handler(relocs, labels, tlb_handler, p, f); final_len = p - tlb_handler; -#else /* CONFIG_MIPS64 */ +#else /* CONFIG_64BIT */ f = final_handler + 32; if ((p - tlb_handler) <= 32) { /* Just copy the handler. */ @@ -1235,7 +1270,7 @@ static void __init build_r4000_tlb_refill_handler(void) copy_handler(relocs, labels, split, p, final_handler); final_len = (f - (final_handler + 32)) + (p - split); } -#endif /* CONFIG_MIPS64 */ +#endif /* CONFIG_64BIT */ resolve_relocs(relocs, labels); printk("Synthesized TLB refill handler (%u instructions).\n", @@ -1245,13 +1280,19 @@ static void __init build_r4000_tlb_refill_handler(void) { int i; - for (i = 0; i < 64; i++) - printk("%08x\n", final_handler[i]); + f = final_handler; +#ifdef CONFIG_64BIT + if (final_len > 32) + final_len = 64; + else + f = final_handler + 32; +#endif /* CONFIG_64BIT */ + for (i = 0; i < final_len; i++) + printk("%08x\n", f[i]); } #endif - memcpy((void *)CAC_BASE, final_handler, 0x100); - flush_icache_range(CAC_BASE, CAC_BASE + 0x100); + memcpy((void *)ebase, final_handler, 0x100); } /* @@ -1277,37 +1318,41 @@ u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE]; u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE]; static void __init -iPTE_LW(u32 **p, struct label **l, unsigned int pte, int offset, - unsigned int ptr) +iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr) { #ifdef CONFIG_SMP # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_lld(p, pte, offset, ptr); + i_lld(p, pte, 0, ptr); else # endif - i_LL(p, pte, offset, ptr); + i_LL(p, pte, 0, ptr); #else # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_ld(p, pte, offset, ptr); + i_ld(p, pte, 0, ptr); else # endif - i_LW(p, pte, offset, ptr); + i_LW(p, pte, 0, ptr); #endif } static void __init -iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset, - unsigned int ptr) +iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr, + unsigned int mode) { +#ifdef CONFIG_64BIT_PHYS_ADDR + unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); +#endif + + i_ori(p, pte, pte, mode); #ifdef CONFIG_SMP # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_scd(p, pte, offset, ptr); + i_scd(p, pte, 0, ptr); else # endif - i_SC(p, pte, offset, ptr); + i_SC(p, pte, 0, ptr); if (r10000_llsc_war()) il_beqzl(p, r, pte, label_smp_pgtable_change); @@ -1318,7 +1363,7 @@ iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset, if (!cpu_has_64bits) { /* no i_nop needed */ i_ll(p, pte, sizeof(pte_t) / 2, ptr); - i_ori(p, pte, pte, _PAGE_VALID); + i_ori(p, pte, pte, hwmode); i_sc(p, pte, sizeof(pte_t) / 2, ptr); il_beqz(p, r, pte, label_smp_pgtable_change); /* no i_nop needed */ @@ -1331,15 +1376,15 @@ iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, int offset, #else # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_sd(p, pte, offset, ptr); + i_sd(p, pte, 0, ptr); else # endif - i_SW(p, pte, offset, ptr); + i_SW(p, pte, 0, ptr); # ifdef CONFIG_64BIT_PHYS_ADDR if (!cpu_has_64bits) { i_lw(p, pte, sizeof(pte_t) / 2, ptr); - i_ori(p, pte, pte, _PAGE_VALID); + i_ori(p, pte, pte, hwmode); i_sw(p, pte, sizeof(pte_t) / 2, ptr); i_lw(p, pte, 0, ptr); } @@ -1359,7 +1404,7 @@ build_pte_present(u32 **p, struct label **l, struct reloc **r, i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); il_bnez(p, r, pte, lid); - iPTE_LW(p, l, pte, 0, ptr); + iPTE_LW(p, l, pte, ptr); } /* Make PTE valid, store result in PTR. */ @@ -1367,8 +1412,9 @@ static void __init build_make_valid(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr) { - i_ori(p, pte, pte, _PAGE_VALID | _PAGE_ACCESSED); - iPTE_SW(p, r, pte, 0, ptr); + unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; + + iPTE_SW(p, r, pte, ptr, mode); } /* @@ -1382,7 +1428,7 @@ build_pte_writable(u32 **p, struct label **l, struct reloc **r, i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); il_bnez(p, r, pte, lid); - iPTE_LW(p, l, pte, 0, ptr); + iPTE_LW(p, l, pte, ptr); } /* Make PTE writable, update software status bits as well, then store @@ -1392,9 +1438,10 @@ static void __init build_make_write(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr) { - i_ori(p, pte, pte, - _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); - iPTE_SW(p, r, pte, 0, ptr); + unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID + | _PAGE_DIRTY); + + iPTE_SW(p, r, pte, ptr, mode); } /* @@ -1407,41 +1454,48 @@ build_pte_modifiable(u32 **p, struct label **l, struct reloc **r, { i_andi(p, pte, pte, _PAGE_WRITE); il_beqz(p, r, pte, lid); - iPTE_LW(p, l, pte, 0, ptr); + iPTE_LW(p, l, pte, ptr); } /* * R3000 style TLB load/store/modify handlers. */ -/* This places the pte in the page table at PTR into ENTRYLO0. */ +/* + * This places the pte into ENTRYLO0 and writes it with tlbwi. + * Then it returns. + */ static void __init -build_r3000_pte_reload(u32 **p, unsigned int ptr) +build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) { - i_lw(p, ptr, 0, ptr); - i_nop(p); /* load delay */ - i_mtc0(p, ptr, C0_ENTRYLO0); - i_nop(p); /* cp0 delay */ + i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ + i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ + i_tlbwi(p); + i_jr(p, tmp); + i_rfe(p); /* branch delay */ } /* - * The index register may have the probe fail bit set, - * because we would trap on access kseg2, i.e. without refill. + * This places the pte into ENTRYLO0 and writes it with tlbwi + * or tlbwr as appropriate. This is because the index register + * may have the probe fail bit set as a result of a trap on a + * kseg2 access, i.e. without refill. Then it returns. */ static void __init -build_r3000_tlb_write(u32 **p, struct label **l, struct reloc **r, - unsigned int tmp) +build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r, + unsigned int pte, unsigned int tmp) { i_mfc0(p, tmp, C0_INDEX); - i_nop(p); /* cp0 delay */ - il_bltz(p, r, tmp, label_r3000_write_probe_fail); - i_nop(p); /* branch delay */ - i_tlbwi(p); - il_b(p, r, label_r3000_write_probe_ok); - i_nop(p); /* branch delay */ + i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ + il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ + i_mfc0(p, tmp, C0_EPC); /* branch delay */ + i_tlbwi(p); /* cp0 delay */ + i_jr(p, tmp); + i_rfe(p); /* branch delay */ l_r3000_write_probe_fail(l, *p); - i_tlbwr(p); - l_r3000_write_probe_ok(l, *p); + i_tlbwr(p); /* cp0 delay */ + i_jr(p, tmp); + i_rfe(p); /* branch delay */ } static void __init @@ -1461,17 +1515,7 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, i_andi(p, pte, pte, 0xffc); /* load delay */ i_addu(p, ptr, ptr, pte); i_lw(p, pte, 0, ptr); - i_nop(p); /* load delay */ - i_tlbp(p); -} - -static void __init -build_r3000_tlbchange_handler_tail(u32 **p, unsigned int tmp) -{ - i_mfc0(p, tmp, C0_EPC); - i_nop(p); /* cp0 delay */ - i_jr(p, tmp); - i_rfe(p); /* branch delay */ + i_tlbp(p); /* load delay */ } static void __init build_r3000_tlb_load_handler(void) @@ -1486,10 +1530,9 @@ static void __init build_r3000_tlb_load_handler(void) build_r3000_tlbchange_handler_head(&p, K0, K1); build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); + i_nop(&p); /* load delay */ build_make_valid(&p, &r, K0, K1); - build_r3000_pte_reload(&p, K1); - build_r3000_tlb_write(&p, &l, &r, K0); - build_r3000_tlbchange_handler_tail(&p, K0); + build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); l_nopage_tlbl(&l, p); i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); @@ -1506,13 +1549,10 @@ static void __init build_r3000_tlb_load_handler(void) { int i; - for (i = 0; i < FASTPATH_SIZE; i++) + for (i = 0; i < (p - handle_tlbl); i++) printk("%08x\n", handle_tlbl[i]); } #endif - - flush_icache_range((unsigned long)handle_tlbl, - (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32)); } static void __init build_r3000_tlb_store_handler(void) @@ -1527,10 +1567,9 @@ static void __init build_r3000_tlb_store_handler(void) build_r3000_tlbchange_handler_head(&p, K0, K1); build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); + i_nop(&p); /* load delay */ build_make_write(&p, &r, K0, K1); - build_r3000_pte_reload(&p, K1); - build_r3000_tlb_write(&p, &l, &r, K0); - build_r3000_tlbchange_handler_tail(&p, K0); + build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); l_nopage_tlbs(&l, p); i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); @@ -1547,13 +1586,10 @@ static void __init build_r3000_tlb_store_handler(void) { int i; - for (i = 0; i < FASTPATH_SIZE; i++) + for (i = 0; i < (p - handle_tlbs); i++) printk("%08x\n", handle_tlbs[i]); } #endif - - flush_icache_range((unsigned long)handle_tlbs, - (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32)); } static void __init build_r3000_tlb_modify_handler(void) @@ -1568,10 +1604,9 @@ static void __init build_r3000_tlb_modify_handler(void) build_r3000_tlbchange_handler_head(&p, K0, K1); build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); + i_nop(&p); /* load delay */ build_make_write(&p, &r, K0, K1); - build_r3000_pte_reload(&p, K1); - i_tlbwi(&p); - build_r3000_tlbchange_handler_tail(&p, K0); + build_r3000_pte_reload_tlbwi(&p, K0, K1); l_nopage_tlbm(&l, p); i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); @@ -1588,13 +1623,10 @@ static void __init build_r3000_tlb_modify_handler(void) { int i; - for (i = 0; i < FASTPATH_SIZE; i++) + for (i = 0; i < (p - handle_tlbm); i++) printk("%08x\n", handle_tlbm[i]); } #endif - - flush_icache_range((unsigned long)handle_tlbm, - (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32)); } /* @@ -1605,7 +1637,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct label **l, struct reloc **r, unsigned int pte, unsigned int ptr) { -#ifdef CONFIG_MIPS64 +#ifdef CONFIG_64BIT build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ #else build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ @@ -1620,7 +1652,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct label **l, #ifdef CONFIG_SMP l_smp_pgtable_change(l, *p); # endif - iPTE_LW(p, l, pte, 0, ptr); /* get even pte */ + iPTE_LW(p, l, pte, ptr); /* get even pte */ build_tlb_probe_entry(p); } @@ -1636,7 +1668,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct label **l, l_leave(l, *p); i_eret(p); /* return from trap */ -#ifdef CONFIG_MIPS64 +#ifdef CONFIG_64BIT build_get_pgd_vmalloc64(p, l, r, tmp, ptr); #endif } @@ -1680,13 +1712,10 @@ static void __init build_r4000_tlb_load_handler(void) { int i; - for (i = 0; i < FASTPATH_SIZE; i++) + for (i = 0; i < (p - handle_tlbl); i++) printk("%08x\n", handle_tlbl[i]); } #endif - - flush_icache_range((unsigned long)handle_tlbl, - (unsigned long)handle_tlbl + FASTPATH_SIZE * sizeof(u32)); } static void __init build_r4000_tlb_store_handler(void) @@ -1719,13 +1748,10 @@ static void __init build_r4000_tlb_store_handler(void) { int i; - for (i = 0; i < FASTPATH_SIZE; i++) + for (i = 0; i < (p - handle_tlbs); i++) printk("%08x\n", handle_tlbs[i]); } #endif - - flush_icache_range((unsigned long)handle_tlbs, - (unsigned long)handle_tlbs + FASTPATH_SIZE * sizeof(u32)); } static void __init build_r4000_tlb_modify_handler(void) @@ -1759,13 +1785,10 @@ static void __init build_r4000_tlb_modify_handler(void) { int i; - for (i = 0; i < FASTPATH_SIZE; i++) + for (i = 0; i < (p - handle_tlbm); i++) printk("%08x\n", handle_tlbm[i]); } #endif - - flush_icache_range((unsigned long)handle_tlbm, - (unsigned long)handle_tlbm + FASTPATH_SIZE * sizeof(u32)); } void __init build_tlb_refill_handler(void) @@ -1813,3 +1836,13 @@ void __init build_tlb_refill_handler(void) } } } + +void __init flush_tlb_handlers(void) +{ + flush_icache_range((unsigned long)handle_tlbl, + (unsigned long)handle_tlbl + sizeof(handle_tlbl)); + flush_icache_range((unsigned long)handle_tlbs, + (unsigned long)handle_tlbs + sizeof(handle_tlbs)); + flush_icache_range((unsigned long)handle_tlbm, + (unsigned long)handle_tlbm + sizeof(handle_tlbm)); +}