+# endif
+#endif
+}
+
+/*
+ * Check if PTE is present, if not then jump to LABEL. PTR points to
+ * the page table where this PTE is located, PTE will be re-loaded
+ * with it's original value.
+ */
+static void __init
+build_pte_present(u32 **p, struct label **l, struct reloc **r,
+ unsigned int pte, unsigned int ptr, enum label_id lid)
+{
+ i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+ i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+ il_bnez(p, r, pte, lid);
+ iPTE_LW(p, l, pte, ptr);
+}
+
+/* Make PTE valid, store result in PTR. */
+static void __init
+build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
+ unsigned int ptr)
+{
+ unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
+
+ iPTE_SW(p, r, pte, ptr, mode);
+}
+
+/*
+ * Check if PTE can be written to, if not branch to LABEL. Regardless
+ * restore PTE with value from PTR when done.
+ */
+static void __init
+build_pte_writable(u32 **p, struct label **l, struct reloc **r,
+ unsigned int pte, unsigned int ptr, enum label_id lid)
+{
+ i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
+ i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
+ il_bnez(p, r, pte, lid);
+ iPTE_LW(p, l, pte, ptr);
+}
+
+/* Make PTE writable, update software status bits as well, then store
+ * at PTR.
+ */
+static void __init
+build_make_write(u32 **p, struct reloc **r, unsigned int pte,
+ unsigned int ptr)
+{
+ unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
+ | _PAGE_DIRTY);
+
+ iPTE_SW(p, r, pte, ptr, mode);
+}
+
+/*
+ * Check if PTE can be modified, if not branch to LABEL. Regardless
+ * restore PTE with value from PTR when done.
+ */
+static void __init
+build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
+ unsigned int pte, unsigned int ptr, enum label_id lid)
+{
+ i_andi(p, pte, pte, _PAGE_WRITE);
+ il_beqz(p, r, pte, lid);
+ iPTE_LW(p, l, pte, ptr);
+}
+
+/*
+ * R3000 style TLB load/store/modify handlers.
+ */
+
+/*
+ * This places the pte into ENTRYLO0 and writes it with tlbwi.
+ * Then it returns.
+ */
+static void __init
+build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
+{
+ i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+ i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
+ i_tlbwi(p);
+ i_jr(p, tmp);
+ i_rfe(p); /* branch delay */
+}
+
+/*
+ * This places the pte into ENTRYLO0 and writes it with tlbwi
+ * or tlbwr as appropriate. This is because the index register
+ * may have the probe fail bit set as a result of a trap on a
+ * kseg2 access, i.e. without refill. Then it returns.
+ */
+static void __init
+build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r,
+ unsigned int pte, unsigned int tmp)
+{
+ i_mfc0(p, tmp, C0_INDEX);
+ i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+ il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
+ i_mfc0(p, tmp, C0_EPC); /* branch delay */
+ i_tlbwi(p); /* cp0 delay */
+ i_jr(p, tmp);
+ i_rfe(p); /* branch delay */
+ l_r3000_write_probe_fail(l, *p);
+ i_tlbwr(p); /* cp0 delay */
+ i_jr(p, tmp);
+ i_rfe(p); /* branch delay */
+}
+
+static void __init
+build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
+ unsigned int ptr)
+{
+ long pgdc = (long)pgd_current;
+
+ i_mfc0(p, pte, C0_BADVADDR);
+ i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */
+ i_lw(p, ptr, rel_lo(pgdc), ptr);
+ i_srl(p, pte, pte, 22); /* load delay */
+ i_sll(p, pte, pte, 2);
+ i_addu(p, ptr, ptr, pte);
+ i_mfc0(p, pte, C0_CONTEXT);
+ i_lw(p, ptr, 0, ptr); /* cp0 delay */
+ i_andi(p, pte, pte, 0xffc); /* load delay */
+ i_addu(p, ptr, ptr, pte);
+ i_lw(p, pte, 0, ptr);
+ i_tlbp(p); /* load delay */
+}
+
+static void __init build_r3000_tlb_load_handler(void)
+{
+ u32 *p = handle_tlbl;
+ struct label *l = labels;
+ struct reloc *r = relocs;
+ int i;
+
+ memset(handle_tlbl, 0, sizeof(handle_tlbl));
+ memset(labels, 0, sizeof(labels));
+ memset(relocs, 0, sizeof(relocs));
+
+ build_r3000_tlbchange_handler_head(&p, K0, K1);
+ build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
+ i_nop(&p); /* load delay */
+ build_make_valid(&p, &r, K0, K1);
+ build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
+
+ l_nopage_tlbl(&l, p);
+ i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
+ i_nop(&p);
+
+ if ((p - handle_tlbl) > FASTPATH_SIZE)
+ panic("TLB load handler fastpath space exceeded");
+
+ resolve_relocs(relocs, labels);
+ pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbl));
+
+ pr_debug("\t.set push\n");
+ pr_debug("\t.set noreorder\n");
+ for (i = 0; i < (p - handle_tlbl); i++)
+ pr_debug("\t.word 0x%08x\n", handle_tlbl[i]);
+ pr_debug("\t.set pop\n");
+}
+
+static void __init build_r3000_tlb_store_handler(void)
+{
+ u32 *p = handle_tlbs;
+ struct label *l = labels;
+ struct reloc *r = relocs;
+ int i;
+
+ memset(handle_tlbs, 0, sizeof(handle_tlbs));
+ memset(labels, 0, sizeof(labels));
+ memset(relocs, 0, sizeof(relocs));
+
+ build_r3000_tlbchange_handler_head(&p, K0, K1);
+ build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
+ i_nop(&p); /* load delay */
+ build_make_write(&p, &r, K0, K1);
+ build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
+
+ l_nopage_tlbs(&l, p);
+ i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+ i_nop(&p);
+
+ if ((p - handle_tlbs) > FASTPATH_SIZE)
+ panic("TLB store handler fastpath space exceeded");
+
+ resolve_relocs(relocs, labels);
+ pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbs));
+
+ pr_debug("\t.set push\n");
+ pr_debug("\t.set noreorder\n");
+ for (i = 0; i < (p - handle_tlbs); i++)
+ pr_debug("\t.word 0x%08x\n", handle_tlbs[i]);
+ pr_debug("\t.set pop\n");
+}
+
+static void __init build_r3000_tlb_modify_handler(void)
+{
+ u32 *p = handle_tlbm;
+ struct label *l = labels;
+ struct reloc *r = relocs;
+ int i;
+
+ memset(handle_tlbm, 0, sizeof(handle_tlbm));
+ memset(labels, 0, sizeof(labels));
+ memset(relocs, 0, sizeof(relocs));
+
+ build_r3000_tlbchange_handler_head(&p, K0, K1);
+ build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
+ i_nop(&p); /* load delay */
+ build_make_write(&p, &r, K0, K1);
+ build_r3000_pte_reload_tlbwi(&p, K0, K1);
+
+ l_nopage_tlbm(&l, p);
+ i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+ i_nop(&p);
+
+ if ((p - handle_tlbm) > FASTPATH_SIZE)
+ panic("TLB modify handler fastpath space exceeded");
+
+ resolve_relocs(relocs, labels);
+ pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbm));
+
+ pr_debug("\t.set push\n");
+ pr_debug("\t.set noreorder\n");
+ for (i = 0; i < (p - handle_tlbm); i++)
+ pr_debug("\t.word 0x%08x\n", handle_tlbm[i]);
+ pr_debug("\t.set pop\n");
+}
+
+/*
+ * R4000 style TLB load/store/modify handlers.
+ */
+static void __init
+build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
+ struct reloc **r, unsigned int pte,
+ unsigned int ptr)
+{
+#ifdef CONFIG_64BIT
+ build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
+#else
+ build_get_pgde32(p, pte, ptr); /* get pgd in ptr */