arch/mips/mm/tlb-sb1.c

   1 /*
   2  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
   3  * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org)
   4  * Copyright (C) 2000, 2001, 2002, 2003 Broadcom Corporation
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19  */
  20 #include <linux/config.h>
  21 #include <linux/init.h>
  22 #include <asm/mmu_context.h>
  23 #include <asm/bootinfo.h>
  24 #include <asm/cpu.h>
  25
  26 #ifdef CONFIG_MIPS32
  27 extern void except_vec0_sb1(void);
  28 extern void except_vec1_generic(void);
  29 #endif
  30 #ifdef CONFIG_MIPS64
  31 extern void except_vec0_generic(void);
  32 extern void except_vec1_sb1(void);
  33 #endif
  34
  35 #define UNIQUE_ENTRYHI(idx) (KSEG0 + ((idx) << (PAGE_SHIFT + 1)))
  36
  37 /* Dump the current entry* and pagemask registers */
  38 static inline void dump_cur_tlb_regs(void)
  39 {
  40         unsigned int entryhihi, entryhilo, entrylo0hi, entrylo0lo, entrylo1hi;
  41         unsigned int entrylo1lo, pagemask;
  42
  43         __asm__ __volatile__ (
  44                 ".set push             \n"
  45                 ".set noreorder        \n"
  46                 ".set mips64           \n"
  47                 ".set noat             \n"
  48                 "     tlbr             \n"
  49                 "     dmfc0  $1, $10   \n"
  50                 "     dsrl32 %0, $1, 0 \n"
  51                 "     sll    %1, $1, 0 \n"
  52                 "     dmfc0  $1, $2    \n"
  53                 "     dsrl32 %2, $1, 0 \n"
  54                 "     sll    %3, $1, 0 \n"
  55                 "     dmfc0  $1, $3    \n"
  56                 "     dsrl32 %4, $1, 0 \n"
  57                 "     sll    %5, $1, 0 \n"
  58                 "     mfc0   %6, $5    \n"
  59                 ".set pop              \n"
  60                 : "=r" (entryhihi), "=r" (entryhilo),
  61                   "=r" (entrylo0hi), "=r" (entrylo0lo),
  62                   "=r" (entrylo1hi), "=r" (entrylo1lo),
  63                   "=r" (pagemask));
  64
  65         printk("%08X%08X %08X%08X %08X%08X %08X",
  66                entryhihi, entryhilo,
  67                entrylo0hi, entrylo0lo,
  68                entrylo1hi, entrylo1lo,
  69                pagemask);
  70 }
  71
  72 void sb1_dump_tlb(void)
  73 {
  74         unsigned long old_ctx;
  75         unsigned long flags;
  76         int entry;
  77         local_irq_save(flags);
  78         old_ctx = read_c0_entryhi();
  79         printk("Current TLB registers state:\n"
  80                "      EntryHi       EntryLo0          EntryLo1     PageMask  Index\n"
  81                "--------------------------------------------------------------------\n");
  82         dump_cur_tlb_regs();
  83         printk(" %08X\n", read_c0_index());
  84         printk("\n\nFull TLB Dump:\n"
  85                "Idx      EntryHi       EntryLo0          EntryLo1     PageMask\n"
  86                "--------------------------------------------------------------\n");
  87         for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
  88                 write_c0_index(entry);
  89                 printk("\n%02i ", entry);
  90                 dump_cur_tlb_regs();
  91         }
  92         printk("\n");
  93         write_c0_entryhi(old_ctx);
  94         local_irq_restore(flags);
  95 }
  96
  97 void local_flush_tlb_all(void)
  98 {
  99         unsigned long flags;
 100         unsigned long old_ctx;
 101         int entry;
 102
 103         local_irq_save(flags);
 104         /* Save old context and create impossible VPN2 value */
 105         old_ctx = read_c0_entryhi() & ASID_MASK;
 106         write_c0_entrylo0(0);
 107         write_c0_entrylo1(0);
 108
 109         entry = read_c0_wired();
 110         while (entry < current_cpu_data.tlbsize) {
 111                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 112                 write_c0_index(entry);
 113                 tlb_write_indexed();
 114                 entry++;
 115         }
 116         write_c0_entryhi(old_ctx);
 117         local_irq_restore(flags);
 118 }
 119
 120
 121 /*
 122  * Use a bogus region of memory (starting at 0) to sanitize the TLB's.
 123  * Use increments of the maximum page size (16MB), and check for duplicate
 124  * entries before doing a given write.  Then, when we're safe from collisions
 125  * with the firmware, go back and give all the entries invalid addresses with
 126  * the normal flush routine.  Wired entries will be killed as well!
 127  */
 128 void sb1_sanitize_tlb(void)
 129 {
 130         int entry;
 131         long addr = 0;
 132
 133         long inc = 1<<24;  /* 16MB */
 134         /* Save old context and create impossible VPN2 value */
 135         write_c0_entrylo0(0);
 136         write_c0_entrylo1(0);
 137         for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
 138                 do {
 139                         addr += inc;
 140                         write_c0_entryhi(addr);
 141                         tlb_probe();
 142                 } while ((int)(read_c0_index()) >= 0);
 143                 write_c0_index(entry);
 144                 tlb_write_indexed();
 145         }
 146         /* Now that we know we're safe from collisions, we can safely flush
 147            the TLB with the "normal" routine. */
 148         local_flush_tlb_all();
 149 }
 150
 151 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 152         unsigned long end)
 153 {
 154         struct mm_struct *mm = vma->vm_mm;
 155         unsigned long flags;
 156         int cpu;
 157
 158         local_irq_save(flags);
 159         cpu = smp_processor_id();
 160         if (cpu_context(cpu, mm) != 0) {
 161                 int size;
 162                 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 163                 size = (size + 1) >> 1;
 164                 if (size <= (current_cpu_data.tlbsize/2)) {
 165                         int oldpid = read_c0_entryhi() & ASID_MASK;
 166                         int newpid = cpu_asid(cpu, mm);
 167
 168                         start &= (PAGE_MASK << 1);
 169                         end += ((PAGE_SIZE << 1) - 1);
 170                         end &= (PAGE_MASK << 1);
 171                         while (start < end) {
 172                                 int idx;
 173
 174                                 write_c0_entryhi(start | newpid);
 175                                 start += (PAGE_SIZE << 1);
 176                                 tlb_probe();
 177                                 idx = read_c0_index();
 178                                 write_c0_entrylo0(0);
 179                                 write_c0_entrylo1(0);
 180                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 181                                 if (idx < 0)
 182                                         continue;
 183                                 tlb_write_indexed();
 184                         }
 185                         write_c0_entryhi(oldpid);
 186                 } else {
 187                         drop_mmu_context(mm, cpu);
 188                 }
 189         }
 190         local_irq_restore(flags);
 191 }
 192
 193 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 194 {
 195         unsigned long flags;
 196         int size;
 197
 198         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 199         size = (size + 1) >> 1;
 200
 201         local_irq_save(flags);
 202         if (size <= (current_cpu_data.tlbsize/2)) {
 203                 int pid = read_c0_entryhi();
 204
 205                 start &= (PAGE_MASK << 1);
 206                 end += ((PAGE_SIZE << 1) - 1);
 207                 end &= (PAGE_MASK << 1);
 208
 209                 while (start < end) {
 210                         int idx;
 211
 212                         write_c0_entryhi(start);
 213                         start += (PAGE_SIZE << 1);
 214                         tlb_probe();
 215                         idx = read_c0_index();
 216                         write_c0_entrylo0(0);
 217                         write_c0_entrylo1(0);
 218                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 219                         if (idx < 0)
 220                                 continue;
 221                         tlb_write_indexed();
 222                 }
 223                 write_c0_entryhi(pid);
 224         } else {
 225                 local_flush_tlb_all();
 226         }
 227         local_irq_restore(flags);
 228 }
 229
 230 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 231 {
 232         unsigned long flags;
 233         int cpu = smp_processor_id();
 234
 235         local_irq_save(flags);
 236         if (cpu_context(cpu, vma->vm_mm) != 0) {
 237                 int oldpid, newpid, idx;
 238                 newpid = cpu_asid(cpu, vma->vm_mm);
 239                 page &= (PAGE_MASK << 1);
 240                 oldpid = read_c0_entryhi() & ASID_MASK;
 241                 write_c0_entryhi(page | newpid);
 242                 tlb_probe();
 243                 idx = read_c0_index();
 244                 write_c0_entrylo0(0);
 245                 write_c0_entrylo1(0);
 246                 if (idx < 0)
 247                         goto finish;
 248                 /* Make sure all entries differ. */
 249                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 250                 tlb_write_indexed();
 251         finish:
 252                 write_c0_entryhi(oldpid);
 253         }
 254         local_irq_restore(flags);
 255 }
 256
 257 /*
 258  * Remove one kernel space TLB entry.  This entry is assumed to be marked
 259  * global so we don't do the ASID thing.
 260  */
 261 void local_flush_tlb_one(unsigned long page)
 262 {
 263         unsigned long flags;
 264         int oldpid, idx;
 265
 266         page &= (PAGE_MASK << 1);
 267         oldpid = read_c0_entryhi() & ASID_MASK;
 268
 269         local_irq_save(flags);
 270         write_c0_entryhi(page);
 271         tlb_probe();
 272         idx = read_c0_index();
 273         if (idx >= 0) {
 274                 /* Make sure all entries differ. */
 275                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 276                 write_c0_entrylo0(0);
 277                 write_c0_entrylo1(0);
 278                 tlb_write_indexed();
 279         }
 280
 281         write_c0_entryhi(oldpid);
 282         local_irq_restore(flags);
 283 }
 284
 285 /* All entries common to a mm share an asid.  To effectively flush
 286    these entries, we just bump the asid. */
 287 void local_flush_tlb_mm(struct mm_struct *mm)
 288 {
 289         int cpu = smp_processor_id();
 290         if (cpu_context(cpu, mm) != 0) {
 291                 drop_mmu_context(mm, cpu);
 292         }
 293 }
 294
 295 /* Stolen from mips32 routines */
 296
 297 void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
 298 {
 299         unsigned long flags;
 300         pgd_t *pgdp;
 301         pmd_t *pmdp;
 302         pte_t *ptep;
 303         int idx, pid;
 304
 305         /*
 306          * Handle debugger faulting in for debugee.
 307          */
 308         if (current->active_mm != vma->vm_mm)
 309                 return;
 310
 311         local_irq_save(flags);
 312
 313         pid = read_c0_entryhi() & ASID_MASK;
 314         address &= (PAGE_MASK << 1);
 315         write_c0_entryhi(address | (pid));
 316         pgdp = pgd_offset(vma->vm_mm, address);
 317         tlb_probe();
 318         pmdp = pmd_offset(pgdp, address);
 319         idx = read_c0_index();
 320         ptep = pte_offset_map(pmdp, address);
 321         write_c0_entrylo0(pte_val(*ptep++) >> 6);
 322         write_c0_entrylo1(pte_val(*ptep) >> 6);
 323         if (idx < 0) {
 324                 tlb_write_random();
 325         } else {
 326                 tlb_write_indexed();
 327         }
 328         local_irq_restore(flags);
 329 }
 330
 331 void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 332         unsigned long entryhi, unsigned long pagemask)
 333 {
 334         unsigned long flags;
 335         unsigned long wired;
 336         unsigned long old_pagemask;
 337         unsigned long old_ctx;
 338
 339         local_irq_save(flags);
 340         old_ctx = read_c0_entryhi() & 0xff;
 341         old_pagemask = read_c0_pagemask();
 342         wired = read_c0_wired();
 343         write_c0_wired(wired + 1);
 344         write_c0_index(wired);
 345
 346         write_c0_pagemask(pagemask);
 347         write_c0_entryhi(entryhi);
 348         write_c0_entrylo0(entrylo0);
 349         write_c0_entrylo1(entrylo1);
 350         tlb_write_indexed();
 351
 352         write_c0_entryhi(old_ctx);
 353         write_c0_pagemask(old_pagemask);
 354
 355         local_flush_tlb_all();
 356         local_irq_restore(flags);
 357 }
 358
 359 /*
 360  * This is called from loadmmu.c.  We have to set up all the
 361  * memory management function pointers, as well as initialize
 362  * the caches and tlbs
 363  */
 364 void tlb_init(void)
 365 {
 366         write_c0_pagemask(PM_DEFAULT_MASK);
 367         write_c0_wired(0);
 368
 369         /*
 370          * We don't know what state the firmware left the TLB's in, so this is
 371          * the ultra-conservative way to flush the TLB's and avoid machine
 372          * check exceptions due to duplicate TLB entries
 373          */
 374         sb1_sanitize_tlb();
 375
 376 #ifdef CONFIG_MIPS32
 377         memcpy((void *)KSEG0, &except_vec0_sb1, 0x80);
 378         memcpy((void *)(KSEG0 + 0x080), &except_vec1_generic, 0x80);
 379         flush_icache_range(KSEG0, KSEG0 + 0x100);
 380 #endif
 381 #ifdef CONFIG_MIPS64
 382         memcpy((void *)CKSEG0, &except_vec0_generic, 0x80);
 383         memcpy((void *)(CKSEG0 + 0x80), &except_vec1_sb1, 0x80);
 384         flush_icache_range(CKSEG0, CKSEG0 + 0x100);
 385 #endif
 386 }