arch/mips/mm/tlb-sb1.c

   1 /*
   2  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
   3  * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org)
   4  * Copyright (C) 2000, 2001, 2002, 2003 Broadcom Corporation
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19  */
  20 #include <linux/config.h>
  21 #include <linux/init.h>
  22 #include <asm/mmu_context.h>
  23 #include <asm/bootinfo.h>
  24 #include <asm/cpu.h>
  25
  26 extern void build_tlb_refill_handler(void);
  27
  28 #define UNIQUE_ENTRYHI(idx) (KSEG0 + ((idx) << (PAGE_SHIFT + 1)))
  29
  30 /* Dump the current entry* and pagemask registers */
  31 static inline void dump_cur_tlb_regs(void)
  32 {
  33         unsigned int entryhihi, entryhilo, entrylo0hi, entrylo0lo, entrylo1hi;
  34         unsigned int entrylo1lo, pagemask;
  35
  36         __asm__ __volatile__ (
  37                 ".set push             \n"
  38                 ".set noreorder        \n"
  39                 ".set mips64           \n"
  40                 ".set noat             \n"
  41                 "     tlbr             \n"
  42                 "     dmfc0  $1, $10   \n"
  43                 "     dsrl32 %0, $1, 0 \n"
  44                 "     sll    %1, $1, 0 \n"
  45                 "     dmfc0  $1, $2    \n"
  46                 "     dsrl32 %2, $1, 0 \n"
  47                 "     sll    %3, $1, 0 \n"
  48                 "     dmfc0  $1, $3    \n"
  49                 "     dsrl32 %4, $1, 0 \n"
  50                 "     sll    %5, $1, 0 \n"
  51                 "     mfc0   %6, $5    \n"
  52                 ".set pop              \n"
  53                 : "=r" (entryhihi), "=r" (entryhilo),
  54                   "=r" (entrylo0hi), "=r" (entrylo0lo),
  55                   "=r" (entrylo1hi), "=r" (entrylo1lo),
  56                   "=r" (pagemask));
  57
  58         printk("%08X%08X %08X%08X %08X%08X %08X",
  59                entryhihi, entryhilo,
  60                entrylo0hi, entrylo0lo,
  61                entrylo1hi, entrylo1lo,
  62                pagemask);
  63 }
  64
  65 void sb1_dump_tlb(void)
  66 {
  67         unsigned long old_ctx;
  68         unsigned long flags;
  69         int entry;
  70         local_irq_save(flags);
  71         old_ctx = read_c0_entryhi();
  72         printk("Current TLB registers state:\n"
  73                "      EntryHi       EntryLo0          EntryLo1     PageMask  Index\n"
  74                "--------------------------------------------------------------------\n");
  75         dump_cur_tlb_regs();
  76         printk(" %08X\n", read_c0_index());
  77         printk("\n\nFull TLB Dump:\n"
  78                "Idx      EntryHi       EntryLo0          EntryLo1     PageMask\n"
  79                "--------------------------------------------------------------\n");
  80         for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
  81                 write_c0_index(entry);
  82                 printk("\n%02i ", entry);
  83                 dump_cur_tlb_regs();
  84         }
  85         printk("\n");
  86         write_c0_entryhi(old_ctx);
  87         local_irq_restore(flags);
  88 }
  89
  90 void local_flush_tlb_all(void)
  91 {
  92         unsigned long flags;
  93         unsigned long old_ctx;
  94         int entry;
  95
  96         local_irq_save(flags);
  97         /* Save old context and create impossible VPN2 value */
  98         old_ctx = read_c0_entryhi() & ASID_MASK;
  99         write_c0_entrylo0(0);
 100         write_c0_entrylo1(0);
 101
 102         entry = read_c0_wired();
 103         while (entry < current_cpu_data.tlbsize) {
 104                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 105                 write_c0_index(entry);
 106                 tlb_write_indexed();
 107                 entry++;
 108         }
 109         write_c0_entryhi(old_ctx);
 110         local_irq_restore(flags);
 111 }
 112
 113
 114 /*
 115  * Use a bogus region of memory (starting at 0) to sanitize the TLB's.
 116  * Use increments of the maximum page size (16MB), and check for duplicate
 117  * entries before doing a given write.  Then, when we're safe from collisions
 118  * with the firmware, go back and give all the entries invalid addresses with
 119  * the normal flush routine.  Wired entries will be killed as well!
 120  */
 121 static void __init sb1_sanitize_tlb(void)
 122 {
 123         int entry;
 124         long addr = 0;
 125
 126         long inc = 1<<24;  /* 16MB */
 127         /* Save old context and create impossible VPN2 value */
 128         write_c0_entrylo0(0);
 129         write_c0_entrylo1(0);
 130         for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
 131                 do {
 132                         addr += inc;
 133                         write_c0_entryhi(addr);
 134                         tlb_probe();
 135                 } while ((int)(read_c0_index()) >= 0);
 136                 write_c0_index(entry);
 137                 tlb_write_indexed();
 138         }
 139         /* Now that we know we're safe from collisions, we can safely flush
 140            the TLB with the "normal" routine. */
 141         local_flush_tlb_all();
 142 }
 143
 144 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 145         unsigned long end)
 146 {
 147         struct mm_struct *mm = vma->vm_mm;
 148         unsigned long flags;
 149         int cpu;
 150
 151         local_irq_save(flags);
 152         cpu = smp_processor_id();
 153         if (cpu_context(cpu, mm) != 0) {
 154                 int size;
 155                 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 156                 size = (size + 1) >> 1;
 157                 if (size <= (current_cpu_data.tlbsize/2)) {
 158                         int oldpid = read_c0_entryhi() & ASID_MASK;
 159                         int newpid = cpu_asid(cpu, mm);
 160
 161                         start &= (PAGE_MASK << 1);
 162                         end += ((PAGE_SIZE << 1) - 1);
 163                         end &= (PAGE_MASK << 1);
 164                         while (start < end) {
 165                                 int idx;
 166
 167                                 write_c0_entryhi(start | newpid);
 168                                 start += (PAGE_SIZE << 1);
 169                                 tlb_probe();
 170                                 idx = read_c0_index();
 171                                 write_c0_entrylo0(0);
 172                                 write_c0_entrylo1(0);
 173                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 174                                 if (idx < 0)
 175                                         continue;
 176                                 tlb_write_indexed();
 177                         }
 178                         write_c0_entryhi(oldpid);
 179                 } else {
 180                         drop_mmu_context(mm, cpu);
 181                 }
 182         }
 183         local_irq_restore(flags);
 184 }
 185
 186 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 187 {
 188         unsigned long flags;
 189         int size;
 190
 191         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 192         size = (size + 1) >> 1;
 193
 194         local_irq_save(flags);
 195         if (size <= (current_cpu_data.tlbsize/2)) {
 196                 int pid = read_c0_entryhi();
 197
 198                 start &= (PAGE_MASK << 1);
 199                 end += ((PAGE_SIZE << 1) - 1);
 200                 end &= (PAGE_MASK << 1);
 201
 202                 while (start < end) {
 203                         int idx;
 204
 205                         write_c0_entryhi(start);
 206                         start += (PAGE_SIZE << 1);
 207                         tlb_probe();
 208                         idx = read_c0_index();
 209                         write_c0_entrylo0(0);
 210                         write_c0_entrylo1(0);
 211                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 212                         if (idx < 0)
 213                                 continue;
 214                         tlb_write_indexed();
 215                 }
 216                 write_c0_entryhi(pid);
 217         } else {
 218                 local_flush_tlb_all();
 219         }
 220         local_irq_restore(flags);
 221 }
 222
 223 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 224 {
 225         unsigned long flags;
 226         int cpu = smp_processor_id();
 227
 228         local_irq_save(flags);
 229         if (cpu_context(cpu, vma->vm_mm) != 0) {
 230                 int oldpid, newpid, idx;
 231                 newpid = cpu_asid(cpu, vma->vm_mm);
 232                 page &= (PAGE_MASK << 1);
 233                 oldpid = read_c0_entryhi() & ASID_MASK;
 234                 write_c0_entryhi(page | newpid);
 235                 tlb_probe();
 236                 idx = read_c0_index();
 237                 write_c0_entrylo0(0);
 238                 write_c0_entrylo1(0);
 239                 if (idx < 0)
 240                         goto finish;
 241                 /* Make sure all entries differ. */
 242                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 243                 tlb_write_indexed();
 244         finish:
 245                 write_c0_entryhi(oldpid);
 246         }
 247         local_irq_restore(flags);
 248 }
 249
 250 /*
 251  * Remove one kernel space TLB entry.  This entry is assumed to be marked
 252  * global so we don't do the ASID thing.
 253  */
 254 void local_flush_tlb_one(unsigned long page)
 255 {
 256         unsigned long flags;
 257         int oldpid, idx;
 258
 259         page &= (PAGE_MASK << 1);
 260         oldpid = read_c0_entryhi() & ASID_MASK;
 261
 262         local_irq_save(flags);
 263         write_c0_entryhi(page);
 264         tlb_probe();
 265         idx = read_c0_index();
 266         if (idx >= 0) {
 267                 /* Make sure all entries differ. */
 268                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 269                 write_c0_entrylo0(0);
 270                 write_c0_entrylo1(0);
 271                 tlb_write_indexed();
 272         }
 273
 274         write_c0_entryhi(oldpid);
 275         local_irq_restore(flags);
 276 }
 277
 278 /* All entries common to a mm share an asid.  To effectively flush
 279    these entries, we just bump the asid. */
 280 void local_flush_tlb_mm(struct mm_struct *mm)
 281 {
 282         int cpu;
 283
 284         preempt_disable();
 285
 286         cpu = smp_processor_id();
 287
 288         if (cpu_context(cpu, mm) != 0) {
 289                 drop_mmu_context(mm, cpu);
 290         }
 291
 292         preempt_enable();
 293 }
 294
 295 /* Stolen from mips32 routines */
 296
 297 void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
 298 {
 299         unsigned long flags;
 300         pgd_t *pgdp;
 301         pmd_t *pmdp;
 302         pte_t *ptep;
 303         int idx, pid;
 304
 305         /*
 306          * Handle debugger faulting in for debugee.
 307          */
 308         if (current->active_mm != vma->vm_mm)
 309                 return;
 310
 311         local_irq_save(flags);
 312
 313         pid = read_c0_entryhi() & ASID_MASK;
 314         address &= (PAGE_MASK << 1);
 315         write_c0_entryhi(address | (pid));
 316         pgdp = pgd_offset(vma->vm_mm, address);
 317         tlb_probe();
 318         pmdp = pmd_offset(pgdp, address);
 319         idx = read_c0_index();
 320         ptep = pte_offset_map(pmdp, address);
 321         write_c0_entrylo0(pte_val(*ptep++) >> 6);
 322         write_c0_entrylo1(pte_val(*ptep) >> 6);
 323         if (idx < 0) {
 324                 tlb_write_random();
 325         } else {
 326                 tlb_write_indexed();
 327         }
 328         local_irq_restore(flags);
 329 }
 330
 331 void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 332         unsigned long entryhi, unsigned long pagemask)
 333 {
 334         unsigned long flags;
 335         unsigned long wired;
 336         unsigned long old_pagemask;
 337         unsigned long old_ctx;
 338
 339         local_irq_save(flags);
 340         old_ctx = read_c0_entryhi() & 0xff;
 341         old_pagemask = read_c0_pagemask();
 342         wired = read_c0_wired();
 343         write_c0_wired(wired + 1);
 344         write_c0_index(wired);
 345
 346         write_c0_pagemask(pagemask);
 347         write_c0_entryhi(entryhi);
 348         write_c0_entrylo0(entrylo0);
 349         write_c0_entrylo1(entrylo1);
 350         tlb_write_indexed();
 351
 352         write_c0_entryhi(old_ctx);
 353         write_c0_pagemask(old_pagemask);
 354
 355         local_flush_tlb_all();
 356         local_irq_restore(flags);
 357 }
 358
 359 /*
 360  * This is called from loadmmu.c.  We have to set up all the
 361  * memory management function pointers, as well as initialize
 362  * the caches and tlbs
 363  */
 364 void tlb_init(void)
 365 {
 366         write_c0_pagemask(PM_DEFAULT_MASK);
 367         write_c0_wired(0);
 368
 369         /*
 370          * We don't know what state the firmware left the TLB's in, so this is
 371          * the ultra-conservative way to flush the TLB's and avoid machine
 372          * check exceptions due to duplicate TLB entries
 373          */
 374         sb1_sanitize_tlb();
 375
 376         build_tlb_refill_handler();
 377 }