arch/ia64/sn/kernel/sn2/sn2_smp.c

   1 /*
   2  * SN2 Platform specific SMP Support
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 2000-2004 Silicon Graphics, Inc. All rights reserved.
   9  */
  10
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/threads.h>
  15 #include <linux/sched.h>
  16 #include <linux/smp.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/irq.h>
  19 #include <linux/mmzone.h>
  20 #include <linux/module.h>
  21
  22 #include <asm/processor.h>
  23 #include <asm/irq.h>
  24 #include <asm/sn/sgi.h>
  25 #include <asm/sal.h>
  26 #include <asm/system.h>
  27 #include <asm/delay.h>
  28 #include <asm/io.h>
  29 #include <asm/smp.h>
  30 #include <asm/numa.h>
  31 #include <asm/bitops.h>
  32 #include <asm/hw_irq.h>
  33 #include <asm/current.h>
  34 #include <asm/sn/sn_cpuid.h>
  35 #include <asm/sn/addrs.h>
  36 #include <asm/sn/sn2/shub_mmr.h>
  37 #include <asm/sn/nodepda.h>
  38 #include <asm/sn/rw_mmr.h>
  39
  40 void sn2_ptc_deadlock_recovery(unsigned long data0, unsigned long data1);
  41
  42
  43 static spinlock_t sn2_global_ptc_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
  44
  45 static unsigned long sn2_ptc_deadlock_count;
  46
  47
  48 static inline unsigned long
  49 wait_piowc(void)
  50 {
  51         volatile unsigned long *piows;
  52         unsigned long   ws;
  53
  54         piows = pda->pio_write_status_addr;
  55         do {
  56                 ia64_mfa();
  57         } while (((ws = *piows) & SH_PIO_WRITE_STATUS_0_PENDING_WRITE_COUNT_MASK) !=
  58                         SH_PIO_WRITE_STATUS_0_PENDING_WRITE_COUNT_MASK);
  59         return ws;
  60 }
  61
  62
  63
  64 /**
  65  * sn2_global_tlb_purge - globally purge translation cache of virtual address range
  66  * @start: start of virtual address range
  67  * @end: end of virtual address range
  68  * @nbits: specifies number of bytes to purge per instruction (num = 1<<(nbits & 0xfc))
  69  *
  70  * Purges the translation caches of all processors of the given virtual address
  71  * range.
  72  *
  73  * Note:
  74  *      - cpu_vm_mask is a bit mask that indicates which cpus have loaded the context.
  75  *      - cpu_vm_mask is converted into a nodemask of the nodes containing the
  76  *        cpus in cpu_vm_mask.
  77  *      - if only one bit is set in cpu_vm_mask & it is the current cpu,
  78  *        then only the local TLB needs to be flushed. This flushing can be done
  79  *        using ptc.l. This is the common case & avoids the global spinlock.
  80  *      - if multiple cpus have loaded the context, then flushing has to be
  81  *        done with ptc.g/MMRs under protection of the global ptc_lock.
  82  */
  83
  84 void
  85 sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbits)
  86 {
  87         int                     i, cnode, mynasid, cpu, lcpu=0, nasid, flushed=0;
  88         volatile unsigned       long    *ptc0, *ptc1;
  89         unsigned long           flags=0, data0, data1;
  90         struct mm_struct        *mm=current->active_mm;
  91         short                   nasids[NR_NODES], nix;
  92         DECLARE_BITMAP(nodes_flushed, NR_NODES);
  93
  94         bitmap_zero(nodes_flushed, NR_NODES);
  95
  96         i = 0;
  97
  98         for_each_cpu_mask(cpu, mm->cpu_vm_mask) {
  99                 cnode = cpu_to_node(cpu);
 100                 __set_bit(cnode, nodes_flushed);
 101                 lcpu = cpu;
 102                 i++;
 103         }
 104
 105         preempt_disable();
 106
 107         if (likely(i == 1 && lcpu == smp_processor_id())) {
 108                 do {
 109                         ia64_ptcl(start, nbits<<2);
 110                         start += (1UL << nbits);
 111                 } while (start < end);
 112                 ia64_srlz_i();
 113                 preempt_enable();
 114                 return;
 115         }
 116
 117         nix = 0;
 118         for (cnode=find_first_bit(&nodes_flushed, NR_NODES); cnode < NR_NODES;
 119                         cnode=find_next_bit(&nodes_flushed, NR_NODES, ++cnode))
 120                 nasids[nix++] = cnodeid_to_nasid(cnode);
 121
 122
 123         data0 = (1UL<<SH_PTC_0_A_SHFT) |
 124                 (nbits<<SH_PTC_0_PS_SHFT) |
 125                 ((ia64_get_rr(start)>>8)<<SH_PTC_0_RID_SHFT) |
 126                 (1UL<<SH_PTC_0_START_SHFT);
 127
 128         ptc0 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_0);
 129         ptc1 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_1);
 130
 131
 132         mynasid = smp_physical_node_id();
 133
 134         spin_lock_irqsave(&sn2_global_ptc_lock, flags);
 135
 136         do {
 137                 data1 = start | (1UL<<SH_PTC_1_START_SHFT);
 138                 for (i=0; i<nix; i++) {
 139                         nasid = nasids[i];
 140                         if (likely(nasid == mynasid)) {
 141                                 ia64_ptcga(start, nbits<<2);
 142                                 ia64_srlz_i();
 143                         } else {
 144                                 ptc0 = CHANGE_NASID(nasid, ptc0);
 145                                 ptc1 = CHANGE_NASID(nasid, ptc1);
 146                                 pio_atomic_phys_write_mmrs(ptc0, data0, ptc1, data1);
 147                                 flushed = 1;
 148                         }
 149                 }
 150
 151                 if (flushed && (wait_piowc() & SH_PIO_WRITE_STATUS_0_WRITE_DEADLOCK_MASK)) {
 152                         sn2_ptc_deadlock_recovery(data0, data1);
 153                 }
 154
 155                 start += (1UL << nbits);
 156
 157         } while (start < end);
 158
 159         spin_unlock_irqrestore(&sn2_global_ptc_lock, flags);
 160
 161         preempt_enable();
 162 }
 163
 164 /*
 165  * sn2_ptc_deadlock_recovery
 166  *
 167  * Recover from PTC deadlocks conditions. Recovery requires stepping thru each
 168  * TLB flush transaction.  The recovery sequence is somewhat tricky & is
 169  * coded in assembly language.
 170  */
 171 void
 172 sn2_ptc_deadlock_recovery(unsigned long data0, unsigned long data1)
 173 {
 174         extern void sn2_ptc_deadlock_recovery_core(long*, long, long*, long, long*);
 175         int     cnode, mycnode, nasid;
 176         long    *ptc0, *ptc1, *piows;
 177
 178         sn2_ptc_deadlock_count++;
 179
 180         ptc0 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_0);
 181         ptc1 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_1);
 182         piows = (long*)pda->pio_write_status_addr;
 183
 184         mycnode = numa_node_id();
 185
 186         for (cnode = 0; cnode < numnodes; cnode++) {
 187                 if (is_headless_node(cnode) || cnode == mycnode)
 188                         continue;
 189                 nasid = cnodeid_to_nasid(cnode);
 190                 ptc0 = CHANGE_NASID(nasid, ptc0);
 191                 ptc1 = CHANGE_NASID(nasid, ptc1);
 192                 sn2_ptc_deadlock_recovery_core(ptc0, data0, ptc1, data1, piows);
 193         }
 194 }
 195
 196 /**
 197  * sn_send_IPI_phys - send an IPI to a Nasid and slice
 198  * @physid: physical cpuid to receive the interrupt.
 199  * @vector: command to send
 200  * @delivery_mode: delivery mechanism
 201  *
 202  * Sends an IPI (interprocessor interrupt) to the processor specified by
 203  * @physid
 204  *
 205  * @delivery_mode can be one of the following
 206  *
 207  * %IA64_IPI_DM_INT - pend an interrupt
 208  * %IA64_IPI_DM_PMI - pend a PMI
 209  * %IA64_IPI_DM_NMI - pend an NMI
 210  * %IA64_IPI_DM_INIT - pend an INIT interrupt
 211  */
 212 void
 213 sn_send_IPI_phys(long physid, int vector, int delivery_mode)
 214 {
 215         long            nasid, slice, val;
 216         unsigned long   flags=0;
 217         volatile long   *p;
 218
 219         nasid = cpu_physical_id_to_nasid(physid);
 220         slice = cpu_physical_id_to_slice(physid);
 221
 222         p = (long*)GLOBAL_MMR_PHYS_ADDR(nasid, SH_IPI_INT);
 223         val =   (1UL<<SH_IPI_INT_SEND_SHFT) |
 224                 (physid<<SH_IPI_INT_PID_SHFT) |
 225                 ((long)delivery_mode<<SH_IPI_INT_TYPE_SHFT) |
 226                 ((long)vector<<SH_IPI_INT_IDX_SHFT) |
 227                 (0x000feeUL<<SH_IPI_INT_BASE_SHFT);
 228
 229         mb();
 230         if (enable_shub_wars_1_1() ) {
 231                 spin_lock_irqsave(&sn2_global_ptc_lock, flags);
 232         }
 233         pio_phys_write_mmr(p, val);
 234         if (enable_shub_wars_1_1() ) {
 235                 wait_piowc();
 236                 spin_unlock_irqrestore(&sn2_global_ptc_lock, flags);
 237         }
 238
 239 }
 240 EXPORT_SYMBOL(sn_send_IPI_phys);
 241
 242 /**
 243  * sn2_send_IPI - send an IPI to a processor
 244  * @cpuid: target of the IPI
 245  * @vector: command to send
 246  * @delivery_mode: delivery mechanism
 247  * @redirect: redirect the IPI?
 248  *
 249  * Sends an IPI (InterProcessor Interrupt) to the processor specified by
 250  * @cpuid.  @vector specifies the command to send, while @delivery_mode can
 251  * be one of the following
 252  *
 253  * %IA64_IPI_DM_INT - pend an interrupt
 254  * %IA64_IPI_DM_PMI - pend a PMI
 255  * %IA64_IPI_DM_NMI - pend an NMI
 256  * %IA64_IPI_DM_INIT - pend an INIT interrupt
 257  */
 258 void
 259 sn2_send_IPI(int cpuid, int vector, int delivery_mode, int redirect)
 260 {
 261         long            physid;
 262
 263         physid = cpu_physical_id(cpuid);
 264
 265         sn_send_IPI_phys(physid, vector, delivery_mode);
 266 }