4 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
5 * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com>
7 * Lots of stuff stolen from arch/alpha/kernel/smp.c
9 * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized
10 * the existing code (on the lines of x86 port).
11 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy
12 * calibration on each CPU.
13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id
14 * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor
15 * & cpu_online_map now gets done here (instead of setup.c)
16 * 99/10/05 davidm Update to bring it in sync with new command-line processing
18 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and
19 * smp_call_function_single to resend IPI on timeouts
21 #include <linux/module.h>
22 #include <linux/kernel.h>
23 #include <linux/sched.h>
24 #include <linux/init.h>
25 #include <linux/interrupt.h>
26 #include <linux/smp.h>
27 #include <linux/kernel_stat.h>
29 #include <linux/cache.h>
30 #include <linux/delay.h>
31 #include <linux/cache.h>
32 #include <linux/efi.h>
34 #include <asm/atomic.h>
35 #include <asm/bitops.h>
36 #include <asm/current.h>
37 #include <asm/delay.h>
38 #include <asm/machvec.h>
42 #include <asm/pgalloc.h>
43 #include <asm/pgtable.h>
44 #include <asm/processor.h>
45 #include <asm/ptrace.h>
47 #include <asm/system.h>
48 #include <asm/tlbflush.h>
49 #include <asm/unistd.h>
53 * Structure and data for smp_call_function(). This is designed to minimise static memory
54 * requirements. It also looks cleaner.
56 static spinlock_t call_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
58 struct call_data_struct {
59 void (*func) (void *info);
66 static volatile struct call_data_struct *call_data;
68 #define IPI_CALL_FUNC 0
69 #define IPI_CPU_STOP 1
71 /* This needs to be cacheline aligned because it is written to by *other* CPUs. */
72 static DEFINE_PER_CPU(u64, ipi_operation) ____cacheline_aligned;
77 extern void cpu_halt (void);
81 cpu_clear(smp_processor_id(), cpu_online_map);
88 handle_IPI (int irq, void *dev_id, struct pt_regs *regs)
90 int this_cpu = get_cpu();
91 unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation);
94 mb(); /* Order interrupt and bit testing. */
95 while ((ops = xchg(pending_ipis, 0)) != 0) {
96 mb(); /* Order bit clearing and data access. */
101 ops &= ~(1 << which);
106 struct call_data_struct *data;
107 void (*func)(void *info);
111 /* release the 'pointer lock' */
112 data = (struct call_data_struct *) call_data;
118 atomic_inc(&data->started);
120 * At this point the structure may be gone unless
125 /* Notify the sending CPU that the task is done. */
128 atomic_inc(&data->finished);
137 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which);
141 mb(); /* Order data access and bit testing. */
148 * Called with preeemption disabled.
151 send_IPI_single (int dest_cpu, int op)
153 set_bit(op, &per_cpu(ipi_operation, dest_cpu));
154 platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0);
158 * Called with preeemption disabled.
161 send_IPI_allbutself (int op)
165 for (i = 0; i < NR_CPUS; i++) {
166 if (cpu_online(i) && i != smp_processor_id())
167 send_IPI_single(i, op);
172 * Called with preeemption disabled.
175 send_IPI_all (int op)
179 for (i = 0; i < NR_CPUS; i++)
181 send_IPI_single(i, op);
185 * Called with preeemption disabled.
188 send_IPI_self (int op)
190 send_IPI_single(smp_processor_id(), op);
194 * Called with preeemption disabled.
197 smp_send_reschedule (int cpu)
199 platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
203 smp_flush_tlb_all (void)
205 on_each_cpu((void (*)(void *))local_flush_tlb_all, 0, 1, 1);
207 EXPORT_SYMBOL(smp_flush_tlb_all);
210 smp_flush_tlb_mm (struct mm_struct *mm)
212 /* this happens for the common case of a single-threaded fork(): */
213 if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
215 local_finish_flush_tlb_mm(mm);
220 * We could optimize this further by using mm->cpu_vm_mask to track which CPUs
221 * have been running in the address space. It's not clear that this is worth the
222 * trouble though: to avoid races, we have to raise the IPI on the target CPU
223 * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is
226 on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1);
230 * Run a function on another CPU
231 * <func> The function to run. This must be fast and non-blocking.
232 * <info> An arbitrary pointer to pass to the function.
233 * <nonatomic> Currently unused.
234 * <wait> If true, wait until function has completed on other CPUs.
235 * [RETURNS] 0 on success, else a negative status code.
237 * Does not return until the remote CPU is nearly ready to execute <func>
238 * or is or has executed.
242 smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int nonatomic,
245 struct call_data_struct data;
247 int me = get_cpu(); /* prevent preemption and reschedule on another processor */
250 printk("%s: trying to call self\n", __FUNCTION__);
257 atomic_set(&data.started, 0);
260 atomic_set(&data.finished, 0);
262 spin_lock_bh(&call_lock);
265 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */
266 send_IPI_single(cpuid, IPI_CALL_FUNC);
268 /* Wait for response */
269 while (atomic_read(&data.started) != cpus)
273 while (atomic_read(&data.finished) != cpus)
277 spin_unlock_bh(&call_lock);
281 EXPORT_SYMBOL(smp_call_function_single);
284 * this function sends a 'generic call function' IPI to all other CPUs
289 * [SUMMARY] Run a function on all other CPUs.
290 * <func> The function to run. This must be fast and non-blocking.
291 * <info> An arbitrary pointer to pass to the function.
292 * <nonatomic> currently unused.
293 * <wait> If true, wait (atomically) until function has completed on other CPUs.
294 * [RETURNS] 0 on success, else a negative status code.
296 * Does not return until remote CPUs are nearly ready to execute <func> or are or have
299 * You must not call this function with disabled interrupts or from a
300 * hardware interrupt handler or from a bottom half handler.
303 smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait)
305 struct call_data_struct data;
306 int cpus = num_online_cpus()-1;
313 atomic_set(&data.started, 0);
316 atomic_set(&data.finished, 0);
318 spin_lock(&call_lock);
321 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */
322 send_IPI_allbutself(IPI_CALL_FUNC);
324 /* Wait for response */
325 while (atomic_read(&data.started) != cpus)
329 while (atomic_read(&data.finished) != cpus)
333 spin_unlock(&call_lock);
336 EXPORT_SYMBOL(smp_call_function);
339 * this function calls the 'stop' function on all other CPUs in the system.
344 send_IPI_allbutself(IPI_CPU_STOP);
348 setup_profiling_timer (unsigned int multiplier)