-/* Multi-cpu list version. */
-static void hypervisor_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask)
-{
- struct trap_per_cpu *tb;
- u16 *cpu_list;
- u64 *mondo;
- cpumask_t error_mask;
- unsigned long flags, status;
- int cnt, retries, this_cpu, prev_sent, i;
-
- /* We have to do this whole thing with interrupts fully disabled.
- * Otherwise if we send an xcall from interrupt context it will
- * corrupt both our mondo block and cpu list state.
- *
- * One consequence of this is that we cannot use timeout mechanisms
- * that depend upon interrupts being delivered locally. So, for
- * example, we cannot sample jiffies and expect it to advance.
- *
- * Fortunately, udelay() uses %stick/%tick so we can use that.
- */
- local_irq_save(flags);
-
- this_cpu = smp_processor_id();
- tb = &trap_block[this_cpu];
-
- mondo = __va(tb->cpu_mondo_block_pa);
- mondo[0] = data0;
- mondo[1] = data1;
- mondo[2] = data2;
- wmb();
-
- cpu_list = __va(tb->cpu_list_pa);
-
- /* Setup the initial cpu list. */
- cnt = 0;
- for_each_cpu_mask(i, mask)
- cpu_list[cnt++] = i;
-
- cpus_clear(error_mask);
- retries = 0;
- prev_sent = 0;
- do {
- int forward_progress, n_sent;
-
- status = sun4v_cpu_mondo_send(cnt,
- tb->cpu_list_pa,
- tb->cpu_mondo_block_pa);
-
- /* HV_EOK means all cpus received the xcall, we're done. */
- if (likely(status == HV_EOK))
- break;
-
- /* First, see if we made any forward progress.
- *
- * The hypervisor indicates successful sends by setting
- * cpu list entries to the value 0xffff.
- */
- n_sent = 0;
- for (i = 0; i < cnt; i++) {
- if (likely(cpu_list[i] == 0xffff))
- n_sent++;
- }
-
- forward_progress = 0;
- if (n_sent > prev_sent)
- forward_progress = 1;
-
- prev_sent = n_sent;
-
- /* If we get a HV_ECPUERROR, then one or more of the cpus
- * in the list are in error state. Use the cpu_state()
- * hypervisor call to find out which cpus are in error state.
- */
- if (unlikely(status == HV_ECPUERROR)) {
- for (i = 0; i < cnt; i++) {
- long err;
- u16 cpu;
-
- cpu = cpu_list[i];
- if (cpu == 0xffff)
- continue;
-
- err = sun4v_cpu_state(cpu);
- if (err >= 0 &&
- err == HV_CPU_STATE_ERROR) {
- cpu_list[i] = 0xffff;
- cpu_set(cpu, error_mask);
- }
- }
- } else if (unlikely(status != HV_EWOULDBLOCK))
- goto fatal_mondo_error;
-
- /* Don't bother rewriting the CPU list, just leave the
- * 0xffff and non-0xffff entries in there and the
- * hypervisor will do the right thing.
- *
- * Only advance timeout state if we didn't make any
- * forward progress.
- */
- if (unlikely(!forward_progress)) {
- if (unlikely(++retries > 10000))
- goto fatal_mondo_timeout;
-
- /* Delay a little bit to let other cpus catch up
- * on their cpu mondo queue work.
- */
- udelay(2 * cnt);
- }
- } while (1);
-
- local_irq_restore(flags);
-
- if (unlikely(!cpus_empty(error_mask)))
- goto fatal_mondo_cpu_error;
-
- return;
-
-fatal_mondo_cpu_error:
- printk(KERN_CRIT "CPU[%d]: SUN4V mondo cpu error, some target cpus "
- "were in error state\n",
- this_cpu);
- printk(KERN_CRIT "CPU[%d]: Error mask [ ", this_cpu);
- for_each_cpu_mask(i, error_mask)
- printk("%d ", i);
- printk("]\n");
- return;
-
-fatal_mondo_timeout:
- local_irq_restore(flags);
- printk(KERN_CRIT "CPU[%d]: SUN4V mondo timeout, no forward "
- " progress after %d retries.\n",
- this_cpu, retries);
- goto dump_cpu_list_and_out;
-
-fatal_mondo_error:
- local_irq_restore(flags);
- printk(KERN_CRIT "CPU[%d]: Unexpected SUN4V mondo error %lu\n",
- this_cpu, status);
- printk(KERN_CRIT "CPU[%d]: Args were cnt(%d) cpulist_pa(%lx) "
- "mondo_block_pa(%lx)\n",
- this_cpu, cnt, tb->cpu_list_pa, tb->cpu_mondo_block_pa);
-
-dump_cpu_list_and_out:
- printk(KERN_CRIT "CPU[%d]: CPU list [ ", this_cpu);
- for (i = 0; i < cnt; i++)
- printk("%u ", cpu_list[i]);
- printk("]\n");
-}
-