X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fsparc64%2Fkernel%2Firq.c;h=11e645c9ec508e97b10a898b908c9bab75073f4f;hb=43bc926fffd92024b46cafaf7350d669ba9ca884;hp=4dcb8af94090cfc490da271877561d988b3a2aea;hpb=cee37fe97739d85991964371c1f3a745c00dd236;p=linux-2.6.git diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c index 4dcb8af94..11e645c9e 100644 --- a/arch/sparc64/kernel/irq.c +++ b/arch/sparc64/kernel/irq.c @@ -21,12 +21,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -37,6 +39,8 @@ #include #include #include +#include +#include #ifdef CONFIG_SMP static void distribute_irqs(void); @@ -70,31 +74,7 @@ struct irq_work_struct { struct irq_work_struct __irq_work[NR_CPUS]; #define irq_work(__cpu, __pil) &(__irq_work[(__cpu)].irq_worklists[(__pil)]) -#ifdef CONFIG_PCI -/* This is a table of physical addresses used to deal with IBF_DMA_SYNC. - * It is used for PCI only to synchronize DMA transfers with IRQ delivery - * for devices behind busses other than APB on Sabre systems. - * - * Currently these physical addresses are just config space accesses - * to the command register for that device. - */ -unsigned long pci_dma_wsync; -unsigned long dma_sync_reg_table[256]; -unsigned char dma_sync_reg_table_entry = 0; -#endif - -/* This is based upon code in the 32-bit Sparc kernel written mostly by - * David Redman (djhr@tadpole.co.uk). - */ -#define MAX_STATIC_ALLOC 4 -static struct irqaction static_irqaction[MAX_STATIC_ALLOC]; -static int static_irq_count; - -/* This is exported so that fast IRQ handlers can get at it... -DaveM */ -struct irqaction *irq_action[NR_IRQS+1] = { - NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL -}; +static struct irqaction *irq_action[NR_IRQS+1]; /* This only synchronizes entities which modify IRQ handler * state and some selected user-level spots that want to @@ -137,9 +117,7 @@ int show_interrupts(struct seq_file *p, void *v) #ifndef CONFIG_SMP seq_printf(p, "%10u ", kstat_irqs(i)); #else - for (j = 0; j < NR_CPUS; j++) { - if (!cpu_online(j)) - continue; + for_each_online_cpu(j) { seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]); } @@ -158,12 +136,48 @@ out_unlock: return 0; } +extern unsigned long real_hard_smp_processor_id(void); + +static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid) +{ + unsigned int tid; + + if (this_is_starfire) { + tid = starfire_translate(imap, cpuid); + tid <<= IMAP_TID_SHIFT; + tid &= IMAP_TID_UPA; + } else { + if (tlb_type == cheetah || tlb_type == cheetah_plus) { + unsigned long ver; + + __asm__ ("rdpr %%ver, %0" : "=r" (ver)); + if ((ver >> 32UL) == __JALAPENO_ID || + (ver >> 32UL) == __SERRANO_ID) { + tid = cpuid << IMAP_TID_SHIFT; + tid &= IMAP_TID_JBUS; + } else { + unsigned int a = cpuid & 0x1f; + unsigned int n = (cpuid >> 5) & 0x1f; + + tid = ((a << IMAP_AID_SHIFT) | + (n << IMAP_NID_SHIFT)); + tid &= (IMAP_AID_SAFARI | + IMAP_NID_SAFARI);; + } + } else { + tid = cpuid << IMAP_TID_SHIFT; + tid &= IMAP_TID_UPA; + } + } + + return tid; +} + /* Now these are always passed a true fully specified sun4u INO. */ void enable_irq(unsigned int irq) { struct ino_bucket *bucket = __bucket(irq); - unsigned long imap; - unsigned long tid; + unsigned long imap, cpuid; imap = bucket->imap; if (imap == 0UL) @@ -171,47 +185,38 @@ void enable_irq(unsigned int irq) preempt_disable(); - if (tlb_type == cheetah || tlb_type == cheetah_plus) { - unsigned long ver; - - __asm__ ("rdpr %%ver, %0" : "=r" (ver)); - if ((ver >> 32) == 0x003e0016) { - /* We set it to our JBUS ID. */ - __asm__ __volatile__("ldxa [%%g0] %1, %0" - : "=r" (tid) - : "i" (ASI_JBUS_CONFIG)); - tid = ((tid & (0x1fUL<<17)) << 9); - tid &= IMAP_TID_JBUS; - } else { - /* We set it to our Safari AID. */ - __asm__ __volatile__("ldxa [%%g0] %1, %0" - : "=r" (tid) - : "i" (ASI_SAFARI_CONFIG)); - tid = ((tid & (0x3ffUL<<17)) << 9); - tid &= IMAP_AID_SAFARI; - } - } else if (this_is_starfire == 0) { - /* We set it to our UPA MID. */ - __asm__ __volatile__("ldxa [%%g0] %1, %0" - : "=r" (tid) - : "i" (ASI_UPA_CONFIG)); - tid = ((tid & UPA_CONFIG_MID) << 9); - tid &= IMAP_TID_UPA; + /* This gets the physical processor ID, even on uniprocessor, + * so we can always program the interrupt target correctly. + */ + cpuid = real_hard_smp_processor_id(); + + if (tlb_type == hypervisor) { + unsigned int ino = __irq_ino(irq); + int err; + + err = sun4v_intr_settarget(ino, cpuid); + if (err != HV_EOK) + printk("sun4v_intr_settarget(%x,%lu): err(%d)\n", + ino, cpuid, err); + err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED); + if (err != HV_EOK) + printk("sun4v_intr_setenabled(%x): err(%d)\n", + ino, err); } else { - tid = (starfire_translate(imap, smp_processor_id()) << 26); - tid &= IMAP_TID_UPA; + unsigned int tid = sun4u_compute_tid(imap, cpuid); + + /* NOTE NOTE NOTE, IGN and INO are read-only, IGN is a product + * of this SYSIO's preconfigured IGN in the SYSIO Control + * Register, the hardware just mirrors that value here. + * However for Graphics and UPA Slave devices the full + * IMAP_INR field can be set by the programmer here. + * + * Things like FFB can now be handled via the new IRQ + * mechanism. + */ + upa_writel(tid | IMAP_VALID, imap); } - /* NOTE NOTE NOTE, IGN and INO are read-only, IGN is a product - * of this SYSIO's preconfigured IGN in the SYSIO Control - * Register, the hardware just mirrors that value here. - * However for Graphics and UPA Slave devices the full - * IMAP_INR field can be set by the programmer here. - * - * Things like FFB can now be handled via the new IRQ mechanism. - */ - upa_writel(tid | IMAP_VALID, imap); - preempt_enable(); } @@ -223,16 +228,26 @@ void disable_irq(unsigned int irq) imap = bucket->imap; if (imap != 0UL) { - u32 tmp; + if (tlb_type == hypervisor) { + unsigned int ino = __irq_ino(irq); + int err; + + err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); + if (err != HV_EOK) + printk("sun4v_intr_setenabled(%x): " + "err(%d)\n", ino, err); + } else { + u32 tmp; - /* NOTE: We do not want to futz with the IRQ clear registers - * and move the state to IDLE, the SCSI code does call - * disable_irq() to assure atomicity in the queue cmd - * SCSI adapter driver code. Thus we'd lose interrupts. - */ - tmp = upa_readl(imap); - tmp &= ~IMAP_VALID; - upa_writel(tmp, imap); + /* NOTE: We do not want to futz with the IRQ clear registers + * and move the state to IDLE, the SCSI code does call + * disable_irq() to assure atomicity in the queue cmd + * SCSI adapter driver code. Thus we'd lose interrupts. + */ + tmp = upa_readl(imap); + tmp &= ~IMAP_VALID; + upa_writel(tmp, imap); + } } } @@ -240,17 +255,22 @@ void disable_irq(unsigned int irq) * the CPU %tick register and not by some normal vectored interrupt * source. To handle this special case, we use this dummy INO bucket. */ +static struct irq_desc pil0_dummy_desc; static struct ino_bucket pil0_dummy_bucket = { - 0, /* irq_chain */ - 0, /* pil */ - 0, /* pending */ - 0, /* flags */ - 0, /* __unused */ - NULL, /* irq_info */ - 0UL, /* iclr */ - 0UL, /* imap */ + .irq_info = &pil0_dummy_desc, }; +static void build_irq_error(const char *msg, unsigned int ino, int pil, int inofixup, + unsigned long iclr, unsigned long imap, + struct ino_bucket *bucket) +{ + prom_printf("IRQ: INO %04x (%d:%016lx:%016lx) --> " + "(%d:%d:%016lx:%016lx), halting...\n", + ino, bucket->pil, bucket->iclr, bucket->imap, + pil, inofixup, iclr, imap); + prom_halt(); +} + unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long imap) { struct ino_bucket *bucket; @@ -265,6 +285,8 @@ unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long return __irq(&pil0_dummy_bucket); } + BUG_ON(tlb_type == hypervisor); + /* RULE: Both must be specified in all other cases. */ if (iclr == 0UL || imap == 0UL) { prom_printf("Invalid build_irq %d %d %016lx %016lx\n", @@ -279,27 +301,64 @@ unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long prom_halt(); } - /* Ok, looks good, set it up. Don't touch the irq_chain or - * the pending flag. - */ bucket = &ivector_table[ino]; - if ((bucket->flags & IBF_ACTIVE) || - (bucket->irq_info != NULL)) { - /* This is a gross fatal error if it happens here. */ - prom_printf("IRQ: Trying to reinit INO bucket, fatal error.\n"); - prom_printf("IRQ: Request INO %04x (%d:%d:%016lx:%016lx)\n", - ino, pil, inofixup, iclr, imap); - prom_printf("IRQ: Existing (%d:%016lx:%016lx)\n", - bucket->pil, bucket->iclr, bucket->imap); - prom_printf("IRQ: Cannot continue, halting...\n"); + if (bucket->flags & IBF_ACTIVE) + build_irq_error("IRQ: Trying to build active INO bucket.\n", + ino, pil, inofixup, iclr, imap, bucket); + + if (bucket->irq_info) { + if (bucket->imap != imap || bucket->iclr != iclr) + build_irq_error("IRQ: Trying to reinit INO bucket.\n", + ino, pil, inofixup, iclr, imap, bucket); + + goto out; + } + + bucket->irq_info = kzalloc(sizeof(struct irq_desc), GFP_ATOMIC); + if (!bucket->irq_info) { + prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n"); prom_halt(); } + + /* Ok, looks good, set it up. Don't touch the irq_chain or + * the pending flag. + */ bucket->imap = imap; bucket->iclr = iclr; bucket->pil = pil; bucket->flags = 0; - bucket->irq_info = NULL; +out: + return __irq(bucket); +} + +unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino, int pil, unsigned char flags) +{ + struct ino_bucket *bucket; + unsigned long sysino; + + sysino = sun4v_devino_to_sysino(devhandle, devino); + + bucket = &ivector_table[sysino]; + + /* Catch accidental accesses to these things. IMAP/ICLR handling + * is done by hypervisor calls on sun4v platforms, not by direct + * register accesses. + * + * But we need to make them look unique for the disable_irq() logic + * in free_irq(). + */ + bucket->imap = ~0UL - sysino; + bucket->iclr = ~0UL - sysino; + + bucket->pil = pil; + bucket->flags = flags; + + bucket->irq_info = kzalloc(sizeof(struct irq_desc), GFP_ATOMIC); + if (!bucket->irq_info) { + prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n"); + prom_halt(); + } return __irq(bucket); } @@ -318,26 +377,65 @@ static void atomic_bucket_insert(struct ino_bucket *bucket) __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate)); } +static int check_irq_sharing(int pil, unsigned long irqflags) +{ + struct irqaction *action, *tmp; + + action = *(irq_action + pil); + if (action) { + if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ)) { + for (tmp = action; tmp->next; tmp = tmp->next) + ; + } else { + return -EBUSY; + } + } + return 0; +} + +static void append_irq_action(int pil, struct irqaction *action) +{ + struct irqaction **pp = irq_action + pil; + + while (*pp) + pp = &((*pp)->next); + *pp = action; +} + +static struct irqaction *get_action_slot(struct ino_bucket *bucket) +{ + struct irq_desc *desc = bucket->irq_info; + int max_irq, i; + + max_irq = 1; + if (bucket->flags & IBF_PCI) + max_irq = MAX_IRQ_DESC_ACTION; + for (i = 0; i < max_irq; i++) { + struct irqaction *p = &desc->action[i]; + u32 mask = (1 << i); + + if (desc->action_active_mask & mask) + continue; + + desc->action_active_mask |= mask; + return p; + } + return NULL; +} + int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_regs *), unsigned long irqflags, const char *name, void *dev_id) { - struct irqaction *action, *tmp = NULL; + struct irqaction *action; struct ino_bucket *bucket = __bucket(irq); unsigned long flags; int pending = 0; - if ((bucket != &pil0_dummy_bucket) && - (bucket < &ivector_table[0] || - bucket >= &ivector_table[NUM_IVECS])) { - unsigned int *caller; - - __asm__ __volatile__("mov %%i7, %0" : "=r" (caller)); - printk(KERN_CRIT "request_irq: Old style IRQ registry attempt " - "from %p, irq %08x.\n", caller, irq); + if (unlikely(!handler)) return -EINVAL; - } - if (!handler) - return -EINVAL; + + if (unlikely(!bucket->irq_info)) + return -ENODEV; if ((bucket != &pil0_dummy_bucket) && (irqflags & SA_SAMPLE_RANDOM)) { /* @@ -355,93 +453,20 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_ spin_lock_irqsave(&irq_action_lock, flags); - action = *(bucket->pil + irq_action); - if (action) { - if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ)) - for (tmp = action; tmp->next; tmp = tmp->next) - ; - else { - spin_unlock_irqrestore(&irq_action_lock, flags); - return -EBUSY; - } - action = NULL; /* Or else! */ + if (check_irq_sharing(bucket->pil, irqflags)) { + spin_unlock_irqrestore(&irq_action_lock, flags); + return -EBUSY; } - /* If this is flagged as statically allocated then we use our - * private struct which is never freed. - */ - if (irqflags & SA_STATIC_ALLOC) { - if (static_irq_count < MAX_STATIC_ALLOC) - action = &static_irqaction[static_irq_count++]; - else - printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed " - "using kmalloc\n", irq, name); - } - if (action == NULL) - action = (struct irqaction *)kmalloc(sizeof(struct irqaction), - GFP_ATOMIC); - + action = get_action_slot(bucket); if (!action) { spin_unlock_irqrestore(&irq_action_lock, flags); return -ENOMEM; } - if (bucket == &pil0_dummy_bucket) { - bucket->irq_info = action; - bucket->flags |= IBF_ACTIVE; - } else { - if ((bucket->flags & IBF_ACTIVE) != 0) { - void *orig = bucket->irq_info; - void **vector = NULL; - - if ((bucket->flags & IBF_PCI) == 0) { - printk("IRQ: Trying to share non-PCI bucket.\n"); - goto free_and_ebusy; - } - if ((bucket->flags & IBF_MULTI) == 0) { - vector = kmalloc(sizeof(void *) * 4, GFP_ATOMIC); - if (vector == NULL) - goto free_and_enomem; - - /* We might have slept. */ - if ((bucket->flags & IBF_MULTI) != 0) { - int ent; - - kfree(vector); - vector = (void **)bucket->irq_info; - for(ent = 0; ent < 4; ent++) { - if (vector[ent] == NULL) { - vector[ent] = action; - break; - } - } - if (ent == 4) - goto free_and_ebusy; - } else { - vector[0] = orig; - vector[1] = action; - vector[2] = NULL; - vector[3] = NULL; - bucket->irq_info = vector; - bucket->flags |= IBF_MULTI; - } - } else { - int ent; - - vector = (void **)orig; - for (ent = 0; ent < 4; ent++) { - if (vector[ent] == NULL) { - vector[ent] = action; - break; - } - } - if (ent == 4) - goto free_and_ebusy; - } - } else { - bucket->irq_info = action; - bucket->flags |= IBF_ACTIVE; - } + bucket->flags |= IBF_ACTIVE; + pending = 0; + if (bucket != &pil0_dummy_bucket) { pending = bucket->pending; if (pending) bucket->pending = 0; @@ -455,10 +480,7 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_ put_ino_in_irqaction(action, irq); put_smpaff_in_irqaction(action, CPU_MASK_NONE); - if (tmp) - tmp->next = action; - else - *(bucket->pil + irq_action) = action; + append_irq_action(bucket->pil, action); enable_irq(irq); @@ -467,147 +489,104 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_ atomic_bucket_insert(bucket); set_softint(1 << bucket->pil); } + spin_unlock_irqrestore(&irq_action_lock, flags); - if ((bucket != &pil0_dummy_bucket) && (!(irqflags & SA_STATIC_ALLOC))) + + if (bucket != &pil0_dummy_bucket) register_irq_proc(__irq_ino(irq)); #ifdef CONFIG_SMP distribute_irqs(); #endif return 0; - -free_and_ebusy: - kfree(action); - spin_unlock_irqrestore(&irq_action_lock, flags); - return -EBUSY; - -free_and_enomem: - kfree(action); - spin_unlock_irqrestore(&irq_action_lock, flags); - return -ENOMEM; } EXPORT_SYMBOL(request_irq); -void free_irq(unsigned int irq, void *dev_id) +static struct irqaction *unlink_irq_action(unsigned int irq, void *dev_id) { - struct irqaction *action; - struct irqaction *tmp = NULL; - unsigned long flags; - struct ino_bucket *bucket = __bucket(irq), *bp; + struct ino_bucket *bucket = __bucket(irq); + struct irqaction *action, **pp; - if ((bucket != &pil0_dummy_bucket) && - (bucket < &ivector_table[0] || - bucket >= &ivector_table[NUM_IVECS])) { - unsigned int *caller; + pp = irq_action + bucket->pil; + action = *pp; + if (unlikely(!action)) + return NULL; - __asm__ __volatile__("mov %%i7, %0" : "=r" (caller)); - printk(KERN_CRIT "free_irq: Old style IRQ removal attempt " - "from %p, irq %08x.\n", caller, irq); - return; - } - - spin_lock_irqsave(&irq_action_lock, flags); - - action = *(bucket->pil + irq_action); - if (!action->handler) { + if (unlikely(!action->handler)) { printk("Freeing free IRQ %d\n", bucket->pil); - return; - } - if (dev_id) { - for ( ; action; action = action->next) { - if (action->dev_id == dev_id) - break; - tmp = action; - } - if (!action) { - printk("Trying to free free shared IRQ %d\n", bucket->pil); - spin_unlock_irqrestore(&irq_action_lock, flags); - return; - } - } else if (action->flags & SA_SHIRQ) { - printk("Trying to free shared IRQ %d with NULL device ID\n", bucket->pil); - spin_unlock_irqrestore(&irq_action_lock, flags); - return; + return NULL; } - if (action->flags & SA_STATIC_ALLOC) { - printk("Attempt to free statically allocated IRQ %d (%s)\n", - bucket->pil, action->name); - spin_unlock_irqrestore(&irq_action_lock, flags); - return; + while (action && action->dev_id != dev_id) { + pp = &action->next; + action = *pp; } - if (action && tmp) - tmp->next = action->next; - else - *(bucket->pil + irq_action) = action->next; + if (likely(action)) + *pp = action->next; + + return action; +} + +void free_irq(unsigned int irq, void *dev_id) +{ + struct irqaction *action; + struct ino_bucket *bucket; + unsigned long flags; + + spin_lock_irqsave(&irq_action_lock, flags); + + action = unlink_irq_action(irq, dev_id); spin_unlock_irqrestore(&irq_action_lock, flags); + if (unlikely(!action)) + return; + synchronize_irq(irq); spin_lock_irqsave(&irq_action_lock, flags); + bucket = __bucket(irq); if (bucket != &pil0_dummy_bucket) { - unsigned long imap = bucket->imap; - void **vector, *orig; - int ent; - - orig = bucket->irq_info; - vector = (void **)orig; - - if ((bucket->flags & IBF_MULTI) != 0) { - int other = 0; - void *orphan = NULL; - for (ent = 0; ent < 4; ent++) { - if (vector[ent] == action) - vector[ent] = NULL; - else if (vector[ent] != NULL) { - orphan = vector[ent]; - other++; - } - } + struct irq_desc *desc = bucket->irq_info; + int ent, i; - /* Only free when no other shared irq - * uses this bucket. - */ - if (other) { - if (other == 1) { - /* Convert back to non-shared bucket. */ - bucket->irq_info = orphan; - bucket->flags &= ~(IBF_MULTI); - kfree(vector); - } - goto out; + for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) { + struct irqaction *p = &desc->action[i]; + + if (p == action) { + desc->action_active_mask &= ~(1 << i); + break; } - } else { - bucket->irq_info = NULL; } - /* This unique interrupt source is now inactive. */ - bucket->flags &= ~IBF_ACTIVE; + if (!desc->action_active_mask) { + unsigned long imap = bucket->imap; - /* See if any other buckets share this bucket's IMAP - * and are still active. - */ - for (ent = 0; ent < NUM_IVECS; ent++) { - bp = &ivector_table[ent]; - if (bp != bucket && - bp->imap == imap && - (bp->flags & IBF_ACTIVE) != 0) - break; - } + /* This unique interrupt source is now inactive. */ + bucket->flags &= ~IBF_ACTIVE; - /* Only disable when no other sub-irq levels of - * the same IMAP are active. - */ - if (ent == NUM_IVECS) - disable_irq(irq); + /* See if any other buckets share this bucket's IMAP + * and are still active. + */ + for (ent = 0; ent < NUM_IVECS; ent++) { + struct ino_bucket *bp = &ivector_table[ent]; + if (bp != bucket && + bp->imap == imap && + (bp->flags & IBF_ACTIVE) != 0) + break; + } + + /* Only disable when no other sub-irq levels of + * the same IMAP are active. + */ + if (ent == NUM_IVECS) + disable_irq(irq); + } } -out: - kfree(action); spin_unlock_irqrestore(&irq_action_lock, flags); } @@ -646,99 +625,66 @@ void synchronize_irq(unsigned int irq) } #endif /* CONFIG_SMP */ -void catch_disabled_ivec(struct pt_regs *regs) +static void process_bucket(int irq, struct ino_bucket *bp, struct pt_regs *regs) { - int cpu = smp_processor_id(); - struct ino_bucket *bucket = __bucket(*irq_work(cpu, 0)); + struct irq_desc *desc = bp->irq_info; + unsigned char flags = bp->flags; + u32 action_mask, i; + int random; - /* We can actually see this on Ultra/PCI PCI cards, which are bridges - * to other devices. Here a single IMAP enabled potentially multiple - * unique interrupt sources (which each do have a unique ICLR register. - * - * So what we do is just register that the IVEC arrived, when registered - * for real the request_irq() code will check the bit and signal - * a local CPU interrupt for it. - */ -#if 0 - printk("IVEC: Spurious interrupt vector (%x) received at (%016lx)\n", - bucket - &ivector_table[0], regs->tpc); -#endif - *irq_work(cpu, 0) = 0; - bucket->pending = 1; -} - -/* Tune this... */ -#define FORWARD_VOLUME 12 + bp->flags |= IBF_INPROGRESS; -#ifdef CONFIG_SMP - -static inline void redirect_intr(int cpu, struct ino_bucket *bp) -{ - /* Ok, here is what is going on: - * 1) Retargeting IRQs on Starfire is very - * expensive so just forget about it on them. - * 2) Moving around very high priority interrupts - * is a losing game. - * 3) If the current cpu is idle, interrupts are - * useful work, so keep them here. But do not - * pass to our neighbour if he is not very idle. - * 4) If sysadmin explicitly asks for directed intrs, - * Just Do It. - */ - struct irqaction *ap = bp->irq_info; - cpumask_t cpu_mask; - unsigned int buddy, ticks; - - cpu_mask = get_smpaff_in_irqaction(ap); - cpus_and(cpu_mask, cpu_mask, cpu_online_map); - if (cpus_empty(cpu_mask)) - cpu_mask = cpu_online_map; - - if (this_is_starfire != 0 || - bp->pil >= 10 || current->pid == 0) + if (unlikely(!(flags & IBF_ACTIVE))) { + bp->pending = 1; goto out; - - /* 'cpu' is the MID (ie. UPAID), calculate the MID - * of our buddy. - */ - buddy = cpu + 1; - if (buddy >= NR_CPUS) - buddy = 0; - - ticks = 0; - while (!cpu_isset(buddy, cpu_mask)) { - if (++buddy >= NR_CPUS) - buddy = 0; - if (++ticks > NR_CPUS) { - put_smpaff_in_irqaction(ap, CPU_MASK_NONE); - goto out; - } } - if (buddy == cpu) - goto out; + if (desc->pre_handler) + desc->pre_handler(bp, + desc->pre_handler_arg1, + desc->pre_handler_arg2); - /* Voo-doo programming. */ - if (cpu_data(buddy).idle_volume < FORWARD_VOLUME) - goto out; + action_mask = desc->action_active_mask; + random = 0; + for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) { + struct irqaction *p = &desc->action[i]; + u32 mask = (1 << i); - /* This just so happens to be correct on Cheetah - * at the moment. - */ - buddy <<= 26; + if (!(action_mask & mask)) + continue; - /* Push it to our buddy. */ - upa_writel(buddy | IMAP_VALID, bp->imap); + action_mask &= ~mask; + if (p->handler(__irq(bp), p->dev_id, regs) == IRQ_HANDLED) + random |= p->flags; + + if (!action_mask) + break; + } + if (bp->pil != 0) { + if (tlb_type == hypervisor) { + unsigned int ino = __irq_ino(bp); + int err; + + err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); + if (err != HV_EOK) + printk("sun4v_intr_setstate(%x): " + "err(%d)\n", ino, err); + } else { + upa_writel(ICLR_IDLE, bp->iclr); + } + + /* Test and add entropy */ + if (random & SA_SAMPLE_RANDOM) + add_interrupt_randomness(irq); + } out: - return; + bp->flags &= ~IBF_INPROGRESS; } -#endif - void handler_irq(int irq, struct pt_regs *regs) { - struct ino_bucket *bp, *nbp; + struct ino_bucket *bp; int cpu = smp_processor_id(); #ifndef CONFIG_SMP @@ -756,8 +702,6 @@ void handler_irq(int irq, struct pt_regs *regs) clear_softint(clr_mask); } #else - int should_forward = 0; - clear_softint(1 << irq); #endif @@ -772,199 +716,76 @@ void handler_irq(int irq, struct pt_regs *regs) #else bp = __bucket(xchg32(irq_work(cpu, irq), 0)); #endif - for ( ; bp != NULL; bp = nbp) { - unsigned char flags = bp->flags; - unsigned char random = 0; + while (bp) { + struct ino_bucket *nbp = __bucket(bp->irq_chain); - nbp = __bucket(bp->irq_chain); bp->irq_chain = 0; - - bp->flags |= IBF_INPROGRESS; - - if ((flags & IBF_ACTIVE) != 0) { -#ifdef CONFIG_PCI - if ((flags & IBF_DMA_SYNC) != 0) { - upa_readl(dma_sync_reg_table[bp->synctab_ent]); - upa_readq(pci_dma_wsync); - } -#endif - if ((flags & IBF_MULTI) == 0) { - struct irqaction *ap = bp->irq_info; - int ret; - - ret = ap->handler(__irq(bp), ap->dev_id, regs); - if (ret == IRQ_HANDLED) - random |= ap->flags; - } else { - void **vector = (void **)bp->irq_info; - int ent; - for (ent = 0; ent < 4; ent++) { - struct irqaction *ap = vector[ent]; - if (ap != NULL) { - int ret; - - ret = ap->handler(__irq(bp), - ap->dev_id, - regs); - if (ret == IRQ_HANDLED) - random |= ap->flags; - } - } - } - /* Only the dummy bucket lacks IMAP/ICLR. */ - if (bp->pil != 0) { -#ifdef CONFIG_SMP - if (should_forward) { - redirect_intr(cpu, bp); - should_forward = 0; - } -#endif - upa_writel(ICLR_IDLE, bp->iclr); - - /* Test and add entropy */ - if (random & SA_SAMPLE_RANDOM) - add_interrupt_randomness(irq); - } - } else - bp->pending = 1; - - bp->flags &= ~IBF_INPROGRESS; + process_bucket(irq, bp, regs); + bp = nbp; } irq_exit(); } #ifdef CONFIG_BLK_DEV_FD -extern void floppy_interrupt(int irq, void *dev_cookie, struct pt_regs *regs); +extern irqreturn_t floppy_interrupt(int, void *, struct pt_regs *); -void sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs) -{ - struct irqaction *action = *(irq + irq_action); - struct ino_bucket *bucket; - int cpu = smp_processor_id(); - - irq_enter(); - kstat_this_cpu.irqs[irq]++; +/* XXX No easy way to include asm/floppy.h XXX */ +extern unsigned char *pdma_vaddr; +extern unsigned long pdma_size; +extern volatile int doing_pdma; +extern unsigned long fdc_status; - *(irq_work(cpu, irq)) = 0; - bucket = get_ino_in_irqaction(action) + ivector_table; - - bucket->flags |= IBF_INPROGRESS; - - floppy_interrupt(irq, dev_cookie, regs); - upa_writel(ICLR_IDLE, bucket->iclr); - - bucket->flags &= ~IBF_INPROGRESS; - - irq_exit(); -} -#endif - -/* The following assumes that the branch lies before the place we - * are branching to. This is the case for a trap vector... - * You have been warned. - */ -#define SPARC_BRANCH(dest_addr, inst_addr) \ - (0x10800000 | ((((dest_addr)-(inst_addr))>>2)&0x3fffff)) - -#define SPARC_NOP (0x01000000) - -static void install_fast_irq(unsigned int cpu_irq, - irqreturn_t (*handler)(int, void *, struct pt_regs *)) +irqreturn_t sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs) { - extern unsigned long sparc64_ttable_tl0; - unsigned long ttent = (unsigned long) &sparc64_ttable_tl0; - unsigned int *insns; - - ttent += 0x820; - ttent += (cpu_irq - 1) << 5; - insns = (unsigned int *) ttent; - insns[0] = SPARC_BRANCH(((unsigned long) handler), - ((unsigned long)&insns[0])); - insns[1] = SPARC_NOP; - __asm__ __volatile__("membar #StoreStore; flush %0" : : "r" (ttent)); -} - -int request_fast_irq(unsigned int irq, - irqreturn_t (*handler)(int, void *, struct pt_regs *), - unsigned long irqflags, const char *name, void *dev_id) -{ - struct irqaction *action; - struct ino_bucket *bucket = __bucket(irq); - unsigned long flags; - - /* No pil0 dummy buckets allowed here. */ - if (bucket < &ivector_table[0] || - bucket >= &ivector_table[NUM_IVECS]) { - unsigned int *caller; - - __asm__ __volatile__("mov %%i7, %0" : "=r" (caller)); - printk(KERN_CRIT "request_fast_irq: Old style IRQ registry attempt " - "from %p, irq %08x.\n", caller, irq); - return -EINVAL; - } - - if (!handler) - return -EINVAL; + if (likely(doing_pdma)) { + void __iomem *stat = (void __iomem *) fdc_status; + unsigned char *vaddr = pdma_vaddr; + unsigned long size = pdma_size; + u8 val; + + while (size) { + val = readb(stat); + if (unlikely(!(val & 0x80))) { + pdma_vaddr = vaddr; + pdma_size = size; + return IRQ_HANDLED; + } + if (unlikely(!(val & 0x20))) { + pdma_vaddr = vaddr; + pdma_size = size; + doing_pdma = 0; + goto main_interrupt; + } + if (val & 0x40) { + /* read */ + *vaddr++ = readb(stat + 1); + } else { + unsigned char data = *vaddr++; - if ((bucket->pil == 0) || (bucket->pil == 14)) { - printk("request_fast_irq: Trying to register shared IRQ 0 or 14.\n"); - return -EBUSY; - } + /* write */ + writeb(data, stat + 1); + } + size--; + } - spin_lock_irqsave(&irq_action_lock, flags); + pdma_vaddr = vaddr; + pdma_size = size; - action = *(bucket->pil + irq_action); - if (action) { - if (action->flags & SA_SHIRQ) - panic("Trying to register fast irq when already shared.\n"); - if (irqflags & SA_SHIRQ) - panic("Trying to register fast irq as shared.\n"); - printk("request_fast_irq: Trying to register yet already owned.\n"); - spin_unlock_irqrestore(&irq_action_lock, flags); - return -EBUSY; - } + /* Send Terminal Count pulse to floppy controller. */ + val = readb(auxio_register); + val |= AUXIO_AUX1_FTCNT; + writeb(val, auxio_register); + val &= ~AUXIO_AUX1_FTCNT; + writeb(val, auxio_register); - /* - * We do not check for SA_SAMPLE_RANDOM in this path. Neither do we - * support smp intr affinity in this path. - */ - if (irqflags & SA_STATIC_ALLOC) { - if (static_irq_count < MAX_STATIC_ALLOC) - action = &static_irqaction[static_irq_count++]; - else - printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed " - "using kmalloc\n", bucket->pil, name); - } - if (action == NULL) - action = (struct irqaction *)kmalloc(sizeof(struct irqaction), - GFP_ATOMIC); - if (!action) { - spin_unlock_irqrestore(&irq_action_lock, flags); - return -ENOMEM; + doing_pdma = 0; } - install_fast_irq(bucket->pil, handler); - - bucket->irq_info = action; - bucket->flags |= IBF_ACTIVE; - action->handler = handler; - action->flags = irqflags; - action->dev_id = NULL; - action->name = name; - action->next = NULL; - put_ino_in_irqaction(action, irq); - put_smpaff_in_irqaction(action, CPU_MASK_NONE); - - *(bucket->pil + irq_action) = action; - enable_irq(irq); - - spin_unlock_irqrestore(&irq_action_lock, flags); - -#ifdef CONFIG_SMP - distribute_irqs(); -#endif - return 0; +main_interrupt: + return floppy_interrupt(irq, dev_cookie, regs); } +EXPORT_SYMBOL(sparc_floppy_irq); +#endif /* We really don't need these at all on the Sparc. We only have * stubs here because they are exported to modules. @@ -987,25 +808,23 @@ EXPORT_SYMBOL(probe_irq_off); static int retarget_one_irq(struct irqaction *p, int goal_cpu) { struct ino_bucket *bucket = get_ino_in_irqaction(p) + ivector_table; - unsigned long imap = bucket->imap; - unsigned int tid; while (!cpu_online(goal_cpu)) { if (++goal_cpu >= NR_CPUS) goal_cpu = 0; } - if (tlb_type == cheetah || tlb_type == cheetah_plus) { - tid = goal_cpu << 26; - tid &= IMAP_AID_SAFARI; - } else if (this_is_starfire == 0) { - tid = goal_cpu << 26; - tid &= IMAP_TID_UPA; + if (tlb_type == hypervisor) { + unsigned int ino = __irq_ino(bucket); + + sun4v_intr_settarget(ino, goal_cpu); + sun4v_intr_setenabled(ino, HV_INTR_ENABLED); } else { - tid = (starfire_translate(imap, goal_cpu) << 26); - tid &= IMAP_TID_UPA; + unsigned long imap = bucket->imap; + unsigned int tid = sun4u_compute_tid(imap, goal_cpu); + + upa_writel(tid | IMAP_VALID, imap); } - upa_writel(tid | IMAP_VALID, imap); do { if (++goal_cpu >= NR_CPUS) @@ -1030,7 +849,10 @@ static void distribute_irqs(void) */ for (level = 1; level < NR_IRQS; level++) { struct irqaction *p = irq_action[level]; - if (level == 12) continue; + + if (level == 12) + continue; + while(p) { cpu = retarget_one_irq(p, cpu); p = p->next; @@ -1040,8 +862,14 @@ static void distribute_irqs(void) } #endif +struct sun5_timer { + u64 count0; + u64 limit0; + u64 count1; + u64 limit1; +}; -struct sun5_timer *prom_timers; +static struct sun5_timer *prom_timers; static u64 prom_limit0, prom_limit1; static void map_prom_timers(void) @@ -1097,47 +925,116 @@ static void kill_prom_timer(void) : "g1", "g2"); } -void enable_prom_timer(void) +void init_irqwork_curcpu(void) { - if (!prom_timers) - return; + int cpu = hard_smp_processor_id(); - /* Set it to whatever was there before. */ - prom_timers->limit1 = prom_limit1; - prom_timers->count1 = 0; - prom_timers->limit0 = prom_limit0; - prom_timers->count0 = 0; + memset(__irq_work + cpu, 0, sizeof(struct irq_work_struct)); } -void init_irqwork_curcpu(void) +static void __cpuinit register_one_mondo(unsigned long paddr, unsigned long type) { - register struct irq_work_struct *workp asm("o2"); - register unsigned long tmp asm("o3"); - int cpu = hard_smp_processor_id(); + unsigned long num_entries = 128; + unsigned long status; - memset(__irq_work + cpu, 0, sizeof(*workp)); - - /* Make sure we are called with PSTATE_IE disabled. */ - __asm__ __volatile__("rdpr %%pstate, %0\n\t" - : "=r" (tmp)); - if (tmp & PSTATE_IE) { - prom_printf("BUG: init_irqwork_curcpu() called with " - "PSTATE_IE enabled, bailing.\n"); - __asm__ __volatile__("mov %%i7, %0\n\t" - : "=r" (tmp)); - prom_printf("BUG: Called from %lx\n", tmp); + status = sun4v_cpu_qconf(type, paddr, num_entries); + if (status != HV_EOK) { + prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, " + "err %lu\n", type, paddr, num_entries, status); prom_halt(); } +} - /* Set interrupt globals. */ - workp = &__irq_work[cpu]; - __asm__ __volatile__( - "rdpr %%pstate, %0\n\t" - "wrpr %0, %1, %%pstate\n\t" - "mov %2, %%g6\n\t" - "wrpr %0, 0x0, %%pstate\n\t" - : "=&r" (tmp) - : "i" (PSTATE_IG), "r" (workp)); +static void __cpuinit sun4v_register_mondo_queues(int this_cpu) +{ + struct trap_per_cpu *tb = &trap_block[this_cpu]; + + register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO); + register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO); + register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR); + register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR); +} + +static void __cpuinit alloc_one_mondo(unsigned long *pa_ptr, int use_bootmem) +{ + void *page; + + if (use_bootmem) + page = alloc_bootmem_low_pages(PAGE_SIZE); + else + page = (void *) get_zeroed_page(GFP_ATOMIC); + + if (!page) { + prom_printf("SUN4V: Error, cannot allocate mondo queue.\n"); + prom_halt(); + } + + *pa_ptr = __pa(page); +} + +static void __cpuinit alloc_one_kbuf(unsigned long *pa_ptr, int use_bootmem) +{ + void *page; + + if (use_bootmem) + page = alloc_bootmem_low_pages(PAGE_SIZE); + else + page = (void *) get_zeroed_page(GFP_ATOMIC); + + if (!page) { + prom_printf("SUN4V: Error, cannot allocate kbuf page.\n"); + prom_halt(); + } + + *pa_ptr = __pa(page); +} + +static void __cpuinit init_cpu_send_mondo_info(struct trap_per_cpu *tb, int use_bootmem) +{ +#ifdef CONFIG_SMP + void *page; + + BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > (PAGE_SIZE - 64)); + + if (use_bootmem) + page = alloc_bootmem_low_pages(PAGE_SIZE); + else + page = (void *) get_zeroed_page(GFP_ATOMIC); + + if (!page) { + prom_printf("SUN4V: Error, cannot allocate cpu mondo page.\n"); + prom_halt(); + } + + tb->cpu_mondo_block_pa = __pa(page); + tb->cpu_list_pa = __pa(page + 64); +#endif +} + +/* Allocate and register the mondo and error queues for this cpu. */ +void __cpuinit sun4v_init_mondo_queues(int use_bootmem, int cpu, int alloc, int load) +{ + struct trap_per_cpu *tb = &trap_block[cpu]; + + if (alloc) { + alloc_one_mondo(&tb->cpu_mondo_pa, use_bootmem); + alloc_one_mondo(&tb->dev_mondo_pa, use_bootmem); + alloc_one_mondo(&tb->resum_mondo_pa, use_bootmem); + alloc_one_kbuf(&tb->resum_kernel_buf_pa, use_bootmem); + alloc_one_mondo(&tb->nonresum_mondo_pa, use_bootmem); + alloc_one_kbuf(&tb->nonresum_kernel_buf_pa, use_bootmem); + + init_cpu_send_mondo_info(tb, use_bootmem); + } + + if (load) { + if (cpu != hard_smp_processor_id()) { + prom_printf("SUN4V: init mondo on cpu %d not %d\n", + cpu, hard_smp_processor_id()); + prom_halt(); + } + sun4v_register_mondo_queues(cpu); + } } /* Only invoked on boot processor. */ @@ -1147,6 +1044,9 @@ void __init init_IRQ(void) kill_prom_timer(); memset(&ivector_table[0], 0, sizeof(ivector_table)); + if (tlb_type == hypervisor) + sun4v_init_mondo_queues(1, hard_smp_processor_id(), 1, 1); + /* We need to clear any IRQ's pending in the soft interrupt * registers, a spurious one could be left around from the * PROM timer which we just disabled. @@ -1175,7 +1075,8 @@ static int irq_affinity_read_proc (char *page, char **start, off_t off, int count, int *eof, void *data) { struct ino_bucket *bp = ivector_table + (long)data; - struct irqaction *ap = bp->irq_info; + struct irq_desc *desc = bp->irq_info; + struct irqaction *ap = desc->action; cpumask_t mask; int len; @@ -1193,11 +1094,13 @@ static int irq_affinity_read_proc (char *page, char **start, off_t off, static inline void set_intr_affinity(int irq, cpumask_t hw_aff) { struct ino_bucket *bp = ivector_table + irq; + struct irq_desc *desc = bp->irq_info; + struct irqaction *ap = desc->action; /* Users specify affinity in terms of hw cpu ids. * As soon as we do this, handler_irq() might see and take action. */ - put_smpaff_in_irqaction((struct irqaction *)bp->irq_info, hw_aff); + put_smpaff_in_irqaction(ap, hw_aff); /* Migration is simply done by the next cpu to service this * interrupt.