arch/sparc64/kernel/irq.c

   1 /* $Id: irq.c,v 1.114 2002/01/11 08:45:38 davem Exp $
   2  * irq.c: UltraSparc IRQ handling/init/registry.
   3  *
   4  * Copyright (C) 1997  David S. Miller  (davem@caip.rutgers.edu)
   5  * Copyright (C) 1998  Eddie C. Dost    (ecd@skynet.be)
   6  * Copyright (C) 1998  Jakub Jelinek    (jj@ultra.linux.cz)
   7  */
   8
   9 #include <linux/config.h>
  10 #include <linux/module.h>
  11 #include <linux/sched.h>
  12 #include <linux/ptrace.h>
  13 #include <linux/errno.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/signal.h>
  16 #include <linux/mm.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/slab.h>
  19 #include <linux/random.h>
  20 #include <linux/init.h>
  21 #include <linux/delay.h>
  22 #include <linux/proc_fs.h>
  23 #include <linux/seq_file.h>
  24
  25 #include <asm/ptrace.h>
  26 #include <asm/processor.h>
  27 #include <asm/atomic.h>
  28 #include <asm/system.h>
  29 #include <asm/irq.h>
  30 #include <asm/sbus.h>
  31 #include <asm/iommu.h>
  32 #include <asm/upa.h>
  33 #include <asm/oplib.h>
  34 #include <asm/timer.h>
  35 #include <asm/smp.h>
  36 #include <asm/hardirq.h>
  37 #include <asm/starfire.h>
  38 #include <asm/uaccess.h>
  39 #include <asm/cache.h>
  40 #include <asm/cpudata.h>
  41
  42 #ifdef CONFIG_SMP
  43 static void distribute_irqs(void);
  44 #endif
  45
  46 /* UPA nodes send interrupt packet to UltraSparc with first data reg
  47  * value low 5 (7 on Starfire) bits holding the IRQ identifier being
  48  * delivered.  We must translate this into a non-vector IRQ so we can
  49  * set the softint on this cpu.
  50  *
  51  * To make processing these packets efficient and race free we use
  52  * an array of irq buckets below.  The interrupt vector handler in
  53  * entry.S feeds incoming packets into per-cpu pil-indexed lists.
  54  * The IVEC handler does not need to act atomically, the PIL dispatch
  55  * code uses CAS to get an atomic snapshot of the list and clear it
  56  * at the same time.
  57  */
  58
  59 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
  60
  61 /* This has to be in the main kernel image, it cannot be
  62  * turned into per-cpu data.  The reason is that the main
  63  * kernel image is locked into the TLB and this structure
  64  * is accessed from the vectored interrupt trap handler.  If
  65  * access to this structure takes a TLB miss it could cause
  66  * the 5-level sparc v9 trap stack to overflow.
  67  */
  68 struct irq_work_struct {
  69         unsigned int    irq_worklists[16];
  70 };
  71 struct irq_work_struct __irq_work[NR_CPUS];
  72 #define irq_work(__cpu, __pil)  &(__irq_work[(__cpu)].irq_worklists[(__pil)])
  73
  74 #ifdef CONFIG_PCI
  75 /* This is a table of physical addresses used to deal with IBF_DMA_SYNC.
  76  * It is used for PCI only to synchronize DMA transfers with IRQ delivery
  77  * for devices behind busses other than APB on Sabre systems.
  78  *
  79  * Currently these physical addresses are just config space accesses
  80  * to the command register for that device.
  81  */
  82 unsigned long pci_dma_wsync;
  83 unsigned long dma_sync_reg_table[256];
  84 unsigned char dma_sync_reg_table_entry = 0;
  85 #endif
  86
  87 /* This is based upon code in the 32-bit Sparc kernel written mostly by
  88  * David Redman (djhr@tadpole.co.uk).
  89  */
  90 #define MAX_STATIC_ALLOC        4
  91 static struct irqaction static_irqaction[MAX_STATIC_ALLOC];
  92 static int static_irq_count;
  93
  94 /* This is exported so that fast IRQ handlers can get at it... -DaveM */
  95 struct irqaction *irq_action[NR_IRQS+1] = {
  96           NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL,
  97           NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL
  98 };
  99
 100 /* This only synchronizes entities which modify IRQ handler
 101  * state and some selected user-level spots that want to
 102  * read things in the table.  IRQ handler processing orders
 103  * its' accesses such that no locking is needed.
 104  */
 105 static spinlock_t irq_action_lock = SPIN_LOCK_UNLOCKED;
 106
 107 static void register_irq_proc (unsigned int irq);
 108
 109 /*
 110  * Upper 2b of irqaction->flags holds the ino.
 111  * irqaction->mask holds the smp affinity information.
 112  */
 113 #define put_ino_in_irqaction(action, irq) \
 114         action->flags &= 0xffffffffffffUL; \
 115         if (__bucket(irq) == &pil0_dummy_bucket) \
 116                 action->flags |= 0xdeadUL << 48;  \
 117         else \
 118                 action->flags |= __irq_ino(irq) << 48;
 119 #define get_ino_in_irqaction(action)    (action->flags >> 48)
 120
 121 #if NR_CPUS > 64
 122 #error irqaction embedded smp affinity does not work with > 64 cpus, FIXME
 123 #endif
 124
 125 #define put_smpaff_in_irqaction(action, smpaff) (action)->mask = (smpaff)
 126 #define get_smpaff_in_irqaction(action)         ((action)->mask)
 127
 128 int show_interrupts(struct seq_file *p, void *v)
 129 {
 130         unsigned long flags;
 131         int i = *(loff_t *) v;
 132         struct irqaction *action;
 133 #ifdef CONFIG_SMP
 134         int j;
 135 #endif
 136
 137         spin_lock_irqsave(&irq_action_lock, flags);
 138         if (i <= NR_IRQS) {
 139                 if (!(action = *(i + irq_action)))
 140                         goto out_unlock;
 141                 seq_printf(p, "%3d: ", i);
 142 #ifndef CONFIG_SMP
 143                 seq_printf(p, "%10u ", kstat_irqs(i));
 144 #else
 145                 for (j = 0; j < NR_CPUS; j++) {
 146                         if (!cpu_online(j))
 147                                 continue;
 148                         seq_printf(p, "%10u ",
 149                                    kstat_cpu(j).irqs[i]);
 150                 }
 151 #endif
 152                 seq_printf(p, " %s:%lx", action->name,
 153                            get_ino_in_irqaction(action));
 154                 for (action = action->next; action; action = action->next) {
 155                         seq_printf(p, ", %s:%lx", action->name,
 156                                    get_ino_in_irqaction(action));
 157                 }
 158                 seq_putc(p, '\n');
 159         }
 160 out_unlock:
 161         spin_unlock_irqrestore(&irq_action_lock, flags);
 162
 163         return 0;
 164 }
 165
 166 /* Now these are always passed a true fully specified sun4u INO. */
 167 void enable_irq(unsigned int irq)
 168 {
 169         struct ino_bucket *bucket = __bucket(irq);
 170         unsigned long imap;
 171         unsigned long tid;
 172
 173         imap = bucket->imap;
 174         if (imap == 0UL)
 175                 return;
 176
 177         preempt_disable();
 178
 179         if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 180                 unsigned long ver;
 181
 182                 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
 183                 if ((ver >> 32) == 0x003e0016) {
 184                         /* We set it to our JBUS ID. */
 185                         __asm__ __volatile__("ldxa [%%g0] %1, %0"
 186                                              : "=r" (tid)
 187                                              : "i" (ASI_JBUS_CONFIG));
 188                         tid = ((tid & (0x1fUL<<17)) << 9);
 189                         tid &= IMAP_TID_JBUS;
 190                 } else {
 191                         /* We set it to our Safari AID. */
 192                         __asm__ __volatile__("ldxa [%%g0] %1, %0"
 193                                              : "=r" (tid)
 194                                              : "i" (ASI_SAFARI_CONFIG));
 195                         tid = ((tid & (0x3ffUL<<17)) << 9);
 196                         tid &= IMAP_AID_SAFARI;
 197                 }
 198         } else if (this_is_starfire == 0) {
 199                 /* We set it to our UPA MID. */
 200                 __asm__ __volatile__("ldxa [%%g0] %1, %0"
 201                                      : "=r" (tid)
 202                                      : "i" (ASI_UPA_CONFIG));
 203                 tid = ((tid & UPA_CONFIG_MID) << 9);
 204                 tid &= IMAP_TID_UPA;
 205         } else {
 206                 tid = (starfire_translate(imap, smp_processor_id()) << 26);
 207                 tid &= IMAP_TID_UPA;
 208         }
 209
 210         /* NOTE NOTE NOTE, IGN and INO are read-only, IGN is a product
 211          * of this SYSIO's preconfigured IGN in the SYSIO Control
 212          * Register, the hardware just mirrors that value here.
 213          * However for Graphics and UPA Slave devices the full
 214          * IMAP_INR field can be set by the programmer here.
 215          *
 216          * Things like FFB can now be handled via the new IRQ mechanism.
 217          */
 218         upa_writel(tid | IMAP_VALID, imap);
 219
 220         preempt_enable();
 221 }
 222
 223 /* This now gets passed true ino's as well. */
 224 void disable_irq(unsigned int irq)
 225 {
 226         struct ino_bucket *bucket = __bucket(irq);
 227         unsigned long imap;
 228
 229         imap = bucket->imap;
 230         if (imap != 0UL) {
 231                 u32 tmp;
 232
 233                 /* NOTE: We do not want to futz with the IRQ clear registers
 234                  *       and move the state to IDLE, the SCSI code does call
 235                  *       disable_irq() to assure atomicity in the queue cmd
 236                  *       SCSI adapter driver code.  Thus we'd lose interrupts.
 237                  */
 238                 tmp = upa_readl(imap);
 239                 tmp &= ~IMAP_VALID;
 240                 upa_writel(tmp, imap);
 241         }
 242 }
 243
 244 /* The timer is the one "weird" interrupt which is generated by
 245  * the CPU %tick register and not by some normal vectored interrupt
 246  * source.  To handle this special case, we use this dummy INO bucket.
 247  */
 248 static struct ino_bucket pil0_dummy_bucket = {
 249         0,      /* irq_chain */
 250         0,      /* pil */
 251         0,      /* pending */
 252         0,      /* flags */
 253         0,      /* __unused */
 254         NULL,   /* irq_info */
 255         0UL,    /* iclr */
 256         0UL,    /* imap */
 257 };
 258
 259 unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long imap)
 260 {
 261         struct ino_bucket *bucket;
 262         int ino;
 263
 264         if (pil == 0) {
 265                 if (iclr != 0UL || imap != 0UL) {
 266                         prom_printf("Invalid dummy bucket for PIL0 (%lx:%lx)\n",
 267                                     iclr, imap);
 268                         prom_halt();
 269                 }
 270                 return __irq(&pil0_dummy_bucket);
 271         }
 272
 273         /* RULE: Both must be specified in all other cases. */
 274         if (iclr == 0UL || imap == 0UL) {
 275                 prom_printf("Invalid build_irq %d %d %016lx %016lx\n",
 276                             pil, inofixup, iclr, imap);
 277                 prom_halt();
 278         }
 279
 280         ino = (upa_readl(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
 281         if (ino > NUM_IVECS) {
 282                 prom_printf("Invalid INO %04x (%d:%d:%016lx:%016lx)\n",
 283                             ino, pil, inofixup, iclr, imap);
 284                 prom_halt();
 285         }
 286
 287         /* Ok, looks good, set it up.  Don't touch the irq_chain or
 288          * the pending flag.
 289          */
 290         bucket = &ivector_table[ino];
 291         if ((bucket->flags & IBF_ACTIVE) ||
 292             (bucket->irq_info != NULL)) {
 293                 /* This is a gross fatal error if it happens here. */
 294                 prom_printf("IRQ: Trying to reinit INO bucket, fatal error.\n");
 295                 prom_printf("IRQ: Request INO %04x (%d:%d:%016lx:%016lx)\n",
 296                             ino, pil, inofixup, iclr, imap);
 297                 prom_printf("IRQ: Existing (%d:%016lx:%016lx)\n",
 298                             bucket->pil, bucket->iclr, bucket->imap);
 299                 prom_printf("IRQ: Cannot continue, halting...\n");
 300                 prom_halt();
 301         }
 302         bucket->imap  = imap;
 303         bucket->iclr  = iclr;
 304         bucket->pil   = pil;
 305         bucket->flags = 0;
 306
 307         bucket->irq_info = NULL;
 308
 309         return __irq(bucket);
 310 }
 311
 312 static void atomic_bucket_insert(struct ino_bucket *bucket)
 313 {
 314         unsigned long pstate;
 315         unsigned int *ent;
 316
 317         __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
 318         __asm__ __volatile__("wrpr %0, %1, %%pstate"
 319                              : : "r" (pstate), "i" (PSTATE_IE));
 320         ent = irq_work(smp_processor_id(), bucket->pil);
 321         bucket->irq_chain = *ent;
 322         *ent = __irq(bucket);
 323         __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
 324 }
 325
 326 int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_regs *),
 327                 unsigned long irqflags, const char *name, void *dev_id)
 328 {
 329         struct irqaction *action, *tmp = NULL;
 330         struct ino_bucket *bucket = __bucket(irq);
 331         unsigned long flags;
 332         int pending = 0;
 333
 334         if ((bucket != &pil0_dummy_bucket) &&
 335             (bucket < &ivector_table[0] ||
 336              bucket >= &ivector_table[NUM_IVECS])) {
 337                 unsigned int *caller;
 338
 339                 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
 340                 printk(KERN_CRIT "request_irq: Old style IRQ registry attempt "
 341                        "from %p, irq %08x.\n", caller, irq);
 342                 return -EINVAL;
 343         }
 344         if (!handler)
 345             return -EINVAL;
 346
 347         if ((bucket != &pil0_dummy_bucket) && (irqflags & SA_SAMPLE_RANDOM)) {
 348                 /*
 349                  * This function might sleep, we want to call it first,
 350                  * outside of the atomic block. In SA_STATIC_ALLOC case,
 351                  * random driver's kmalloc will fail, but it is safe.
 352                  * If already initialized, random driver will not reinit.
 353                  * Yes, this might clear the entropy pool if the wrong
 354                  * driver is attempted to be loaded, without actually
 355                  * installing a new handler, but is this really a problem,
 356                  * only the sysadmin is able to do this.
 357                  */
 358                 rand_initialize_irq(irq);
 359         }
 360
 361         spin_lock_irqsave(&irq_action_lock, flags);
 362
 363         action = *(bucket->pil + irq_action);
 364         if (action) {
 365                 if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ))
 366                         for (tmp = action; tmp->next; tmp = tmp->next)
 367                                 ;
 368                 else {
 369                         spin_unlock_irqrestore(&irq_action_lock, flags);
 370                         return -EBUSY;
 371                 }
 372                 action = NULL;          /* Or else! */
 373         }
 374
 375         /* If this is flagged as statically allocated then we use our
 376          * private struct which is never freed.
 377          */
 378         if (irqflags & SA_STATIC_ALLOC) {
 379             if (static_irq_count < MAX_STATIC_ALLOC)
 380                 action = &static_irqaction[static_irq_count++];
 381             else
 382                 printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
 383                        "using kmalloc\n", irq, name);
 384         }
 385         if (action == NULL)
 386             action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
 387                                                  GFP_ATOMIC);
 388
 389         if (!action) {
 390                 spin_unlock_irqrestore(&irq_action_lock, flags);
 391                 return -ENOMEM;
 392         }
 393
 394         if (bucket == &pil0_dummy_bucket) {
 395                 bucket->irq_info = action;
 396                 bucket->flags |= IBF_ACTIVE;
 397         } else {
 398                 if ((bucket->flags & IBF_ACTIVE) != 0) {
 399                         void *orig = bucket->irq_info;
 400                         void **vector = NULL;
 401
 402                         if ((bucket->flags & IBF_PCI) == 0) {
 403                                 printk("IRQ: Trying to share non-PCI bucket.\n");
 404                                 goto free_and_ebusy;
 405                         }
 406                         if ((bucket->flags & IBF_MULTI) == 0) {
 407                                 vector = kmalloc(sizeof(void *) * 4, GFP_ATOMIC);
 408                                 if (vector == NULL)
 409                                         goto free_and_enomem;
 410
 411                                 /* We might have slept. */
 412                                 if ((bucket->flags & IBF_MULTI) != 0) {
 413                                         int ent;
 414
 415                                         kfree(vector);
 416                                         vector = (void **)bucket->irq_info;
 417                                         for(ent = 0; ent < 4; ent++) {
 418                                                 if (vector[ent] == NULL) {
 419                                                         vector[ent] = action;
 420                                                         break;
 421                                                 }
 422                                         }
 423                                         if (ent == 4)
 424                                                 goto free_and_ebusy;
 425                                 } else {
 426                                         vector[0] = orig;
 427                                         vector[1] = action;
 428                                         vector[2] = NULL;
 429                                         vector[3] = NULL;
 430                                         bucket->irq_info = vector;
 431                                         bucket->flags |= IBF_MULTI;
 432                                 }
 433                         } else {
 434                                 int ent;
 435
 436                                 vector = (void **)orig;
 437                                 for (ent = 0; ent < 4; ent++) {
 438                                         if (vector[ent] == NULL) {
 439                                                 vector[ent] = action;
 440                                                 break;
 441                                         }
 442                                 }
 443                                 if (ent == 4)
 444                                         goto free_and_ebusy;
 445                         }
 446                 } else {
 447                         bucket->irq_info = action;
 448                         bucket->flags |= IBF_ACTIVE;
 449                 }
 450                 pending = bucket->pending;
 451                 if (pending)
 452                         bucket->pending = 0;
 453         }
 454
 455         action->handler = handler;
 456         action->flags = irqflags;
 457         action->name = name;
 458         action->next = NULL;
 459         action->dev_id = dev_id;
 460         put_ino_in_irqaction(action, irq);
 461         put_smpaff_in_irqaction(action, 0);
 462
 463         if (tmp)
 464                 tmp->next = action;
 465         else
 466                 *(bucket->pil + irq_action) = action;
 467
 468         enable_irq(irq);
 469
 470         /* We ate the IVEC already, this makes sure it does not get lost. */
 471         if (pending) {
 472                 atomic_bucket_insert(bucket);
 473                 set_softint(1 << bucket->pil);
 474         }
 475         spin_unlock_irqrestore(&irq_action_lock, flags);
 476         if ((bucket != &pil0_dummy_bucket) && (!(irqflags & SA_STATIC_ALLOC)))
 477                 register_irq_proc(__irq_ino(irq));
 478
 479 #ifdef CONFIG_SMP
 480         distribute_irqs();
 481 #endif
 482         return 0;
 483
 484 free_and_ebusy:
 485         kfree(action);
 486         spin_unlock_irqrestore(&irq_action_lock, flags);
 487         return -EBUSY;
 488
 489 free_and_enomem:
 490         kfree(action);
 491         spin_unlock_irqrestore(&irq_action_lock, flags);
 492         return -ENOMEM;
 493 }
 494
 495 EXPORT_SYMBOL(request_irq);
 496
 497 void free_irq(unsigned int irq, void *dev_id)
 498 {
 499         struct irqaction *action;
 500         struct irqaction *tmp = NULL;
 501         unsigned long flags;
 502         struct ino_bucket *bucket = __bucket(irq), *bp;
 503
 504         if ((bucket != &pil0_dummy_bucket) &&
 505             (bucket < &ivector_table[0] ||
 506              bucket >= &ivector_table[NUM_IVECS])) {
 507                 unsigned int *caller;
 508
 509                 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
 510                 printk(KERN_CRIT "free_irq: Old style IRQ removal attempt "
 511                        "from %p, irq %08x.\n", caller, irq);
 512                 return;
 513         }
 514
 515         spin_lock_irqsave(&irq_action_lock, flags);
 516
 517         action = *(bucket->pil + irq_action);
 518         if (!action->handler) {
 519                 printk("Freeing free IRQ %d\n", bucket->pil);
 520                 return;
 521         }
 522         if (dev_id) {
 523                 for ( ; action; action = action->next) {
 524                         if (action->dev_id == dev_id)
 525                                 break;
 526                         tmp = action;
 527                 }
 528                 if (!action) {
 529                         printk("Trying to free free shared IRQ %d\n", bucket->pil);
 530                         spin_unlock_irqrestore(&irq_action_lock, flags);
 531                         return;
 532                 }
 533         } else if (action->flags & SA_SHIRQ) {
 534                 printk("Trying to free shared IRQ %d with NULL device ID\n", bucket->pil);
 535                 spin_unlock_irqrestore(&irq_action_lock, flags);
 536                 return;
 537         }
 538
 539         if (action->flags & SA_STATIC_ALLOC) {
 540                 printk("Attempt to free statically allocated IRQ %d (%s)\n",
 541                        bucket->pil, action->name);
 542                 spin_unlock_irqrestore(&irq_action_lock, flags);
 543                 return;
 544         }
 545
 546         if (action && tmp)
 547                 tmp->next = action->next;
 548         else
 549                 *(bucket->pil + irq_action) = action->next;
 550
 551         spin_unlock_irqrestore(&irq_action_lock, flags);
 552
 553         synchronize_irq(irq);
 554
 555         spin_lock_irqsave(&irq_action_lock, flags);
 556
 557         if (bucket != &pil0_dummy_bucket) {
 558                 unsigned long imap = bucket->imap;
 559                 void **vector, *orig;
 560                 int ent;
 561
 562                 orig = bucket->irq_info;
 563                 vector = (void **)orig;
 564
 565                 if ((bucket->flags & IBF_MULTI) != 0) {
 566                         int other = 0;
 567                         void *orphan = NULL;
 568                         for (ent = 0; ent < 4; ent++) {
 569                                 if (vector[ent] == action)
 570                                         vector[ent] = NULL;
 571                                 else if (vector[ent] != NULL) {
 572                                         orphan = vector[ent];
 573                                         other++;
 574                                 }
 575                         }
 576
 577                         /* Only free when no other shared irq
 578                          * uses this bucket.
 579                          */
 580                         if (other) {
 581                                 if (other == 1) {
 582                                         /* Convert back to non-shared bucket. */
 583                                         bucket->irq_info = orphan;
 584                                         bucket->flags &= ~(IBF_MULTI);
 585                                         kfree(vector);
 586                                 }
 587                                 goto out;
 588                         }
 589                 } else {
 590                         bucket->irq_info = NULL;
 591                 }
 592
 593                 /* This unique interrupt source is now inactive. */
 594                 bucket->flags &= ~IBF_ACTIVE;
 595
 596                 /* See if any other buckets share this bucket's IMAP
 597                  * and are still active.
 598                  */
 599                 for (ent = 0; ent < NUM_IVECS; ent++) {
 600                         bp = &ivector_table[ent];
 601                         if (bp != bucket        &&
 602                             bp->imap == imap    &&
 603                             (bp->flags & IBF_ACTIVE) != 0)
 604                                 break;
 605                 }
 606
 607                 /* Only disable when no other sub-irq levels of
 608                  * the same IMAP are active.
 609                  */
 610                 if (ent == NUM_IVECS)
 611                         disable_irq(irq);
 612         }
 613
 614 out:
 615         kfree(action);
 616         spin_unlock_irqrestore(&irq_action_lock, flags);
 617 }
 618
 619 EXPORT_SYMBOL(free_irq);
 620
 621 #ifdef CONFIG_SMP
 622 void synchronize_irq(unsigned int irq)
 623 {
 624         struct ino_bucket *bucket = __bucket(irq);
 625
 626 #if 0
 627         /* The following is how I wish I could implement this.
 628          * Unfortunately the ICLR registers are read-only, you can
 629          * only write ICLR_foo values to them.  To get the current
 630          * IRQ status you would need to get at the IRQ diag registers
 631          * in the PCI/SBUS controller and the layout of those vary
 632          * from one controller to the next, sigh... -DaveM
 633          */
 634         unsigned long iclr = bucket->iclr;
 635
 636         while (1) {
 637                 u32 tmp = upa_readl(iclr);
 638
 639                 if (tmp == ICLR_TRANSMIT ||
 640                     tmp == ICLR_PENDING) {
 641                         cpu_relax();
 642                         continue;
 643                 }
 644                 break;
 645         }
 646 #else
 647         /* So we have to do this with a INPROGRESS bit just like x86.  */
 648         while (bucket->flags & IBF_INPROGRESS)
 649                 cpu_relax();
 650 #endif
 651 }
 652 #endif /* CONFIG_SMP */
 653
 654 void catch_disabled_ivec(struct pt_regs *regs)
 655 {
 656         int cpu = smp_processor_id();
 657         struct ino_bucket *bucket = __bucket(*irq_work(cpu, 0));
 658
 659         /* We can actually see this on Ultra/PCI PCI cards, which are bridges
 660          * to other devices.  Here a single IMAP enabled potentially multiple
 661          * unique interrupt sources (which each do have a unique ICLR register.
 662          *
 663          * So what we do is just register that the IVEC arrived, when registered
 664          * for real the request_irq() code will check the bit and signal
 665          * a local CPU interrupt for it.
 666          */
 667 #if 0
 668         printk("IVEC: Spurious interrupt vector (%x) received at (%016lx)\n",
 669                bucket - &ivector_table[0], regs->tpc);
 670 #endif
 671         *irq_work(cpu, 0) = 0;
 672         bucket->pending = 1;
 673 }
 674
 675 /* Tune this... */
 676 #define FORWARD_VOLUME          12
 677
 678 #ifdef CONFIG_SMP
 679
 680 static inline void redirect_intr(int cpu, struct ino_bucket *bp)
 681 {
 682         /* Ok, here is what is going on:
 683          * 1) Retargeting IRQs on Starfire is very
 684          *    expensive so just forget about it on them.
 685          * 2) Moving around very high priority interrupts
 686          *    is a losing game.
 687          * 3) If the current cpu is idle, interrupts are
 688          *    useful work, so keep them here.  But do not
 689          *    pass to our neighbour if he is not very idle.
 690          * 4) If sysadmin explicitly asks for directed intrs,
 691          *    Just Do It.
 692          */
 693         struct irqaction *ap = bp->irq_info;
 694         cpumask_t cpu_mask = get_smpaff_in_irqaction(ap);
 695         unsigned int buddy, ticks;
 696
 697         cpus_and(cpu_mask, cpu_mask, cpu_online_map);
 698         if (cpus_empty(cpu_mask))
 699                 cpu_mask = cpu_online_map;
 700
 701         if (this_is_starfire != 0 ||
 702             bp->pil >= 10 || current->pid == 0)
 703                 goto out;
 704
 705         /* 'cpu' is the MID (ie. UPAID), calculate the MID
 706          * of our buddy.
 707          */
 708         buddy = cpu + 1;
 709         if (buddy >= NR_CPUS)
 710                 buddy = 0;
 711
 712         ticks = 0;
 713         while (!cpu_isset(buddy, cpu_mask)) {
 714                 if (++buddy >= NR_CPUS)
 715                         buddy = 0;
 716                 if (++ticks > NR_CPUS) {
 717                         put_smpaff_in_irqaction(ap, 0);
 718                         goto out;
 719                 }
 720         }
 721
 722         if (buddy == cpu)
 723                 goto out;
 724
 725         /* Voo-doo programming. */
 726         if (cpu_data(buddy).idle_volume < FORWARD_VOLUME)
 727                 goto out;
 728
 729         /* This just so happens to be correct on Cheetah
 730          * at the moment.
 731          */
 732         buddy <<= 26;
 733
 734         /* Push it to our buddy. */
 735         upa_writel(buddy | IMAP_VALID, bp->imap);
 736
 737 out:
 738         return;
 739 }
 740
 741 #endif
 742
 743 void handler_irq(int irq, struct pt_regs *regs)
 744 {
 745         struct ino_bucket *bp, *nbp;
 746         int cpu = smp_processor_id();
 747
 748 #ifndef CONFIG_SMP
 749         /*
 750          * Check for TICK_INT on level 14 softint.
 751          */
 752         {
 753                 unsigned long clr_mask = 1 << irq;
 754                 unsigned long tick_mask = tick_ops->softint_mask;
 755
 756                 if ((irq == 14) && (get_softint() & tick_mask)) {
 757                         irq = 0;
 758                         clr_mask = tick_mask;
 759                 }
 760                 clear_softint(clr_mask);
 761         }
 762 #else
 763         int should_forward = 1;
 764
 765         clear_softint(1 << irq);
 766 #endif
 767
 768         irq_enter();
 769         kstat_this_cpu.irqs[irq]++;
 770
 771         /* Sliiiick... */
 772 #ifndef CONFIG_SMP
 773         bp = ((irq != 0) ?
 774               __bucket(xchg32(irq_work(cpu, irq), 0)) :
 775               &pil0_dummy_bucket);
 776 #else
 777         bp = __bucket(xchg32(irq_work(cpu, irq), 0));
 778 #endif
 779         for ( ; bp != NULL; bp = nbp) {
 780                 unsigned char flags = bp->flags;
 781                 unsigned char random = 0;
 782
 783                 nbp = __bucket(bp->irq_chain);
 784                 bp->irq_chain = 0;
 785
 786                 bp->flags |= IBF_INPROGRESS;
 787
 788                 if ((flags & IBF_ACTIVE) != 0) {
 789 #ifdef CONFIG_PCI
 790                         if ((flags & IBF_DMA_SYNC) != 0) {
 791                                 upa_readl(dma_sync_reg_table[bp->synctab_ent]);
 792                                 upa_readq(pci_dma_wsync);
 793                         }
 794 #endif
 795                         if ((flags & IBF_MULTI) == 0) {
 796                                 struct irqaction *ap = bp->irq_info;
 797                                 ap->handler(__irq(bp), ap->dev_id, regs);
 798                                 random |= ap->flags & SA_SAMPLE_RANDOM;
 799                         } else {
 800                                 void **vector = (void **)bp->irq_info;
 801                                 int ent;
 802                                 for (ent = 0; ent < 4; ent++) {
 803                                         struct irqaction *ap = vector[ent];
 804                                         if (ap != NULL) {
 805                                                 ap->handler(__irq(bp), ap->dev_id, regs);
 806                                                 random |= ap->flags & SA_SAMPLE_RANDOM;
 807                                         }
 808                                 }
 809                         }
 810                         /* Only the dummy bucket lacks IMAP/ICLR. */
 811                         if (bp->pil != 0) {
 812 #ifdef CONFIG_SMP
 813                                 if (should_forward) {
 814                                         redirect_intr(cpu, bp);
 815                                         should_forward = 0;
 816                                 }
 817 #endif
 818                                 upa_writel(ICLR_IDLE, bp->iclr);
 819                                 /* Test and add entropy */
 820                                 if (random)
 821                                         add_interrupt_randomness(irq);
 822                         }
 823                 } else
 824                         bp->pending = 1;
 825
 826                 bp->flags &= ~IBF_INPROGRESS;
 827         }
 828         irq_exit();
 829 }
 830
 831 #ifdef CONFIG_BLK_DEV_FD
 832 extern void floppy_interrupt(int irq, void *dev_cookie, struct pt_regs *regs);
 833
 834 void sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs)
 835 {
 836         struct irqaction *action = *(irq + irq_action);
 837         struct ino_bucket *bucket;
 838         int cpu = smp_processor_id();
 839
 840         irq_enter();
 841         kstat_this_cpu.irqs[irq]++;
 842
 843         *(irq_work(cpu, irq)) = 0;
 844         bucket = get_ino_in_irqaction(action) + ivector_table;
 845
 846         bucket->flags |= IBF_INPROGRESS;
 847
 848         floppy_interrupt(irq, dev_cookie, regs);
 849         upa_writel(ICLR_IDLE, bucket->iclr);
 850
 851         bucket->flags &= ~IBF_INPROGRESS;
 852
 853         irq_exit();
 854 }
 855 #endif
 856
 857 /* The following assumes that the branch lies before the place we
 858  * are branching to.  This is the case for a trap vector...
 859  * You have been warned.
 860  */
 861 #define SPARC_BRANCH(dest_addr, inst_addr) \
 862           (0x10800000 | ((((dest_addr)-(inst_addr))>>2)&0x3fffff))
 863
 864 #define SPARC_NOP (0x01000000)
 865
 866 static void install_fast_irq(unsigned int cpu_irq,
 867                              irqreturn_t (*handler)(int, void *, struct pt_regs *))
 868 {
 869         extern unsigned long sparc64_ttable_tl0;
 870         unsigned long ttent = (unsigned long) &sparc64_ttable_tl0;
 871         unsigned int *insns;
 872
 873         ttent += 0x820;
 874         ttent += (cpu_irq - 1) << 5;
 875         insns = (unsigned int *) ttent;
 876         insns[0] = SPARC_BRANCH(((unsigned long) handler),
 877                                 ((unsigned long)&insns[0]));
 878         insns[1] = SPARC_NOP;
 879         __asm__ __volatile__("membar #StoreStore; flush %0" : : "r" (ttent));
 880 }
 881
 882 int request_fast_irq(unsigned int irq,
 883                      irqreturn_t (*handler)(int, void *, struct pt_regs *),
 884                      unsigned long irqflags, const char *name, void *dev_id)
 885 {
 886         struct irqaction *action;
 887         struct ino_bucket *bucket = __bucket(irq);
 888         unsigned long flags;
 889
 890         /* No pil0 dummy buckets allowed here. */
 891         if (bucket < &ivector_table[0] ||
 892             bucket >= &ivector_table[NUM_IVECS]) {
 893                 unsigned int *caller;
 894
 895                 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
 896                 printk(KERN_CRIT "request_fast_irq: Old style IRQ registry attempt "
 897                        "from %p, irq %08x.\n", caller, irq);
 898                 return -EINVAL;
 899         }
 900
 901         if (!handler)
 902                 return -EINVAL;
 903
 904         if ((bucket->pil == 0) || (bucket->pil == 14)) {
 905                 printk("request_fast_irq: Trying to register shared IRQ 0 or 14.\n");
 906                 return -EBUSY;
 907         }
 908
 909         spin_lock_irqsave(&irq_action_lock, flags);
 910
 911         action = *(bucket->pil + irq_action);
 912         if (action) {
 913                 if (action->flags & SA_SHIRQ)
 914                         panic("Trying to register fast irq when already shared.\n");
 915                 if (irqflags & SA_SHIRQ)
 916                         panic("Trying to register fast irq as shared.\n");
 917                 printk("request_fast_irq: Trying to register yet already owned.\n");
 918                 spin_unlock_irqrestore(&irq_action_lock, flags);
 919                 return -EBUSY;
 920         }
 921
 922         /*
 923          * We do not check for SA_SAMPLE_RANDOM in this path. Neither do we
 924          * support smp intr affinity in this path.
 925          */
 926         if (irqflags & SA_STATIC_ALLOC) {
 927                 if (static_irq_count < MAX_STATIC_ALLOC)
 928                         action = &static_irqaction[static_irq_count++];
 929                 else
 930                         printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
 931                                "using kmalloc\n", bucket->pil, name);
 932         }
 933         if (action == NULL)
 934                 action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
 935                                                      GFP_ATOMIC);
 936         if (!action) {
 937                 spin_unlock_irqrestore(&irq_action_lock, flags);
 938                 return -ENOMEM;
 939         }
 940         install_fast_irq(bucket->pil, handler);
 941
 942         bucket->irq_info = action;
 943         bucket->flags |= IBF_ACTIVE;
 944
 945         action->handler = handler;
 946         action->flags = irqflags;
 947         action->dev_id = NULL;
 948         action->name = name;
 949         action->next = NULL;
 950         put_ino_in_irqaction(action, irq);
 951         put_smpaff_in_irqaction(action, 0);
 952
 953         *(bucket->pil + irq_action) = action;
 954         enable_irq(irq);
 955
 956         spin_unlock_irqrestore(&irq_action_lock, flags);
 957
 958 #ifdef CONFIG_SMP
 959         distribute_irqs();
 960 #endif
 961         return 0;
 962 }
 963
 964 /* We really don't need these at all on the Sparc.  We only have
 965  * stubs here because they are exported to modules.
 966  */
 967 unsigned long probe_irq_on(void)
 968 {
 969         return 0;
 970 }
 971
 972 EXPORT_SYMBOL(probe_irq_on);
 973
 974 int probe_irq_off(unsigned long mask)
 975 {
 976         return 0;
 977 }
 978
 979 EXPORT_SYMBOL(probe_irq_off);
 980
 981 #ifdef CONFIG_SMP
 982 static int retarget_one_irq(struct irqaction *p, int goal_cpu)
 983 {
 984         struct ino_bucket *bucket = get_ino_in_irqaction(p) + ivector_table;
 985         unsigned long imap = bucket->imap;
 986         unsigned int tid;
 987
 988         while (!cpu_online(goal_cpu)) {
 989                 if (++goal_cpu >= NR_CPUS)
 990                         goal_cpu = 0;
 991         }
 992
 993         if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 994                 tid = goal_cpu << 26;
 995                 tid &= IMAP_AID_SAFARI;
 996         } else if (this_is_starfire == 0) {
 997                 tid = goal_cpu << 26;
 998                 tid &= IMAP_TID_UPA;
 999         } else {
1000                 tid = (starfire_translate(imap, goal_cpu) << 26);
1001                 tid &= IMAP_TID_UPA;
1002         }
1003         upa_writel(tid | IMAP_VALID, imap);
1004
1005         while (!cpu_online(goal_cpu)) {
1006                 if (++goal_cpu >= NR_CPUS)
1007                         goal_cpu = 0;
1008         }
1009
1010         return goal_cpu;
1011 }
1012
1013 /* Called from request_irq. */
1014 static void distribute_irqs(void)
1015 {
1016         unsigned long flags;
1017         int cpu, level;
1018
1019         spin_lock_irqsave(&irq_action_lock, flags);
1020         cpu = 0;
1021
1022         /*
1023          * Skip the timer at [0], and very rare error/power intrs at [15].
1024          * Also level [12], it causes problems on Ex000 systems.
1025          */
1026         for (level = 1; level < NR_IRQS; level++) {
1027                 struct irqaction *p = irq_action[level];
1028                 if (level == 12) continue;
1029                 while(p) {
1030                         cpu = retarget_one_irq(p, cpu);
1031                         p = p->next;
1032                 }
1033         }
1034         spin_unlock_irqrestore(&irq_action_lock, flags);
1035 }
1036 #endif
1037
1038
1039 struct sun5_timer *prom_timers;
1040 static u64 prom_limit0, prom_limit1;
1041
1042 static void map_prom_timers(void)
1043 {
1044         unsigned int addr[3];
1045         int tnode, err;
1046
1047         /* PROM timer node hangs out in the top level of device siblings... */
1048         tnode = prom_finddevice("/counter-timer");
1049
1050         /* Assume if node is not present, PROM uses different tick mechanism
1051          * which we should not care about.
1052          */
1053         if (tnode == 0 || tnode == -1) {
1054                 prom_timers = (struct sun5_timer *) 0;
1055                 return;
1056         }
1057
1058         /* If PROM is really using this, it must be mapped by him. */
1059         err = prom_getproperty(tnode, "address", (char *)addr, sizeof(addr));
1060         if (err == -1) {
1061                 prom_printf("PROM does not have timer mapped, trying to continue.\n");
1062                 prom_timers = (struct sun5_timer *) 0;
1063                 return;
1064         }
1065         prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]);
1066 }
1067
1068 static void kill_prom_timer(void)
1069 {
1070         if (!prom_timers)
1071                 return;
1072
1073         /* Save them away for later. */
1074         prom_limit0 = prom_timers->limit0;
1075         prom_limit1 = prom_timers->limit1;
1076
1077         /* Just as in sun4c/sun4m PROM uses timer which ticks at IRQ 14.
1078          * We turn both off here just to be paranoid.
1079          */
1080         prom_timers->limit0 = 0;
1081         prom_timers->limit1 = 0;
1082
1083         /* Wheee, eat the interrupt packet too... */
1084         __asm__ __volatile__(
1085 "       mov     0x40, %%g2\n"
1086 "       ldxa    [%%g0] %0, %%g1\n"
1087 "       ldxa    [%%g2] %1, %%g1\n"
1088 "       stxa    %%g0, [%%g0] %0\n"
1089 "       membar  #Sync\n"
1090         : /* no outputs */
1091         : "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R)
1092         : "g1", "g2");
1093 }
1094
1095 void enable_prom_timer(void)
1096 {
1097         if (!prom_timers)
1098                 return;
1099
1100         /* Set it to whatever was there before. */
1101         prom_timers->limit1 = prom_limit1;
1102         prom_timers->count1 = 0;
1103         prom_timers->limit0 = prom_limit0;
1104         prom_timers->count0 = 0;
1105 }
1106
1107 void init_irqwork_curcpu(void)
1108 {
1109         register struct irq_work_struct *workp asm("o2");
1110         unsigned long tmp;
1111         int cpu = hard_smp_processor_id();
1112
1113         memset(__irq_work + cpu, 0, sizeof(*workp));
1114
1115         /* Make sure we are called with PSTATE_IE disabled.  */
1116         __asm__ __volatile__("rdpr      %%pstate, %0\n\t"
1117                              : "=r" (tmp));
1118         if (tmp & PSTATE_IE) {
1119                 prom_printf("BUG: init_irqwork_curcpu() called with "
1120                             "PSTATE_IE enabled, bailing.\n");
1121                 __asm__ __volatile__("mov       %%i7, %0\n\t"
1122                                      : "=r" (tmp));
1123                 prom_printf("BUG: Called from %lx\n", tmp);
1124                 prom_halt();
1125         }
1126
1127         /* Set interrupt globals.  */
1128         workp = &__irq_work[cpu];
1129         __asm__ __volatile__(
1130         "rdpr   %%pstate, %0\n\t"
1131         "wrpr   %0, %1, %%pstate\n\t"
1132         "mov    %2, %%g6\n\t"
1133         "wrpr   %0, 0x0, %%pstate\n\t"
1134         : "=&r" (tmp)
1135         : "i" (PSTATE_IG), "r" (workp));
1136 }
1137
1138 /* Only invoked on boot processor. */
1139 void __init init_IRQ(void)
1140 {
1141         map_prom_timers();
1142         kill_prom_timer();
1143         memset(&ivector_table[0], 0, sizeof(ivector_table));
1144
1145         /* We need to clear any IRQ's pending in the soft interrupt
1146          * registers, a spurious one could be left around from the
1147          * PROM timer which we just disabled.
1148          */
1149         clear_softint(get_softint());
1150
1151         /* Now that ivector table is initialized, it is safe
1152          * to receive IRQ vector traps.  We will normally take
1153          * one or two right now, in case some device PROM used
1154          * to boot us wants to speak to us.  We just ignore them.
1155          */
1156         __asm__ __volatile__("rdpr      %%pstate, %%g1\n\t"
1157                              "or        %%g1, %0, %%g1\n\t"
1158                              "wrpr      %%g1, 0x0, %%pstate"
1159                              : /* No outputs */
1160                              : "i" (PSTATE_IE)
1161                              : "g1");
1162 }
1163
1164 static struct proc_dir_entry * root_irq_dir;
1165 static struct proc_dir_entry * irq_dir [NUM_IVECS];
1166
1167 #ifdef CONFIG_SMP
1168
1169 #define HEX_DIGITS 16
1170
1171 static unsigned int parse_hex_value (const char __user *buffer,
1172                 unsigned long count, unsigned long *ret)
1173 {
1174         unsigned char hexnum [HEX_DIGITS];
1175         unsigned long value;
1176         int i;
1177
1178         if (!count)
1179                 return -EINVAL;
1180         if (count > HEX_DIGITS)
1181                 count = HEX_DIGITS;
1182         if (copy_from_user(hexnum, buffer, count))
1183                 return -EFAULT;
1184
1185         /*
1186          * Parse the first 8 characters as a hex string, any non-hex char
1187          * is end-of-string. '00e1', 'e1', '00E1', 'E1' are all the same.
1188          */
1189         value = 0;
1190
1191         for (i = 0; i < count; i++) {
1192                 unsigned int c = hexnum[i];
1193
1194                 switch (c) {
1195                         case '0' ... '9': c -= '0'; break;
1196                         case 'a' ... 'f': c -= 'a'-10; break;
1197                         case 'A' ... 'F': c -= 'A'-10; break;
1198                 default:
1199                         goto out;
1200                 }
1201                 value = (value << 4) | c;
1202         }
1203 out:
1204         *ret = value;
1205         return 0;
1206 }
1207
1208 static int irq_affinity_read_proc (char *page, char **start, off_t off,
1209                         int count, int *eof, void *data)
1210 {
1211         struct ino_bucket *bp = ivector_table + (long)data;
1212         struct irqaction *ap = bp->irq_info;
1213         cpumask_t mask = get_smpaff_in_irqaction(ap);
1214         int len;
1215
1216         if (cpus_empty(mask))
1217                 mask = cpu_online_map;
1218
1219         len = cpumask_scnprintf(page, count, mask);
1220         if (count - len < 2)
1221                 return -EINVAL;
1222         len += sprintf(page + len, "\n");
1223         return len;
1224 }
1225
1226 static inline void set_intr_affinity(int irq, unsigned long hw_aff)
1227 {
1228         struct ino_bucket *bp = ivector_table + irq;
1229
1230         /* Users specify affinity in terms of hw cpu ids.
1231          * As soon as we do this, handler_irq() might see and take action.
1232          */
1233         put_smpaff_in_irqaction((struct irqaction *)bp->irq_info, hw_aff);
1234
1235         /* Migration is simply done by the next cpu to service this
1236          * interrupt.
1237          */
1238 }
1239
1240 static int irq_affinity_write_proc (struct file *file, const char __user *buffer,
1241                                         unsigned long count, void *data)
1242 {
1243         int irq = (long) data, full_count = count, err;
1244         unsigned long new_value, i;
1245
1246         err = parse_hex_value(buffer, count, &new_value);
1247
1248         /*
1249          * Do not allow disabling IRQs completely - it's a too easy
1250          * way to make the system unusable accidentally :-) At least
1251          * one online CPU still has to be targeted.
1252          */
1253         for (i = 0; i < NR_CPUS; i++) {
1254                 if ((new_value & (1UL << i)) != 0 &&
1255                     !cpu_online(i))
1256                         new_value &= ~(1UL << i);
1257         }
1258
1259         if (!new_value)
1260                 return -EINVAL;
1261
1262         set_intr_affinity(irq, new_value);
1263
1264         return full_count;
1265 }
1266
1267 #endif
1268
1269 #define MAX_NAMELEN 10
1270
1271 static void register_irq_proc (unsigned int irq)
1272 {
1273         char name [MAX_NAMELEN];
1274
1275         if (!root_irq_dir || irq_dir[irq])
1276                 return;
1277
1278         memset(name, 0, MAX_NAMELEN);
1279         sprintf(name, "%x", irq);
1280
1281         /* create /proc/irq/1234 */
1282         irq_dir[irq] = proc_mkdir(name, root_irq_dir);
1283
1284 #ifdef CONFIG_SMP
1285         /* XXX SMP affinity not supported on starfire yet. */
1286         if (this_is_starfire == 0) {
1287                 struct proc_dir_entry *entry;
1288
1289                 /* create /proc/irq/1234/smp_affinity */
1290                 entry = create_proc_entry("smp_affinity", 0600, irq_dir[irq]);
1291
1292                 if (entry) {
1293                         entry->nlink = 1;
1294                         entry->data = (void *)(long)irq;
1295                         entry->read_proc = irq_affinity_read_proc;
1296                         entry->write_proc = irq_affinity_write_proc;
1297                 }
1298         }
1299 #endif
1300 }
1301
1302 void init_irq_proc (void)
1303 {
1304         /* create /proc/irq */
1305         root_irq_dir = proc_mkdir("irq", 0);
1306 }
1307