kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/kthread.h>
  18 #include <linux/rcupdate.h>
  19 #include <linux/smp.h>
  20 #include <linux/vs_context.h>
  21
  22 #include <asm/irq.h>
  23 /*
  24    - No shared variables, all the data are CPU local.
  25    - If a softirq needs serialization, let it serialize itself
  26      by its own spinlocks.
  27    - Even if softirq is serialized, only local cpu is marked for
  28      execution. Hence, we get something sort of weak cpu binding.
  29      Though it is still not clear, will it result in better locality
  30      or will not.
  31
  32    Examples:
  33    - NET RX softirq. It is multithreaded and does not require
  34      any global serialization.
  35    - NET TX softirq. It kicks software netdevice queues, hence
  36      it is logically serialized per device, but this serialization
  37      is invisible to common code.
  38    - Tasklets: serialized wrt itself.
  39  */
  40
  41 #ifndef __ARCH_IRQ_STAT
  42 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  43 EXPORT_SYMBOL(irq_stat);
  44 #endif
  45
  46 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  47
  48 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  49
  50 /*
  51  * we cannot loop indefinitely here to avoid userspace starvation,
  52  * but we also don't want to introduce a worst case 1/HZ latency
  53  * to the pending events, so lets the scheduler to balance
  54  * the softirq load for us.
  55  */
  56 static inline void wakeup_softirqd(void)
  57 {
  58         /* Interrupts are disabled: no need to stop preemption */
  59         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  60
  61         if (tsk && tsk->state != TASK_RUNNING)
  62                 wake_up_process(tsk);
  63 }
  64
  65 /*
  66  * This one is for softirq.c-internal use,
  67  * where hardirqs are disabled legitimately:
  68  */
  69 #ifdef CONFIG_TRACE_IRQFLAGS
  70 static void __local_bh_disable(unsigned long ip)
  71 {
  72         unsigned long flags;
  73
  74         WARN_ON_ONCE(in_irq());
  75
  76         raw_local_irq_save(flags);
  77         add_preempt_count(SOFTIRQ_OFFSET);
  78         /*
  79          * Were softirqs turned off above:
  80          */
  81         if (softirq_count() == SOFTIRQ_OFFSET)
  82                 trace_softirqs_off(ip);
  83         raw_local_irq_restore(flags);
  84 }
  85 #else /* !CONFIG_TRACE_IRQFLAGS */
  86 static inline void __local_bh_disable(unsigned long ip)
  87 {
  88         add_preempt_count(SOFTIRQ_OFFSET);
  89         barrier();
  90 }
  91 #endif /* CONFIG_TRACE_IRQFLAGS */
  92
  93 void local_bh_disable(void)
  94 {
  95         __local_bh_disable((unsigned long)__builtin_return_address(0));
  96 }
  97
  98 EXPORT_SYMBOL(local_bh_disable);
  99
 100 void __local_bh_enable(void)
 101 {
 102         WARN_ON_ONCE(in_irq());
 103
 104         /*
 105          * softirqs should never be enabled by __local_bh_enable(),
 106          * it always nests inside local_bh_enable() sections:
 107          */
 108         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 109
 110         sub_preempt_count(SOFTIRQ_OFFSET);
 111 }
 112 EXPORT_SYMBOL_GPL(__local_bh_enable);
 113
 114 /*
 115  * Special-case - softirqs can safely be enabled in
 116  * cond_resched_softirq(), or by __do_softirq(),
 117  * without processing still-pending softirqs:
 118  */
 119 void _local_bh_enable(void)
 120 {
 121         WARN_ON_ONCE(in_irq());
 122         WARN_ON_ONCE(!irqs_disabled());
 123
 124         if (softirq_count() == SOFTIRQ_OFFSET)
 125                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 126         sub_preempt_count(SOFTIRQ_OFFSET);
 127 }
 128
 129 EXPORT_SYMBOL(_local_bh_enable);
 130
 131 void local_bh_enable(void)
 132 {
 133 #ifdef CONFIG_TRACE_IRQFLAGS
 134         unsigned long flags;
 135
 136         WARN_ON_ONCE(in_irq());
 137 #endif
 138         WARN_ON_ONCE(irqs_disabled());
 139
 140 #ifdef CONFIG_TRACE_IRQFLAGS
 141         local_irq_save(flags);
 142 #endif
 143         /*
 144          * Are softirqs going to be turned on now:
 145          */
 146         if (softirq_count() == SOFTIRQ_OFFSET)
 147                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 148         /*
 149          * Keep preemption disabled until we are done with
 150          * softirq processing:
 151          */
 152         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 153
 154         if (unlikely(!in_interrupt() && local_softirq_pending()))
 155                 do_softirq();
 156
 157         dec_preempt_count();
 158 #ifdef CONFIG_TRACE_IRQFLAGS
 159         local_irq_restore(flags);
 160 #endif
 161         preempt_check_resched();
 162 }
 163 EXPORT_SYMBOL(local_bh_enable);
 164
 165 void local_bh_enable_ip(unsigned long ip)
 166 {
 167 #ifdef CONFIG_TRACE_IRQFLAGS
 168         unsigned long flags;
 169
 170         WARN_ON_ONCE(in_irq());
 171
 172         local_irq_save(flags);
 173 #endif
 174         /*
 175          * Are softirqs going to be turned on now:
 176          */
 177         if (softirq_count() == SOFTIRQ_OFFSET)
 178                 trace_softirqs_on(ip);
 179         /*
 180          * Keep preemption disabled until we are done with
 181          * softirq processing:
 182          */
 183         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 184
 185         if (unlikely(!in_interrupt() && local_softirq_pending()))
 186                 do_softirq();
 187
 188         dec_preempt_count();
 189 #ifdef CONFIG_TRACE_IRQFLAGS
 190         local_irq_restore(flags);
 191 #endif
 192         preempt_check_resched();
 193 }
 194 EXPORT_SYMBOL(local_bh_enable_ip);
 195
 196 /*
 197  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 198  * and we fall back to softirqd after that.
 199  *
 200  * This number has been established via experimentation.
 201  * The two things to balance is latency against fairness -
 202  * we want to handle softirqs as soon as possible, but they
 203  * should not be able to lock up the box.
 204  */
 205 #define MAX_SOFTIRQ_RESTART 10
 206
 207 asmlinkage void __do_softirq(void)
 208 {
 209         struct softirq_action *h;
 210         __u32 pending;
 211         int max_restart = MAX_SOFTIRQ_RESTART;
 212         int cpu;
 213
 214         pending = local_softirq_pending();
 215         account_system_vtime(current);
 216
 217         __local_bh_disable((unsigned long)__builtin_return_address(0));
 218         trace_softirq_enter();
 219
 220         cpu = smp_processor_id();
 221 restart:
 222         /* Reset the pending bitmask before enabling irqs */
 223         set_softirq_pending(0);
 224
 225         local_irq_enable();
 226
 227         h = softirq_vec;
 228
 229         do {
 230                 if (pending & 1) {
 231                         h->action(h);
 232                         rcu_bh_qsctr_inc(cpu);
 233                 }
 234                 h++;
 235                 pending >>= 1;
 236         } while (pending);
 237
 238         local_irq_disable();
 239
 240         pending = local_softirq_pending();
 241         if (pending && --max_restart)
 242                 goto restart;
 243
 244         if (pending)
 245                 wakeup_softirqd();
 246
 247         trace_softirq_exit();
 248
 249         account_system_vtime(current);
 250         _local_bh_enable();
 251 }
 252
 253 #ifndef __ARCH_HAS_DO_SOFTIRQ
 254
 255 asmlinkage void do_softirq(void)
 256 {
 257         __u32 pending;
 258         unsigned long flags;
 259
 260         if (in_interrupt())
 261                 return;
 262
 263         local_irq_save(flags);
 264
 265         pending = local_softirq_pending();
 266
 267         if (pending)
 268                 __do_softirq();
 269
 270         local_irq_restore(flags);
 271 }
 272
 273 EXPORT_SYMBOL(do_softirq);
 274
 275 #endif
 276
 277 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 278 # define invoke_softirq()       __do_softirq()
 279 #else
 280 # define invoke_softirq()       do_softirq()
 281 #endif
 282
 283 /*
 284  * Exit an interrupt context. Process softirqs if needed and possible:
 285  */
 286 void irq_exit(void)
 287 {
 288         account_system_vtime(current);
 289         trace_hardirq_exit();
 290         sub_preempt_count(IRQ_EXIT_OFFSET);
 291         if (!in_interrupt() && local_softirq_pending())
 292                 invoke_softirq();
 293         preempt_enable_no_resched();
 294 }
 295
 296 /*
 297  * This function must run with irqs disabled!
 298  */
 299 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 300 {
 301         __raise_softirq_irqoff(nr);
 302
 303         /*
 304          * If we're in an interrupt or softirq, we're done
 305          * (this also catches softirq-disabled code). We will
 306          * actually run the softirq once we return from
 307          * the irq or softirq.
 308          *
 309          * Otherwise we wake up ksoftirqd to make sure we
 310          * schedule the softirq soon.
 311          */
 312         if (!in_interrupt())
 313                 wakeup_softirqd();
 314 }
 315
 316 EXPORT_SYMBOL(raise_softirq_irqoff);
 317
 318 void fastcall raise_softirq(unsigned int nr)
 319 {
 320         unsigned long flags;
 321
 322         local_irq_save(flags);
 323         raise_softirq_irqoff(nr);
 324         local_irq_restore(flags);
 325 }
 326
 327 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 328 {
 329         softirq_vec[nr].data = data;
 330         softirq_vec[nr].action = action;
 331 }
 332
 333 /* Tasklets */
 334 struct tasklet_head
 335 {
 336         struct tasklet_struct *list;
 337 };
 338
 339 /* Some compilers disobey section attribute on statics when not
 340    initialized -- RR */
 341 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 342 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 343
 344 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 345 {
 346         unsigned long flags;
 347
 348         local_irq_save(flags);
 349         t->next = __get_cpu_var(tasklet_vec).list;
 350         __get_cpu_var(tasklet_vec).list = t;
 351         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 352         local_irq_restore(flags);
 353 }
 354
 355 EXPORT_SYMBOL(__tasklet_schedule);
 356
 357 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 358 {
 359         unsigned long flags;
 360
 361         local_irq_save(flags);
 362         t->next = __get_cpu_var(tasklet_hi_vec).list;
 363         __get_cpu_var(tasklet_hi_vec).list = t;
 364         raise_softirq_irqoff(HI_SOFTIRQ);
 365         local_irq_restore(flags);
 366 }
 367
 368 EXPORT_SYMBOL(__tasklet_hi_schedule);
 369
 370 static void tasklet_action(struct softirq_action *a)
 371 {
 372         struct tasklet_struct *list;
 373
 374         local_irq_disable();
 375         list = __get_cpu_var(tasklet_vec).list;
 376         __get_cpu_var(tasklet_vec).list = NULL;
 377         local_irq_enable();
 378
 379         while (list) {
 380                 struct tasklet_struct *t = list;
 381
 382                 list = list->next;
 383
 384                 if (tasklet_trylock(t)) {
 385                         if (!atomic_read(&t->count)) {
 386                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 387                                         BUG();
 388                                 t->func(t->data);
 389                                 tasklet_unlock(t);
 390                                 continue;
 391                         }
 392                         tasklet_unlock(t);
 393                 }
 394
 395                 local_irq_disable();
 396                 t->next = __get_cpu_var(tasklet_vec).list;
 397                 __get_cpu_var(tasklet_vec).list = t;
 398                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 399                 local_irq_enable();
 400         }
 401 }
 402
 403 static void tasklet_hi_action(struct softirq_action *a)
 404 {
 405         struct tasklet_struct *list;
 406
 407         local_irq_disable();
 408         list = __get_cpu_var(tasklet_hi_vec).list;
 409         __get_cpu_var(tasklet_hi_vec).list = NULL;
 410         local_irq_enable();
 411
 412         while (list) {
 413                 struct tasklet_struct *t = list;
 414
 415                 list = list->next;
 416
 417                 if (tasklet_trylock(t)) {
 418                         if (!atomic_read(&t->count)) {
 419                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 420                                         BUG();
 421                                 t->func(t->data);
 422                                 tasklet_unlock(t);
 423                                 continue;
 424                         }
 425                         tasklet_unlock(t);
 426                 }
 427
 428                 local_irq_disable();
 429                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 430                 __get_cpu_var(tasklet_hi_vec).list = t;
 431                 __raise_softirq_irqoff(HI_SOFTIRQ);
 432                 local_irq_enable();
 433         }
 434 }
 435
 436
 437 void tasklet_init(struct tasklet_struct *t,
 438                   void (*func)(unsigned long), unsigned long data)
 439 {
 440         t->next = NULL;
 441         t->state = 0;
 442         atomic_set(&t->count, 0);
 443         t->func = func;
 444         t->data = data;
 445 }
 446
 447 EXPORT_SYMBOL(tasklet_init);
 448
 449 void tasklet_kill(struct tasklet_struct *t)
 450 {
 451         if (in_interrupt())
 452                 printk("Attempt to kill tasklet from interrupt\n");
 453
 454         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 455                 do
 456                         yield();
 457                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 458         }
 459         tasklet_unlock_wait(t);
 460         clear_bit(TASKLET_STATE_SCHED, &t->state);
 461 }
 462
 463 EXPORT_SYMBOL(tasklet_kill);
 464
 465 void __init softirq_init(void)
 466 {
 467         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 468         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 469 }
 470
 471 static int ksoftirqd(void * __bind_cpu)
 472 {
 473         set_user_nice(current, 19);
 474         current->flags |= PF_NOFREEZE;
 475
 476         set_current_state(TASK_INTERRUPTIBLE);
 477
 478         while (!kthread_should_stop()) {
 479                 preempt_disable();
 480                 if (!local_softirq_pending()) {
 481                         preempt_enable_no_resched();
 482                         schedule();
 483                         preempt_disable();
 484                 }
 485
 486                 __set_current_state(TASK_RUNNING);
 487
 488                 while (local_softirq_pending()) {
 489                         /* Preempt disable stops cpu going offline.
 490                            If already offline, we'll be on wrong CPU:
 491                            don't process */
 492                         if (cpu_is_offline((long)__bind_cpu))
 493                                 goto wait_to_die;
 494                         do_softirq();
 495                         preempt_enable_no_resched();
 496                         cond_resched();
 497                         preempt_disable();
 498                 }
 499                 preempt_enable();
 500                 set_current_state(TASK_INTERRUPTIBLE);
 501         }
 502         __set_current_state(TASK_RUNNING);
 503         return 0;
 504
 505 wait_to_die:
 506         preempt_enable();
 507         /* Wait for kthread_stop */
 508         set_current_state(TASK_INTERRUPTIBLE);
 509         while (!kthread_should_stop()) {
 510                 schedule();
 511                 set_current_state(TASK_INTERRUPTIBLE);
 512         }
 513         __set_current_state(TASK_RUNNING);
 514         return 0;
 515 }
 516
 517 #ifdef CONFIG_HOTPLUG_CPU
 518 /*
 519  * tasklet_kill_immediate is called to remove a tasklet which can already be
 520  * scheduled for execution on @cpu.
 521  *
 522  * Unlike tasklet_kill, this function removes the tasklet
 523  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 524  *
 525  * When this function is called, @cpu must be in the CPU_DEAD state.
 526  */
 527 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 528 {
 529         struct tasklet_struct **i;
 530
 531         BUG_ON(cpu_online(cpu));
 532         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 533
 534         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 535                 return;
 536
 537         /* CPU is dead, so no lock needed. */
 538         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 539                 if (*i == t) {
 540                         *i = t->next;
 541                         return;
 542                 }
 543         }
 544         BUG();
 545 }
 546
 547 static void takeover_tasklets(unsigned int cpu)
 548 {
 549         struct tasklet_struct **i;
 550
 551         /* CPU is dead, so no lock needed. */
 552         local_irq_disable();
 553
 554         /* Find end, append list for that CPU. */
 555         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 556         *i = per_cpu(tasklet_vec, cpu).list;
 557         per_cpu(tasklet_vec, cpu).list = NULL;
 558         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 559
 560         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 561         *i = per_cpu(tasklet_hi_vec, cpu).list;
 562         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 563         raise_softirq_irqoff(HI_SOFTIRQ);
 564
 565         local_irq_enable();
 566 }
 567 #endif /* CONFIG_HOTPLUG_CPU */
 568
 569 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 570                                   unsigned long action,
 571                                   void *hcpu)
 572 {
 573         int hotcpu = (unsigned long)hcpu;
 574         struct task_struct *p;
 575
 576         switch (action) {
 577         case CPU_UP_PREPARE:
 578                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 579                 if (IS_ERR(p)) {
 580                         printk("ksoftirqd for %i failed\n", hotcpu);
 581                         return NOTIFY_BAD;
 582                 }
 583                 kthread_bind(p, hotcpu);
 584                 per_cpu(ksoftirqd, hotcpu) = p;
 585                 break;
 586         case CPU_ONLINE:
 587                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 588                 break;
 589 #ifdef CONFIG_HOTPLUG_CPU
 590         case CPU_UP_CANCELED:
 591                 if (!per_cpu(ksoftirqd, hotcpu))
 592                         break;
 593                 /* Unbind so it can run.  Fall thru. */
 594                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 595                              any_online_cpu(cpu_online_map));
 596         case CPU_DEAD:
 597                 p = per_cpu(ksoftirqd, hotcpu);
 598                 per_cpu(ksoftirqd, hotcpu) = NULL;
 599                 kthread_stop(p);
 600                 takeover_tasklets(hotcpu);
 601                 break;
 602 #endif /* CONFIG_HOTPLUG_CPU */
 603         }
 604         return NOTIFY_OK;
 605 }
 606
 607 static struct notifier_block __cpuinitdata cpu_nfb = {
 608         .notifier_call = cpu_callback
 609 };
 610
 611 __init int spawn_ksoftirqd(void)
 612 {
 613         void *cpu = (void *)(long)smp_processor_id();
 614         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 615
 616         BUG_ON(err == NOTIFY_BAD);
 617         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 618         register_cpu_notifier(&cpu_nfb);
 619         return 0;
 620 }
 621
 622 #ifdef CONFIG_SMP
 623 /*
 624  * Call a function on all processors
 625  */
 626 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
 627 {
 628         int ret = 0;
 629
 630         preempt_disable();
 631         ret = smp_call_function(func, info, retry, wait);
 632         local_irq_disable();
 633         func(info);
 634         local_irq_enable();
 635         preempt_enable();
 636         return ret;
 637 }
 638 EXPORT_SYMBOL(on_each_cpu);
 639 #endif