kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/kthread.h>
  18 #include <linux/rcupdate.h>
  19
  20 #include <asm/irq.h>
  21 /*
  22    - No shared variables, all the data are CPU local.
  23    - If a softirq needs serialization, let it serialize itself
  24      by its own spinlocks.
  25    - Even if softirq is serialized, only local cpu is marked for
  26      execution. Hence, we get something sort of weak cpu binding.
  27      Though it is still not clear, will it result in better locality
  28      or will not.
  29
  30    Examples:
  31    - NET RX softirq. It is multithreaded and does not require
  32      any global serialization.
  33    - NET TX softirq. It kicks software netdevice queues, hence
  34      it is logically serialized per device, but this serialization
  35      is invisible to common code.
  36    - Tasklets: serialized wrt itself.
  37  */
  38
  39 #ifndef __ARCH_IRQ_STAT
  40 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  41 EXPORT_SYMBOL(irq_stat);
  42 #endif
  43
  44 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  45
  46 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  47
  48 /*
  49  * we cannot loop indefinitely here to avoid userspace starvation,
  50  * but we also don't want to introduce a worst case 1/HZ latency
  51  * to the pending events, so lets the scheduler to balance
  52  * the softirq load for us.
  53  */
  54 static inline void wakeup_softirqd(void)
  55 {
  56         /* Interrupts are disabled: no need to stop preemption */
  57         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  58
  59         if (tsk && tsk->state != TASK_RUNNING)
  60                 wake_up_process(tsk);
  61 }
  62
  63 /*
  64  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
  65  * and we fall back to softirqd after that.
  66  *
  67  * This number has been established via experimentation.
  68  * The two things to balance is latency against fairness -
  69  * we want to handle softirqs as soon as possible, but they
  70  * should not be able to lock up the box.
  71  */
  72 #define MAX_SOFTIRQ_RESTART 10
  73
  74 asmlinkage void __do_softirq(void)
  75 {
  76         struct softirq_action *h;
  77         __u32 pending;
  78         int max_restart = MAX_SOFTIRQ_RESTART;
  79         int cpu;
  80
  81         pending = local_softirq_pending();
  82
  83         local_bh_disable();
  84         cpu = smp_processor_id();
  85 restart:
  86         /* Reset the pending bitmask before enabling irqs */
  87         local_softirq_pending() = 0;
  88
  89         local_irq_enable();
  90
  91         h = softirq_vec;
  92
  93         do {
  94                 if (pending & 1) {
  95                         h->action(h);
  96                         rcu_bh_qsctr_inc(cpu);
  97                 }
  98                 h++;
  99                 pending >>= 1;
 100         } while (pending);
 101
 102         local_irq_disable();
 103
 104         pending = local_softirq_pending();
 105         if (pending && --max_restart)
 106                 goto restart;
 107
 108         if (pending)
 109                 wakeup_softirqd();
 110
 111         __local_bh_enable();
 112 }
 113
 114 #ifndef __ARCH_HAS_DO_SOFTIRQ
 115
 116 asmlinkage void do_softirq(void)
 117 {
 118         __u32 pending;
 119         unsigned long flags;
 120
 121         if (in_interrupt())
 122                 return;
 123
 124         local_irq_save(flags);
 125
 126         pending = local_softirq_pending();
 127
 128         if (pending)
 129                 __do_softirq();
 130
 131         local_irq_restore(flags);
 132 }
 133
 134 EXPORT_SYMBOL(do_softirq);
 135
 136 #endif
 137
 138 void local_bh_enable(void)
 139 {
 140         __local_bh_enable();
 141         WARN_ON(irqs_disabled());
 142         if (unlikely(!in_interrupt() &&
 143                      local_softirq_pending()))
 144                 invoke_softirq();
 145         preempt_check_resched();
 146 }
 147 EXPORT_SYMBOL(local_bh_enable);
 148
 149 /*
 150  * This function must run with irqs disabled!
 151  */
 152 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 153 {
 154         __raise_softirq_irqoff(nr);
 155
 156         /*
 157          * If we're in an interrupt or softirq, we're done
 158          * (this also catches softirq-disabled code). We will
 159          * actually run the softirq once we return from
 160          * the irq or softirq.
 161          *
 162          * Otherwise we wake up ksoftirqd to make sure we
 163          * schedule the softirq soon.
 164          */
 165         if (!in_interrupt())
 166                 wakeup_softirqd();
 167 }
 168
 169 EXPORT_SYMBOL(raise_softirq_irqoff);
 170
 171 void fastcall raise_softirq(unsigned int nr)
 172 {
 173         unsigned long flags;
 174
 175         local_irq_save(flags);
 176         raise_softirq_irqoff(nr);
 177         local_irq_restore(flags);
 178 }
 179
 180 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 181 {
 182         softirq_vec[nr].data = data;
 183         softirq_vec[nr].action = action;
 184 }
 185
 186 EXPORT_SYMBOL(open_softirq);
 187
 188 /* Tasklets */
 189 struct tasklet_head
 190 {
 191         struct tasklet_struct *list;
 192 };
 193
 194 /* Some compilers disobey section attribute on statics when not
 195    initialized -- RR */
 196 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 197 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 198
 199 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 200 {
 201         unsigned long flags;
 202
 203         local_irq_save(flags);
 204         t->next = __get_cpu_var(tasklet_vec).list;
 205         __get_cpu_var(tasklet_vec).list = t;
 206         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 207         local_irq_restore(flags);
 208 }
 209
 210 EXPORT_SYMBOL(__tasklet_schedule);
 211
 212 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 213 {
 214         unsigned long flags;
 215
 216         local_irq_save(flags);
 217         t->next = __get_cpu_var(tasklet_hi_vec).list;
 218         __get_cpu_var(tasklet_hi_vec).list = t;
 219         raise_softirq_irqoff(HI_SOFTIRQ);
 220         local_irq_restore(flags);
 221 }
 222
 223 EXPORT_SYMBOL(__tasklet_hi_schedule);
 224
 225 static void tasklet_action(struct softirq_action *a)
 226 {
 227         struct tasklet_struct *list;
 228
 229         local_irq_disable();
 230         list = __get_cpu_var(tasklet_vec).list;
 231         __get_cpu_var(tasklet_vec).list = NULL;
 232         local_irq_enable();
 233
 234         while (list) {
 235                 struct tasklet_struct *t = list;
 236
 237                 list = list->next;
 238
 239                 if (tasklet_trylock(t)) {
 240                         if (!atomic_read(&t->count)) {
 241                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 242                                         BUG();
 243                                 t->func(t->data);
 244                                 tasklet_unlock(t);
 245                                 continue;
 246                         }
 247                         tasklet_unlock(t);
 248                 }
 249
 250                 local_irq_disable();
 251                 t->next = __get_cpu_var(tasklet_vec).list;
 252                 __get_cpu_var(tasklet_vec).list = t;
 253                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 254                 local_irq_enable();
 255         }
 256 }
 257
 258 static void tasklet_hi_action(struct softirq_action *a)
 259 {
 260         struct tasklet_struct *list;
 261
 262         local_irq_disable();
 263         list = __get_cpu_var(tasklet_hi_vec).list;
 264         __get_cpu_var(tasklet_hi_vec).list = NULL;
 265         local_irq_enable();
 266
 267         while (list) {
 268                 struct tasklet_struct *t = list;
 269
 270                 list = list->next;
 271
 272                 if (tasklet_trylock(t)) {
 273                         if (!atomic_read(&t->count)) {
 274                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 275                                         BUG();
 276                                 t->func(t->data);
 277                                 tasklet_unlock(t);
 278                                 continue;
 279                         }
 280                         tasklet_unlock(t);
 281                 }
 282
 283                 local_irq_disable();
 284                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 285                 __get_cpu_var(tasklet_hi_vec).list = t;
 286                 __raise_softirq_irqoff(HI_SOFTIRQ);
 287                 local_irq_enable();
 288         }
 289 }
 290
 291
 292 void tasklet_init(struct tasklet_struct *t,
 293                   void (*func)(unsigned long), unsigned long data)
 294 {
 295         t->next = NULL;
 296         t->state = 0;
 297         atomic_set(&t->count, 0);
 298         t->func = func;
 299         t->data = data;
 300 }
 301
 302 EXPORT_SYMBOL(tasklet_init);
 303
 304 void tasklet_kill(struct tasklet_struct *t)
 305 {
 306         if (in_interrupt())
 307                 printk("Attempt to kill tasklet from interrupt\n");
 308
 309         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 310                 do
 311                         yield();
 312                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 313         }
 314         tasklet_unlock_wait(t);
 315         clear_bit(TASKLET_STATE_SCHED, &t->state);
 316 }
 317
 318 EXPORT_SYMBOL(tasklet_kill);
 319
 320 void __init softirq_init(void)
 321 {
 322         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 323         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 324 }
 325
 326 static int ksoftirqd(void * __bind_cpu)
 327 {
 328         set_user_nice(current, 19);
 329         current->flags |= PF_NOFREEZE;
 330
 331         set_current_state(TASK_INTERRUPTIBLE);
 332
 333         while (!kthread_should_stop()) {
 334                 if (!local_softirq_pending())
 335                         schedule();
 336
 337                 __set_current_state(TASK_RUNNING);
 338
 339                 while (local_softirq_pending()) {
 340                         /* Preempt disable stops cpu going offline.
 341                            If already offline, we'll be on wrong CPU:
 342                            don't process */
 343                         preempt_disable();
 344                         if (cpu_is_offline((long)__bind_cpu))
 345                                 goto wait_to_die;
 346                         do_softirq();
 347                         preempt_enable();
 348                         cond_resched();
 349                 }
 350
 351                 set_current_state(TASK_INTERRUPTIBLE);
 352         }
 353         __set_current_state(TASK_RUNNING);
 354         return 0;
 355
 356 wait_to_die:
 357         preempt_enable();
 358         /* Wait for kthread_stop */
 359         set_current_state(TASK_INTERRUPTIBLE);
 360         while (!kthread_should_stop()) {
 361                 schedule();
 362                 set_current_state(TASK_INTERRUPTIBLE);
 363         }
 364         __set_current_state(TASK_RUNNING);
 365         return 0;
 366 }
 367
 368 #ifdef CONFIG_HOTPLUG_CPU
 369 /*
 370  * tasklet_kill_immediate is called to remove a tasklet which can already be
 371  * scheduled for execution on @cpu.
 372  *
 373  * Unlike tasklet_kill, this function removes the tasklet
 374  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 375  *
 376  * When this function is called, @cpu must be in the CPU_DEAD state.
 377  */
 378 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 379 {
 380         struct tasklet_struct **i;
 381
 382         BUG_ON(cpu_online(cpu));
 383         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 384
 385         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 386                 return;
 387
 388         /* CPU is dead, so no lock needed. */
 389         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 390                 if (*i == t) {
 391                         *i = t->next;
 392                         return;
 393                 }
 394         }
 395         BUG();
 396 }
 397
 398 static void takeover_tasklets(unsigned int cpu)
 399 {
 400         struct tasklet_struct **i;
 401
 402         /* CPU is dead, so no lock needed. */
 403         local_irq_disable();
 404
 405         /* Find end, append list for that CPU. */
 406         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 407         *i = per_cpu(tasklet_vec, cpu).list;
 408         per_cpu(tasklet_vec, cpu).list = NULL;
 409         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 410
 411         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 412         *i = per_cpu(tasklet_hi_vec, cpu).list;
 413         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 414         raise_softirq_irqoff(HI_SOFTIRQ);
 415
 416         local_irq_enable();
 417 }
 418 #endif /* CONFIG_HOTPLUG_CPU */
 419
 420 static int __devinit cpu_callback(struct notifier_block *nfb,
 421                                   unsigned long action,
 422                                   void *hcpu)
 423 {
 424         int hotcpu = (unsigned long)hcpu;
 425         struct task_struct *p;
 426
 427         switch (action) {
 428         case CPU_UP_PREPARE:
 429                 BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
 430                 BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
 431                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 432                 if (IS_ERR(p)) {
 433                         printk("ksoftirqd for %i failed\n", hotcpu);
 434                         return NOTIFY_BAD;
 435                 }
 436                 kthread_bind(p, hotcpu);
 437                 per_cpu(ksoftirqd, hotcpu) = p;
 438                 break;
 439         case CPU_ONLINE:
 440                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 441                 break;
 442 #ifdef CONFIG_HOTPLUG_CPU
 443         case CPU_UP_CANCELED:
 444                 /* Unbind so it can run.  Fall thru. */
 445                 kthread_bind(per_cpu(ksoftirqd, hotcpu), smp_processor_id());
 446         case CPU_DEAD:
 447                 p = per_cpu(ksoftirqd, hotcpu);
 448                 per_cpu(ksoftirqd, hotcpu) = NULL;
 449                 kthread_stop(p);
 450                 takeover_tasklets(hotcpu);
 451                 break;
 452 #endif /* CONFIG_HOTPLUG_CPU */
 453         }
 454         return NOTIFY_OK;
 455 }
 456
 457 static struct notifier_block __devinitdata cpu_nfb = {
 458         .notifier_call = cpu_callback
 459 };
 460
 461 __init int spawn_ksoftirqd(void)
 462 {
 463         void *cpu = (void *)(long)smp_processor_id();
 464         cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 465         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 466         register_cpu_notifier(&cpu_nfb);
 467         return 0;
 468 }