kernel/linux/kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/kthread.h>
  18
  19 #include <asm/irq.h>
  20
  21 #include <linux/syscalls.h>
  22
  23 /*
  24    - No shared variables, all the data are CPU local.
  25    - If a softirq needs serialization, let it serialize itself
  26      by its own spinlocks.
  27    - Even if softirq is serialized, only local cpu is marked for
  28      execution. Hence, we get something sort of weak cpu binding.
  29      Though it is still not clear, will it result in better locality
  30      or will not.
  31
  32    Examples:
  33    - NET RX softirq. It is multithreaded and does not require
  34      any global serialization.
  35    - NET TX softirq. It kicks software netdevice queues, hence
  36      it is logically serialized per device, but this serialization
  37      is invisible to common code.
  38    - Tasklets: serialized wrt itself.
  39  */
  40
  41 #ifndef __ARCH_IRQ_STAT
  42 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  43 EXPORT_SYMBOL(irq_stat);
  44 #endif
  45
  46 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  47
  48 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  49
  50 int defer_softirqs = 1;
  51
  52 /*
  53  * we cannot loop indefinitely here to avoid userspace starvation,
  54  * but we also don't want to introduce a worst case 1/HZ latency
  55  * to the pending events, so lets the scheduler to balance
  56  * the softirq load for us.
  57  */
  58 static inline void wakeup_softirqd(void)
  59 {
  60         /* Interrupts are disabled: no need to stop preemption */
  61         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  62
  63         if (tsk && tsk->state != TASK_RUNNING)
  64                 wake_up_process(tsk);
  65 }
  66
  67 /*
  68  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
  69  * and we fall back to softirqd after that.
  70  *
  71  * This number has been established via experimentation.
  72  * The two things to balance is latency against fairness -
  73  * we want to handle softirqs as soon as possible, but they
  74  * should not be able to lock up the box.
  75  */
  76 #define MAX_SOFTIRQ_RESTART 10
  77
  78 asmlinkage void ___do_softirq(void)
  79 {
  80         struct softirq_action *h;
  81         __u32 pending, mask;
  82         int max_restart = MAX_SOFTIRQ_RESTART;
  83
  84         pending = local_softirq_pending();
  85
  86         local_bh_disable();
  87 restart:
  88         /* Reset the pending bitmask before enabling irqs */
  89         local_softirq_pending() = 0;
  90
  91         local_irq_enable();
  92
  93         h = softirq_vec;
  94
  95         mask = 1;
  96         do {
  97                 if (pending & mask) {
  98                         pending ^= mask; /* clear the bit */
  99                         h->action(h);
 100                         if (unlikely(defer_softirqs && need_resched())) {
 101                                 /*
 102                                  * We are interrupting softirq processing
 103                                  * to allow for preemption to occur. Add
 104                                  * back any non-processed pending bits to
 105                                  * this CPU's mask:
 106                                  */
 107                                 local_irq_disable();
 108                                 local_softirq_pending() |= pending;
 109                                 local_irq_enable();
 110                                 goto out;
 111                         }
 112                 }
 113                 h++;
 114                 mask <<= 1;
 115         } while (pending);
 116
 117         local_irq_disable();
 118
 119         pending = local_softirq_pending();
 120         if (pending && --max_restart)
 121                 goto restart;
 122
 123         if (pending)
 124                 wakeup_softirqd();
 125
 126 out:
 127         __local_bh_enable();
 128 }
 129
 130 asmlinkage void __do_softirq(void)
 131 {
 132         if (likely(defer_softirqs)) {
 133                 /*
 134                  * 'preempt harder'. Push all softirq processing off
 135                  * to ksoftirqd. This makes softirq related latencies
 136                  * much more predictable since they run from process
 137                  * context instead of hardirq context.
 138                  */
 139                 if (local_softirq_pending())
 140                         wakeup_softirqd();
 141                 return;
 142         }
 143         ___do_softirq();
 144 }
 145
 146 asmlinkage void _do_softirq(void)
 147 {
 148         local_irq_disable();
 149         ___do_softirq();
 150         local_irq_enable();
 151 }
 152
 153
 154 #ifndef __ARCH_HAS_DO_SOFTIRQ
 155
 156 asmlinkage void do_softirq(void)
 157 {
 158         __u32 pending;
 159         unsigned long flags;
 160
 161         if (in_interrupt())
 162                 return;
 163
 164         local_irq_save(flags);
 165
 166         pending = local_softirq_pending();
 167
 168         if (pending)
 169                 __do_softirq();
 170
 171         local_irq_restore(flags);
 172 }
 173
 174 EXPORT_SYMBOL(do_softirq);
 175
 176 #endif
 177
 178 void local_bh_enable(void)
 179 {
 180         __local_bh_enable();
 181         WARN_ON(irqs_disabled());
 182         if (unlikely(!in_interrupt() &&
 183                      local_softirq_pending()))
 184                 invoke_softirq();
 185         preempt_check_resched();
 186 }
 187 EXPORT_SYMBOL(local_bh_enable);
 188
 189 /*
 190  * This function must run with irqs disabled!
 191  */
 192 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 193 {
 194         __raise_softirq_irqoff(nr);
 195
 196         /*
 197          * If we're in an interrupt or softirq, we're done
 198          * (this also catches softirq-disabled code). We will
 199          * actually run the softirq once we return from
 200          * the irq or softirq.
 201          *
 202          * Otherwise we wake up ksoftirqd to make sure we
 203          * schedule the softirq soon.
 204          */
 205         if (!in_interrupt())
 206                 wakeup_softirqd();
 207 }
 208
 209 EXPORT_SYMBOL(raise_softirq_irqoff);
 210
 211 void fastcall raise_softirq(unsigned int nr)
 212 {
 213         unsigned long flags;
 214
 215         local_irq_save(flags);
 216         raise_softirq_irqoff(nr);
 217         local_irq_restore(flags);
 218 }
 219
 220 EXPORT_SYMBOL(raise_softirq);
 221
 222 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 223 {
 224         softirq_vec[nr].data = data;
 225         softirq_vec[nr].action = action;
 226 }
 227
 228 EXPORT_SYMBOL(open_softirq);
 229
 230 /* Tasklets */
 231 struct tasklet_head
 232 {
 233         struct tasklet_struct *list;
 234 };
 235
 236 /* Some compilers disobey section attribute on statics when not
 237    initialized -- RR */
 238 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 239 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 240
 241 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 242 {
 243         unsigned long flags;
 244
 245         local_irq_save(flags);
 246         t->next = __get_cpu_var(tasklet_vec).list;
 247         __get_cpu_var(tasklet_vec).list = t;
 248         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 249         local_irq_restore(flags);
 250 }
 251
 252 EXPORT_SYMBOL(__tasklet_schedule);
 253
 254 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 255 {
 256         unsigned long flags;
 257
 258         local_irq_save(flags);
 259         t->next = __get_cpu_var(tasklet_hi_vec).list;
 260         __get_cpu_var(tasklet_hi_vec).list = t;
 261         raise_softirq_irqoff(HI_SOFTIRQ);
 262         local_irq_restore(flags);
 263 }
 264
 265 EXPORT_SYMBOL(__tasklet_hi_schedule);
 266
 267 static void tasklet_action(struct softirq_action *a)
 268 {
 269         struct tasklet_struct *list;
 270
 271         local_irq_disable();
 272         list = __get_cpu_var(tasklet_vec).list;
 273         __get_cpu_var(tasklet_vec).list = NULL;
 274         local_irq_enable();
 275
 276         while (list) {
 277                 struct tasklet_struct *t = list;
 278
 279                 list = list->next;
 280
 281                 if (tasklet_trylock(t)) {
 282                         if (!atomic_read(&t->count)) {
 283                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 284                                         BUG();
 285                                 t->func(t->data);
 286                                 tasklet_unlock(t);
 287                                 continue;
 288                         }
 289                         tasklet_unlock(t);
 290                 }
 291
 292                 local_irq_disable();
 293                 t->next = __get_cpu_var(tasklet_vec).list;
 294                 __get_cpu_var(tasklet_vec).list = t;
 295                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 296                 local_irq_enable();
 297         }
 298 }
 299
 300 static void tasklet_hi_action(struct softirq_action *a)
 301 {
 302         struct tasklet_struct *list;
 303
 304         local_irq_disable();
 305         list = __get_cpu_var(tasklet_hi_vec).list;
 306         __get_cpu_var(tasklet_hi_vec).list = NULL;
 307         local_irq_enable();
 308
 309         while (list) {
 310                 struct tasklet_struct *t = list;
 311
 312                 list = list->next;
 313
 314                 if (tasklet_trylock(t)) {
 315                         if (!atomic_read(&t->count)) {
 316                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 317                                         BUG();
 318                                 t->func(t->data);
 319                                 tasklet_unlock(t);
 320                                 continue;
 321                         }
 322                         tasklet_unlock(t);
 323                 }
 324
 325                 local_irq_disable();
 326                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 327                 __get_cpu_var(tasklet_hi_vec).list = t;
 328                 __raise_softirq_irqoff(HI_SOFTIRQ);
 329                 local_irq_enable();
 330         }
 331 }
 332
 333
 334 void tasklet_init(struct tasklet_struct *t,
 335                   void (*func)(unsigned long), unsigned long data)
 336 {
 337         t->next = NULL;
 338         t->state = 0;
 339         atomic_set(&t->count, 0);
 340         t->func = func;
 341         t->data = data;
 342 }
 343
 344 EXPORT_SYMBOL(tasklet_init);
 345
 346 void tasklet_kill(struct tasklet_struct *t)
 347 {
 348         if (in_interrupt())
 349                 printk("Attempt to kill tasklet from interrupt\n");
 350
 351         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 352                 do
 353                         yield();
 354                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 355         }
 356         tasklet_unlock_wait(t);
 357         clear_bit(TASKLET_STATE_SCHED, &t->state);
 358 }
 359
 360 EXPORT_SYMBOL(tasklet_kill);
 361
 362 void __init softirq_init(void)
 363 {
 364         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 365         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 366 }
 367
 368 static int ksoftirqd(void * __bind_cpu)
 369 {
 370 #if 1
 371         set_user_nice(current, -5);
 372         current->flags |= PF_NOFREEZE;
 373 #else /* alternative if we want more priority for ksoftirqd */
 374         struct sched_param param = { .sched_priority = MAX_RT_PRIO/4-1 };
 375         param.sched_priority = 1;
 376         sys_sched_setscheduler(current->pid, SCHED_RR, &param);
 377         current->flags |= PF_NOFREEZE;
 378 #endif
 379
 380         set_current_state(TASK_INTERRUPTIBLE);
 381
 382         while (!kthread_should_stop()) {
 383                 if (!local_softirq_pending())
 384                         schedule();
 385
 386                 __set_current_state(TASK_RUNNING);
 387
 388                 while (local_softirq_pending()) {
 389                         /* Preempt disable stops cpu going offline.
 390                            If already offline, we'll be on wrong CPU:
 391                            don't process */
 392                         preempt_disable();
 393                         if (cpu_is_offline((long)__bind_cpu))
 394                                 goto wait_to_die;
 395                         _do_softirq();
 396                         preempt_enable();
 397                         cond_resched();
 398                 }
 399
 400                 set_current_state(TASK_INTERRUPTIBLE);
 401         }
 402         __set_current_state(TASK_RUNNING);
 403         return 0;
 404
 405 wait_to_die:
 406         preempt_enable();
 407         /* Wait for kthread_stop */
 408         set_current_state(TASK_INTERRUPTIBLE);
 409         while (!kthread_should_stop()) {
 410                 schedule();
 411                 set_current_state(TASK_INTERRUPTIBLE);
 412         }
 413         __set_current_state(TASK_RUNNING);
 414         return 0;
 415 }
 416
 417 #ifdef CONFIG_HOTPLUG_CPU
 418 /*
 419  * tasklet_kill_immediate is called to remove a tasklet which can already be
 420  * scheduled for execution on @cpu.
 421  *
 422  * Unlike tasklet_kill, this function removes the tasklet
 423  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 424  *
 425  * When this function is called, @cpu must be in the CPU_DEAD state.
 426  */
 427 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 428 {
 429         struct tasklet_struct **i;
 430
 431         BUG_ON(cpu_online(cpu));
 432         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 433
 434         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 435                 return;
 436
 437         /* CPU is dead, so no lock needed. */
 438         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 439                 if (*i == t) {
 440                         *i = t->next;
 441                         return;
 442                 }
 443         }
 444         BUG();
 445 }
 446
 447 static void takeover_tasklets(unsigned int cpu)
 448 {
 449         struct tasklet_struct **i;
 450
 451         /* CPU is dead, so no lock needed. */
 452         local_irq_disable();
 453
 454         /* Find end, append list for that CPU. */
 455         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 456         *i = per_cpu(tasklet_vec, cpu).list;
 457         per_cpu(tasklet_vec, cpu).list = NULL;
 458         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 459
 460         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 461         *i = per_cpu(tasklet_hi_vec, cpu).list;
 462         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 463         raise_softirq_irqoff(HI_SOFTIRQ);
 464
 465         local_irq_enable();
 466 }
 467 #endif /* CONFIG_HOTPLUG_CPU */
 468
 469 static int __devinit cpu_callback(struct notifier_block *nfb,
 470                                   unsigned long action,
 471                                   void *hcpu)
 472 {
 473         int hotcpu = (unsigned long)hcpu;
 474         struct task_struct *p;
 475
 476         switch (action) {
 477         case CPU_UP_PREPARE:
 478                 BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
 479                 BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
 480                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 481                 if (IS_ERR(p)) {
 482                         printk("ksoftirqd for %i failed\n", hotcpu);
 483                         return NOTIFY_BAD;
 484                 }
 485                 kthread_bind(p, hotcpu);
 486                 per_cpu(ksoftirqd, hotcpu) = p;
 487                 break;
 488         case CPU_ONLINE:
 489                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 490                 break;
 491 #ifdef CONFIG_HOTPLUG_CPU
 492         case CPU_UP_CANCELED:
 493                 /* Unbind so it can run.  Fall thru. */
 494                 kthread_bind(per_cpu(ksoftirqd, hotcpu), smp_processor_id());
 495         case CPU_DEAD:
 496                 p = per_cpu(ksoftirqd, hotcpu);
 497                 per_cpu(ksoftirqd, hotcpu) = NULL;
 498                 kthread_stop(p);
 499                 takeover_tasklets(hotcpu);
 500                 break;
 501 #endif /* CONFIG_HOTPLUG_CPU */
 502         }
 503         return NOTIFY_OK;
 504 }
 505
 506 static struct notifier_block __devinitdata cpu_nfb = {
 507         .notifier_call = cpu_callback
 508 };
 509
 510 __init int spawn_ksoftirqd(void)
 511 {
 512         void *cpu = (void *)(long)smp_processor_id();
 513         cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 514         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 515         register_cpu_notifier(&cpu_nfb);
 516         return 0;
 517 }