Merge branch 'linux-2.6'

[powerpc.git] / kernel / time / tick-sched.c

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 99d35e2..3483e6c 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -21,6 +21,8 @@
 #include <linux/sched.h>
 #include <linux/tick.h>
 
+#include <asm/irq_regs.h>
+
 #include "tick-internal.h"
 
 /*
@@ -33,6 +35,11 @@ static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
  */
 static ktime_t last_jiffies_update;
 
+struct tick_sched *tick_get_tick_sched(int cpu)
+{
+       return &per_cpu(tick_cpu_sched, cpu);
+}
+
 /*
  * Must be called with interrupts disabled !
  */
@@ -160,7 +167,9 @@ void tick_nohz_stop_sched_tick(void)
                goto end;
 
        cpu = smp_processor_id();
-       BUG_ON(local_softirq_pending());
+       if (unlikely(local_softirq_pending()))
+               printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
+                      local_softirq_pending());
 
        now = ktime_get();
        /*
@@ -186,19 +195,19 @@ void tick_nohz_stop_sched_tick(void)
        next_jiffies = get_next_timer_interrupt(last_jiffies);
        delta_jiffies = next_jiffies - last_jiffies;
 
+       if (rcu_needs_cpu(cpu))
+               delta_jiffies = 1;
        /*
         * Do not stop the tick, if we are only one off
         * or if the cpu is required for rcu
         */
-       if (!ts->tick_stopped && (delta_jiffies == 1 || rcu_needs_cpu(cpu)))
+       if (!ts->tick_stopped && delta_jiffies == 1)
                goto out;
 
        /* Schedule the tick, if we are at least one jiffie off */
        if ((long)delta_jiffies >= 1) {
 
-               if (rcu_needs_cpu(cpu))
-                       delta_jiffies = 1;
-               else
+               if (delta_jiffies > 1)
                        cpu_set(cpu, nohz_cpu_mask);
                /*
                 * nohz_stop_sched_tick can be called several times before
@@ -208,10 +217,30 @@ void tick_nohz_stop_sched_tick(void)
                 * the scheduler tick in nohz_restart_sched_tick.
                 */
                if (!ts->tick_stopped) {
+                       if (select_nohz_load_balancer(1)) {
+                               /*
+                                * sched tick not stopped!
+                                */
+                               cpu_clear(cpu, nohz_cpu_mask);
+                               goto out;
+                       }
+
                        ts->idle_tick = ts->sched_timer.expires;
                        ts->tick_stopped = 1;
                        ts->idle_jiffies = last_jiffies;
                }
+
+               /*
+                * If this cpu is the one which updates jiffies, then
+                * give up the assignment and let it be taken by the
+                * cpu which runs the tick timer next, which might be
+                * this cpu as well. If we don't drop this here the
+                * jiffies might be stale and do_timer() never
+                * invoked.
+                */
+               if (cpu == tick_do_timer_cpu)
+                       tick_do_timer_cpu = -1;
+
                /*
                 * calculate the expiry time for the next timer wheel
                 * timer
@@ -264,6 +293,7 @@ void tick_nohz_restart_sched_tick(void)
        now = ktime_get();
 
        local_irq_disable();
+       select_nohz_load_balancer(0);
        tick_do_update_jiffies64(now);
        cpu_clear(cpu, nohz_cpu_mask);
 
@@ -329,12 +359,24 @@ static void tick_nohz_handler(struct clock_event_device *dev)
 {
        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
        struct pt_regs *regs = get_irq_regs();
+       int cpu = smp_processor_id();
        ktime_t now = ktime_get();
 
        dev->next_event.tv64 = KTIME_MAX;
 
+       /*
+        * Check if the do_timer duty was dropped. We don't care about
+        * concurrency: This happens only when the cpu in charge went
+        * into a long sleep. If two cpus happen to assign themself to
+        * this duty, then the jiffies update is still serialized by
+        * xtime_lock.
+        */
+       if (unlikely(tick_do_timer_cpu == -1))
+               tick_do_timer_cpu = cpu;
+
        /* Check, if the jiffies need an update */
-       tick_do_update_jiffies64(now);
+       if (tick_do_timer_cpu == cpu)
+               tick_do_update_jiffies64(now);
 
        /*
         * When we are idle and the tick is stopped, we have to touch
@@ -422,9 +464,23 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
        struct hrtimer_cpu_base *base = timer->base->cpu_base;
        struct pt_regs *regs = get_irq_regs();
        ktime_t now = ktime_get();
+       int cpu = smp_processor_id();
+
+#ifdef CONFIG_NO_HZ
+       /*
+        * Check if the do_timer duty was dropped. We don't care about
+        * concurrency: This happens only when the cpu in charge went
+        * into a long sleep. If two cpus happen to assign themself to
+        * this duty, then the jiffies update is still serialized by
+        * xtime_lock.
+        */
+       if (unlikely(tick_do_timer_cpu == -1))
+               tick_do_timer_cpu = cpu;
+#endif
 
        /* Check, if the jiffies need an update */
-       tick_do_update_jiffies64(now);
+       if (tick_do_timer_cpu == cpu)
+               tick_do_update_jiffies64(now);
 
        /*
         * Do not call, when we are not in irq context and have