From 6d8b23062207c761e4606d1eae0e43cf36141a44 Mon Sep 17 00:00:00 2001 From: Yunhong Jiang Date: Tue, 27 Oct 2015 00:54:35 -0400 Subject: Force tick interrupt and get rid of softirq magic This is a backporting from upstream linux, please refer to http://marc.info/?t=142904577100001&r=1&w=2 for more information, the corresponding kernel commit is 0ff53d09642204c64842 (tick: sched: Force tick interrupt and get rid of softirq magic). The purpose of this patch is, if a tick_sched interrupt is missed, don't try to raise softirq, instead, we can simply trigger the timer interrupt. The reason is, softirq requires context switch and has much higher overhead than interrupt. Upstream status: backport 0ff53d09642204c64842 Change-Id: I70131fb85189d15972fcaa1a2a2fa190270787f9 Back-port-by: Yunhong Jiang --- kernel/kernel/time/hrtimer.c | 11 +++++-- kernel/kernel/time/tick-sched.c | 64 +++++++++++++++-------------------------- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/kernel/kernel/time/hrtimer.c b/kernel/kernel/time/hrtimer.c index 2c6be169b..5d193396e 100644 --- a/kernel/kernel/time/hrtimer.c +++ b/kernel/kernel/time/hrtimer.c @@ -583,6 +583,12 @@ static int hrtimer_reprogram(struct hrtimer *timer, if (hrtimer_callback_running(timer)) return 0; + if (base->cpu_base != cpu_base) + return 0; + + if (cpu_base->in_hrtirq) + return 0; + /* * CLOCK_REALTIME timer might be requested with an absolute * expiry time which is less than base->offset. Nothing wrong @@ -613,12 +619,11 @@ static int hrtimer_reprogram(struct hrtimer *timer, if (cpu_base->hang_detected) return 0; + cpu_base->expires_next = expires; /* * Clockevents returns -ETIME, when the event was in the past. */ - res = tick_program_event(expires, 0); - if (!IS_ERR_VALUE(res)) - cpu_base->expires_next = expires; + res = tick_program_event(expires, 1); return res; } diff --git a/kernel/kernel/time/tick-sched.c b/kernel/kernel/time/tick-sched.c index b3841ba00..f61dbf202 100644 --- a/kernel/kernel/time/tick-sched.c +++ b/kernel/kernel/time/tick-sched.c @@ -576,6 +576,20 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); +static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) +{ + hrtimer_cancel(&ts->sched_timer); + hrtimer_set_expires(&ts->sched_timer, ts->last_tick); + + /* Forward the time to expire in the future */ + hrtimer_forward(&ts->sched_timer, now, tick_period); + + if (ts->nohz_mode == NOHZ_MODE_HIGHRES) + hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED); + else + tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); +} + static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, ktime_t now, int cpu) { @@ -704,22 +718,16 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, goto out; } - if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { - hrtimer_start(&ts->sched_timer, expires, - HRTIMER_MODE_ABS_PINNED); - /* Check, if the timer was already in the past */ - if (hrtimer_active(&ts->sched_timer)) - goto out; - } else if (!tick_program_event(expires, 0)) - goto out; - /* - * We are past the event already. So we crossed a - * jiffie boundary. Update jiffies and raise the - * softirq. - */ - tick_do_update_jiffies64(ktime_get()); + if (ts->nohz_mode == NOHZ_MODE_HIGHRES) + hrtimer_start(&ts->sched_timer, expires, + HRTIMER_MODE_ABS_PINNED); + else + tick_program_event(expires, 1); + } else { + /* Tick is stopped, but required now. Enforce it */ + tick_nohz_restart(ts, now); + } - raise_softirq_irqoff(TIMER_SOFTIRQ); out: ts->next_jiffies = next_jiffies; ts->last_jiffies = last_jiffies; @@ -880,32 +888,6 @@ ktime_t tick_nohz_get_sleep_length(void) return ts->sleep_length; } -static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) -{ - hrtimer_cancel(&ts->sched_timer); - hrtimer_set_expires(&ts->sched_timer, ts->last_tick); - - while (1) { - /* Forward the time to expire in the future */ - hrtimer_forward(&ts->sched_timer, now, tick_period); - - if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { - hrtimer_start_expires(&ts->sched_timer, - HRTIMER_MODE_ABS_PINNED); - /* Check, if the timer was already in the past */ - if (hrtimer_active(&ts->sched_timer)) - break; - } else { - if (!tick_program_event( - hrtimer_get_expires(&ts->sched_timer), 0)) - break; - } - /* Reread time and update jiffies */ - now = ktime_get(); - tick_do_update_jiffies64(now); - } -} - static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) { /* Update jiffies first */ -- cgit 1.2.3-korg