Force tick interrupt and get rid of softirq magic

This is a backporting from upstream linux, please refer to
http://marc.info/?t=142904577100001&r=1&w=2 for more information, the
corresponding kernel commit is 0ff53d09642204c64842 (tick: sched: Force
tick interrupt and get rid of softirq magic).

The purpose of this patch is, if a tick_sched interrupt is missed, don't
try to raise softirq, instead, we can simply trigger the timer interrupt.
The reason is, softirq requires context switch and has much higher overhead
than interrupt.

Upstream status: backport 0ff53d09642204c64842

Change-Id: I70131fb85189d15972fcaa1a2a2fa190270787f9
Back-port-by: Yunhong Jiang <yunhong.jiang@linux.intel.com>

2 files changed, 31 insertions, 44 deletions

diff --git a/kernel/kernel/time/hrtimer.c b/kernel/kernel/time/hrtimer.c
index 2c6be169b..5d193396e 100644
--- a/kernel/kernel/time/hrtimer.c
+++ b/kernel/kernel/time/hrtimer.c
@@ -583,6 +583,12 @@ static int hrtimer_reprogram(struct hrtimer *timer,
 	if (hrtimer_callback_running(timer))
 		return 0;
 
+        if (base->cpu_base != cpu_base)
+		return 0;
+
+       if (cpu_base->in_hrtirq)
+               return 0;
+
 	/*
 	 * CLOCK_REALTIME timer might be requested with an absolute
 	 * expiry time which is less than base->offset. Nothing wrong
@@ -613,12 +619,11 @@ static int hrtimer_reprogram(struct hrtimer *timer,
 	if (cpu_base->hang_detected)
 		return 0;
 
+	cpu_base->expires_next = expires;
 	/*
 	 * Clockevents returns -ETIME, when the event was in the past.
 	 */
-	res = tick_program_event(expires, 0);
-	if (!IS_ERR_VALUE(res))
-		cpu_base->expires_next = expires;
+	res = tick_program_event(expires, 1);
 	return res;
 }
 
diff --git a/kernel/kernel/time/tick-sched.c b/kernel/kernel/time/tick-sched.c
index b3841ba00..f61dbf202 100644
--- a/kernel/kernel/time/tick-sched.c
+++ b/kernel/kernel/time/tick-sched.c
@@ -576,6 +576,20 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 }
 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 
+static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
+{
+        hrtimer_cancel(&ts->sched_timer);
+        hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
+
+        /* Forward the time to expire in the future */
+        hrtimer_forward(&ts->sched_timer, now, tick_period);
+
+        if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
+                hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
+        else
+                tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
+}
+
 static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 					 ktime_t now, int cpu)
 {
@@ -704,22 +718,16 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 			goto out;
 		}
 
-		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
-			hrtimer_start(&ts->sched_timer, expires,
-				      HRTIMER_MODE_ABS_PINNED);
-			/* Check, if the timer was already in the past */
-			if (hrtimer_active(&ts->sched_timer))
-				goto out;
-		} else if (!tick_program_event(expires, 0))
-				goto out;
-		/*
-		 * We are past the event already. So we crossed a
-		 * jiffie boundary. Update jiffies and raise the
-		 * softirq.
-		 */
-		tick_do_update_jiffies64(ktime_get());
+                if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
+                        hrtimer_start(&ts->sched_timer, expires,
+                                      HRTIMER_MODE_ABS_PINNED);
+                else
+                        tick_program_event(expires, 1);
+       } else {
+               /* Tick is stopped, but required now. Enforce it */
+               tick_nohz_restart(ts, now);
+
 	}
-	raise_softirq_irqoff(TIMER_SOFTIRQ);
 out:
 	ts->next_jiffies = next_jiffies;
 	ts->last_jiffies = last_jiffies;
@@ -880,32 +888,6 @@ ktime_t tick_nohz_get_sleep_length(void)
 	return ts->sleep_length;
 }
 
-static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
-{
-	hrtimer_cancel(&ts->sched_timer);
-	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
-
-	while (1) {
-		/* Forward the time to expire in the future */
-		hrtimer_forward(&ts->sched_timer, now, tick_period);
-
-		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
-			hrtimer_start_expires(&ts->sched_timer,
-					      HRTIMER_MODE_ABS_PINNED);
-			/* Check, if the timer was already in the past */
-			if (hrtimer_active(&ts->sched_timer))
-				break;
-		} else {
-			if (!tick_program_event(
-				hrtimer_get_expires(&ts->sched_timer), 0))
-				break;
-		}
-		/* Reread time and update jiffies */
-		now = ktime_get();
-		tick_do_update_jiffies64(now);
-	}
-}
-
 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
 {
 	/* Update jiffies first */