summaryrefslogtreecommitdiffstats
path: root/kernel/drivers/cpuidle
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/drivers/cpuidle')
-rw-r--r--kernel/drivers/cpuidle/coupled.c31
-rw-r--r--kernel/drivers/cpuidle/cpuidle-at91.c3
-rw-r--r--kernel/drivers/cpuidle/cpuidle-big_little.c8
-rw-r--r--kernel/drivers/cpuidle/cpuidle-calxeda.c18
-rw-r--r--kernel/drivers/cpuidle/cpuidle-mvebu-v7.c46
-rw-r--r--kernel/drivers/cpuidle/cpuidle-powernv.c12
-rw-r--r--kernel/drivers/cpuidle/cpuidle-pseries.c11
-rw-r--r--kernel/drivers/cpuidle/cpuidle-zynq.c3
-rw-r--r--kernel/drivers/cpuidle/cpuidle.c57
-rw-r--r--kernel/drivers/cpuidle/cpuidle.h13
-rw-r--r--kernel/drivers/cpuidle/driver.c4
-rw-r--r--kernel/drivers/cpuidle/governors/menu.c4
12 files changed, 141 insertions, 69 deletions
diff --git a/kernel/drivers/cpuidle/coupled.c b/kernel/drivers/cpuidle/coupled.c
index 7936dce4b..d5657d50a 100644
--- a/kernel/drivers/cpuidle/coupled.c
+++ b/kernel/drivers/cpuidle/coupled.c
@@ -119,7 +119,6 @@ struct cpuidle_coupled {
#define CPUIDLE_COUPLED_NOT_IDLE (-1)
-static DEFINE_MUTEX(cpuidle_coupled_lock);
static DEFINE_PER_CPU(struct call_single_data, cpuidle_coupled_poke_cb);
/*
@@ -176,19 +175,39 @@ void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, atomic_t *a)
/**
* cpuidle_state_is_coupled - check if a state is part of a coupled set
- * @dev: struct cpuidle_device for the current cpu
* @drv: struct cpuidle_driver for the platform
* @state: index of the target state in drv->states
*
* Returns true if the target state is coupled with cpus besides this one
*/
-bool cpuidle_state_is_coupled(struct cpuidle_device *dev,
- struct cpuidle_driver *drv, int state)
+bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state)
{
return drv->states[state].flags & CPUIDLE_FLAG_COUPLED;
}
/**
+ * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
+ * @drv: struct cpuidle_driver for the platform
+ *
+ * Returns 0 for valid state values, a negative error code otherwise:
+ * * -EINVAL if any coupled state(safe_state_index) is wrongly set.
+ */
+int cpuidle_coupled_state_verify(struct cpuidle_driver *drv)
+{
+ int i;
+
+ for (i = drv->state_count - 1; i >= 0; i--) {
+ if (cpuidle_state_is_coupled(drv, i) &&
+ (drv->safe_state_index == i ||
+ drv->safe_state_index < 0 ||
+ drv->safe_state_index >= drv->state_count))
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/**
* cpuidle_coupled_set_ready - mark a cpu as ready
* @coupled: the struct coupled that contains the current cpu
*/
@@ -473,7 +492,7 @@ int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
return entered_state;
}
entered_state = cpuidle_enter_state(dev, drv,
- dev->safe_state_index);
+ drv->safe_state_index);
local_irq_disable();
}
@@ -521,7 +540,7 @@ retry:
}
entered_state = cpuidle_enter_state(dev, drv,
- dev->safe_state_index);
+ drv->safe_state_index);
local_irq_disable();
}
diff --git a/kernel/drivers/cpuidle/cpuidle-at91.c b/kernel/drivers/cpuidle/cpuidle-at91.c
index f2446c78d..9c5853b6c 100644
--- a/kernel/drivers/cpuidle/cpuidle-at91.c
+++ b/kernel/drivers/cpuidle/cpuidle-at91.c
@@ -62,5 +62,4 @@ static struct platform_driver at91_cpuidle_driver = {
},
.probe = at91_cpuidle_probe,
};
-
-module_platform_driver(at91_cpuidle_driver);
+builtin_platform_driver(at91_cpuidle_driver);
diff --git a/kernel/drivers/cpuidle/cpuidle-big_little.c b/kernel/drivers/cpuidle/cpuidle-big_little.c
index 40c34faff..db2ede565 100644
--- a/kernel/drivers/cpuidle/cpuidle-big_little.c
+++ b/kernel/drivers/cpuidle/cpuidle-big_little.c
@@ -108,13 +108,7 @@ static int notrace bl_powerdown_finisher(unsigned long arg)
unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
mcpm_set_entry_vector(cpu, cluster, cpu_resume);
-
- /*
- * Residency value passed to mcpm_cpu_suspend back-end
- * has to be given clear semantics. Set to 0 as a
- * temporary value.
- */
- mcpm_cpu_suspend(0);
+ mcpm_cpu_suspend();
/* return value != 0 means failure */
return 1;
diff --git a/kernel/drivers/cpuidle/cpuidle-calxeda.c b/kernel/drivers/cpuidle/cpuidle-calxeda.c
index 9445e6cc0..ea9728fde 100644
--- a/kernel/drivers/cpuidle/cpuidle-calxeda.c
+++ b/kernel/drivers/cpuidle/cpuidle-calxeda.c
@@ -25,16 +25,21 @@
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/platform_device.h>
+#include <linux/psci.h>
+
#include <asm/cpuidle.h>
#include <asm/suspend.h>
-#include <asm/psci.h>
+
+#include <uapi/linux/psci.h>
+
+#define CALXEDA_IDLE_PARAM \
+ ((0 << PSCI_0_2_POWER_STATE_ID_SHIFT) | \
+ (0 << PSCI_0_2_POWER_STATE_AFFL_SHIFT) | \
+ (PSCI_POWER_STATE_TYPE_POWER_DOWN << PSCI_0_2_POWER_STATE_TYPE_SHIFT))
static int calxeda_idle_finish(unsigned long val)
{
- const struct psci_power_state ps = {
- .type = PSCI_POWER_STATE_TYPE_POWER_DOWN,
- };
- return psci_ops.cpu_suspend(ps, __pa(cpu_resume));
+ return psci_ops.cpu_suspend(CALXEDA_IDLE_PARAM, __pa(cpu_resume));
}
static int calxeda_pwrdown_idle(struct cpuidle_device *dev,
@@ -75,5 +80,4 @@ static struct platform_driver calxeda_cpuidle_plat_driver = {
},
.probe = calxeda_cpuidle_probe,
};
-
-module_platform_driver(calxeda_cpuidle_plat_driver);
+builtin_platform_driver(calxeda_cpuidle_plat_driver);
diff --git a/kernel/drivers/cpuidle/cpuidle-mvebu-v7.c b/kernel/drivers/cpuidle/cpuidle-mvebu-v7.c
index 980151f34..01a856971 100644
--- a/kernel/drivers/cpuidle/cpuidle-mvebu-v7.c
+++ b/kernel/drivers/cpuidle/cpuidle-mvebu-v7.c
@@ -99,44 +99,40 @@ static struct cpuidle_driver armada38x_idle_driver = {
static int mvebu_v7_cpuidle_probe(struct platform_device *pdev)
{
- mvebu_v7_cpu_suspend = pdev->dev.platform_data;
+ const struct platform_device_id *id = pdev->id_entry;
- if (!strcmp(pdev->dev.driver->name, "cpuidle-armada-xp"))
- return cpuidle_register(&armadaxp_idle_driver, NULL);
- else if (!strcmp(pdev->dev.driver->name, "cpuidle-armada-370"))
- return cpuidle_register(&armada370_idle_driver, NULL);
- else if (!strcmp(pdev->dev.driver->name, "cpuidle-armada-38x"))
- return cpuidle_register(&armada38x_idle_driver, NULL);
- else
+ if (!id)
return -EINVAL;
-}
-static struct platform_driver armadaxp_cpuidle_plat_driver = {
- .driver = {
- .name = "cpuidle-armada-xp",
- },
- .probe = mvebu_v7_cpuidle_probe,
-};
+ mvebu_v7_cpu_suspend = pdev->dev.platform_data;
-module_platform_driver(armadaxp_cpuidle_plat_driver);
+ return cpuidle_register((struct cpuidle_driver *)id->driver_data, NULL);
+}
-static struct platform_driver armada370_cpuidle_plat_driver = {
- .driver = {
+static const struct platform_device_id mvebu_cpuidle_ids[] = {
+ {
+ .name = "cpuidle-armada-xp",
+ .driver_data = (unsigned long)&armadaxp_idle_driver,
+ }, {
.name = "cpuidle-armada-370",
+ .driver_data = (unsigned long)&armada370_idle_driver,
+ }, {
+ .name = "cpuidle-armada-38x",
+ .driver_data = (unsigned long)&armada38x_idle_driver,
},
- .probe = mvebu_v7_cpuidle_probe,
+ {}
};
-module_platform_driver(armada370_cpuidle_plat_driver);
-
-static struct platform_driver armada38x_cpuidle_plat_driver = {
+static struct platform_driver mvebu_cpuidle_driver = {
+ .probe = mvebu_v7_cpuidle_probe,
.driver = {
- .name = "cpuidle-armada-38x",
+ .name = "cpuidle-mbevu",
+ .suppress_bind_attrs = true,
},
- .probe = mvebu_v7_cpuidle_probe,
+ .id_table = mvebu_cpuidle_ids,
};
-module_platform_driver(armada38x_cpuidle_plat_driver);
+builtin_platform_driver(mvebu_cpuidle_driver);
MODULE_AUTHOR("Gregory CLEMENT <gregory.clement@free-electrons.com>");
MODULE_DESCRIPTION("Marvell EBU v7 cpuidle driver");
diff --git a/kernel/drivers/cpuidle/cpuidle-powernv.c b/kernel/drivers/cpuidle/cpuidle-powernv.c
index 3442764a5..845bafcfa 100644
--- a/kernel/drivers/cpuidle/cpuidle-powernv.c
+++ b/kernel/drivers/cpuidle/cpuidle-powernv.c
@@ -29,18 +29,25 @@ struct cpuidle_driver powernv_idle_driver = {
static int max_idle_state;
static struct cpuidle_state *cpuidle_state_table;
+static u64 snooze_timeout;
+static bool snooze_timeout_en;
static int snooze_loop(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
+ u64 snooze_exit_time;
+
local_irq_enable();
set_thread_flag(TIF_POLLING_NRFLAG);
+ snooze_exit_time = get_tb() + snooze_timeout;
ppc64_runlatch_off();
while (!need_resched()) {
HMT_low();
HMT_very_low();
+ if (snooze_timeout_en && get_tb() > snooze_exit_time)
+ break;
}
HMT_medium();
@@ -261,6 +268,11 @@ static int powernv_idle_probe(void)
cpuidle_state_table = powernv_states;
/* Device tree can indicate more idle states */
max_idle_state = powernv_add_idle_states();
+ if (max_idle_state > 1) {
+ snooze_timeout_en = true;
+ snooze_timeout = powernv_states[1].target_residency *
+ tb_ticks_per_usec;
+ }
} else
return -ENODEV;
diff --git a/kernel/drivers/cpuidle/cpuidle-pseries.c b/kernel/drivers/cpuidle/cpuidle-pseries.c
index bb9e2b6f3..07135e009 100644
--- a/kernel/drivers/cpuidle/cpuidle-pseries.c
+++ b/kernel/drivers/cpuidle/cpuidle-pseries.c
@@ -27,6 +27,8 @@ struct cpuidle_driver pseries_idle_driver = {
static int max_idle_state;
static struct cpuidle_state *cpuidle_state_table;
+static u64 snooze_timeout;
+static bool snooze_timeout_en;
static inline void idle_loop_prolog(unsigned long *in_purr)
{
@@ -58,14 +60,18 @@ static int snooze_loop(struct cpuidle_device *dev,
int index)
{
unsigned long in_purr;
+ u64 snooze_exit_time;
idle_loop_prolog(&in_purr);
local_irq_enable();
set_thread_flag(TIF_POLLING_NRFLAG);
+ snooze_exit_time = get_tb() + snooze_timeout;
while (!need_resched()) {
HMT_low();
HMT_very_low();
+ if (snooze_timeout_en && get_tb() > snooze_exit_time)
+ break;
}
HMT_medium();
@@ -244,6 +250,11 @@ static int pseries_idle_probe(void)
} else
return -ENODEV;
+ if (max_idle_state > 1) {
+ snooze_timeout_en = true;
+ snooze_timeout = cpuidle_state_table[1].target_residency *
+ tb_ticks_per_usec;
+ }
return 0;
}
diff --git a/kernel/drivers/cpuidle/cpuidle-zynq.c b/kernel/drivers/cpuidle/cpuidle-zynq.c
index 543292b1d..6f4257fc5 100644
--- a/kernel/drivers/cpuidle/cpuidle-zynq.c
+++ b/kernel/drivers/cpuidle/cpuidle-zynq.c
@@ -73,5 +73,4 @@ static struct platform_driver zynq_cpuidle_driver = {
},
.probe = zynq_cpuidle_probe,
};
-
-module_platform_driver(zynq_cpuidle_driver);
+builtin_platform_driver(zynq_cpuidle_driver);
diff --git a/kernel/drivers/cpuidle/cpuidle.c b/kernel/drivers/cpuidle/cpuidle.c
index 61c417b9e..17a6dc0e2 100644
--- a/kernel/drivers/cpuidle/cpuidle.c
+++ b/kernel/drivers/cpuidle/cpuidle.c
@@ -65,7 +65,7 @@ int cpuidle_play_dead(void)
return -ENODEV;
/* Find lowest-power state that supports long-term idle */
- for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--)
+ for (i = drv->state_count - 1; i >= 0; i--)
if (drv->states[i].enter_dead)
return drv->states[i].enter_dead(dev, i);
@@ -73,16 +73,21 @@ int cpuidle_play_dead(void)
}
static int find_deepest_state(struct cpuidle_driver *drv,
- struct cpuidle_device *dev, bool freeze)
+ struct cpuidle_device *dev,
+ unsigned int max_latency,
+ unsigned int forbidden_flags,
+ bool freeze)
{
unsigned int latency_req = 0;
- int i, ret = freeze ? -1 : CPUIDLE_DRIVER_STATE_START - 1;
+ int i, ret = -ENXIO;
- for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) {
+ for (i = 0; i < drv->state_count; i++) {
struct cpuidle_state *s = &drv->states[i];
struct cpuidle_state_usage *su = &dev->states_usage[i];
if (s->disabled || su->disable || s->exit_latency <= latency_req
+ || s->exit_latency > max_latency
+ || (s->flags & forbidden_flags)
|| (freeze && !s->enter_freeze))
continue;
@@ -92,6 +97,7 @@ static int find_deepest_state(struct cpuidle_driver *drv,
return ret;
}
+#ifdef CONFIG_SUSPEND
/**
* cpuidle_find_deepest_state - Find the deepest available idle state.
* @drv: cpuidle driver for the given CPU.
@@ -100,26 +106,33 @@ static int find_deepest_state(struct cpuidle_driver *drv,
int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev)
{
- return find_deepest_state(drv, dev, false);
+ return find_deepest_state(drv, dev, UINT_MAX, 0, false);
}
static void enter_freeze_proper(struct cpuidle_driver *drv,
struct cpuidle_device *dev, int index)
{
- tick_freeze();
+ /*
+ * trace_suspend_resume() called by tick_freeze() for the last CPU
+ * executing it contains RCU usage regarded as invalid in the idle
+ * context, so tell RCU about that.
+ */
+ RCU_NONIDLE(tick_freeze());
/*
* The state used here cannot be a "coupled" one, because the "coupled"
* cpuidle mechanism enables interrupts and doing that with timekeeping
* suspended is generally unsafe.
*/
+ stop_critical_timings();
drv->states[index].enter_freeze(dev, drv, index);
WARN_ON(!irqs_disabled());
/*
* timekeeping_resume() that will be called by tick_unfreeze() for the
- * last CPU executing it calls functions containing RCU read-side
+ * first CPU executing it calls functions containing RCU read-side
* critical sections, so tell RCU about that.
*/
RCU_NONIDLE(tick_unfreeze());
+ start_critical_timings();
}
/**
@@ -139,18 +152,19 @@ int cpuidle_enter_freeze(struct cpuidle_driver *drv, struct cpuidle_device *dev)
* that interrupts won't be enabled when it exits and allows the tick to
* be frozen safely.
*/
- index = find_deepest_state(drv, dev, true);
+ index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
if (index >= 0)
enter_freeze_proper(drv, dev, index);
return index;
}
+#endif /* CONFIG_SUSPEND */
/**
* cpuidle_enter_state - enter the state and update stats
* @dev: cpuidle device for this cpu
* @drv: cpuidle driver for this cpu
- * @next_state: index into drv->states of the state to enter
+ * @index: index into the states table in @drv of the state to enter
*/
int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
int index)
@@ -167,17 +181,32 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
* local timer will be shut down. If a local timer is used from another
* CPU as a broadcast timer, this call may fail if it is not available.
*/
- if (broadcast && tick_broadcast_enter())
- return -EBUSY;
+ if (broadcast && tick_broadcast_enter()) {
+ index = find_deepest_state(drv, dev, target_state->exit_latency,
+ CPUIDLE_FLAG_TIMER_STOP, false);
+ if (index < 0) {
+ default_idle_call();
+ return -EBUSY;
+ }
+ target_state = &drv->states[index];
+ }
+
+ /* Take note of the planned idle state. */
+ sched_idle_set_state(target_state);
trace_cpu_idle_rcuidle(index, dev->cpu);
time_start = ktime_get();
+ stop_critical_timings();
entered_state = target_state->enter(dev, drv, index);
+ start_critical_timings();
time_end = ktime_get();
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
+ /* The cpu is no longer idle or about to enter idle. */
+ sched_idle_set_state(NULL);
+
if (broadcast) {
if (WARN_ON_ONCE(!irqs_disabled()))
local_irq_disable();
@@ -185,7 +214,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
tick_broadcast_exit();
}
- if (!cpuidle_state_is_coupled(dev, drv, entered_state))
+ if (!cpuidle_state_is_coupled(drv, entered_state))
local_irq_enable();
diff = ktime_to_us(ktime_sub(time_end, time_start));
@@ -234,7 +263,7 @@ int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
int index)
{
- if (cpuidle_state_is_coupled(dev, drv, index))
+ if (cpuidle_state_is_coupled(drv, index))
return cpuidle_enter_state_coupled(dev, drv, index);
return cpuidle_enter_state(dev, drv, index);
}
@@ -249,7 +278,7 @@ int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
*/
void cpuidle_reflect(struct cpuidle_device *dev, int index)
{
- if (cpuidle_curr_governor->reflect)
+ if (cpuidle_curr_governor->reflect && index >= 0)
cpuidle_curr_governor->reflect(dev, index);
}
diff --git a/kernel/drivers/cpuidle/cpuidle.h b/kernel/drivers/cpuidle/cpuidle.h
index ee97e9672..f87f399b0 100644
--- a/kernel/drivers/cpuidle/cpuidle.h
+++ b/kernel/drivers/cpuidle/cpuidle.h
@@ -34,19 +34,24 @@ extern int cpuidle_add_sysfs(struct cpuidle_device *dev);
extern void cpuidle_remove_sysfs(struct cpuidle_device *dev);
#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
-bool cpuidle_state_is_coupled(struct cpuidle_device *dev,
- struct cpuidle_driver *drv, int state);
+bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state);
+int cpuidle_coupled_state_verify(struct cpuidle_driver *drv);
int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
struct cpuidle_driver *drv, int next_state);
int cpuidle_coupled_register_device(struct cpuidle_device *dev);
void cpuidle_coupled_unregister_device(struct cpuidle_device *dev);
#else
-static inline bool cpuidle_state_is_coupled(struct cpuidle_device *dev,
- struct cpuidle_driver *drv, int state)
+static inline
+bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state)
{
return false;
}
+static inline int cpuidle_coupled_state_verify(struct cpuidle_driver *drv)
+{
+ return 0;
+}
+
static inline int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
struct cpuidle_driver *drv, int next_state)
{
diff --git a/kernel/drivers/cpuidle/driver.c b/kernel/drivers/cpuidle/driver.c
index 5db147859..389ade457 100644
--- a/kernel/drivers/cpuidle/driver.c
+++ b/kernel/drivers/cpuidle/driver.c
@@ -227,6 +227,10 @@ static int __cpuidle_register_driver(struct cpuidle_driver *drv)
if (!drv || !drv->state_count)
return -EINVAL;
+ ret = cpuidle_coupled_state_verify(drv);
+ if (ret)
+ return ret;
+
if (cpuidle_disabled())
return -ENODEV;
diff --git a/kernel/drivers/cpuidle/governors/menu.c b/kernel/drivers/cpuidle/governors/menu.c
index b8a5fa15c..22e4463d1 100644
--- a/kernel/drivers/cpuidle/governors/menu.c
+++ b/kernel/drivers/cpuidle/governors/menu.c
@@ -367,9 +367,9 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
static void menu_reflect(struct cpuidle_device *dev, int index)
{
struct menu_device *data = this_cpu_ptr(&menu_devices);
+
data->last_state_idx = index;
- if (index >= 0)
- data->needs_update = 1;
+ data->needs_update = 1;
}
/**