1/*
2 * PowerNV cpuidle code
3 *
4 * Copyright 2015 IBM Corp.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12#include <linux/types.h>
13#include <linux/mm.h>
14#include <linux/slab.h>
15#include <linux/of.h>
16#include <linux/device.h>
17#include <linux/cpu.h>
18
19#include <asm/firmware.h>
20#include <asm/machdep.h>
21#include <asm/opal.h>
22#include <asm/cputhreads.h>
23#include <asm/cpuidle.h>
24#include <asm/code-patching.h>
25#include <asm/smp.h>
26
27#include "powernv.h"
28#include "subcore.h"
29
30static u32 supported_cpuidle_states;
31
32int pnv_save_sprs_for_winkle(void)
33{
34	int cpu;
35	int rc;
36
37	/*
38	 * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric accross
39	 * all cpus at boot. Get these reg values of current cpu and use the
40	 * same accross all cpus.
41	 */
42	uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1;
43	uint64_t hid0_val = mfspr(SPRN_HID0);
44	uint64_t hid1_val = mfspr(SPRN_HID1);
45	uint64_t hid4_val = mfspr(SPRN_HID4);
46	uint64_t hid5_val = mfspr(SPRN_HID5);
47	uint64_t hmeer_val = mfspr(SPRN_HMEER);
48
49	for_each_possible_cpu(cpu) {
50		uint64_t pir = get_hard_smp_processor_id(cpu);
51		uint64_t hsprg0_val = (uint64_t)&paca[cpu];
52
53		/*
54		 * HSPRG0 is used to store the cpu's pointer to paca. Hence last
55		 * 3 bits are guaranteed to be 0. Program slw to restore HSPRG0
56		 * with 63rd bit set, so that when a thread wakes up at 0x100 we
57		 * can use this bit to distinguish between fastsleep and
58		 * deep winkle.
59		 */
60		hsprg0_val |= 1;
61
62		rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val);
63		if (rc != 0)
64			return rc;
65
66		rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
67		if (rc != 0)
68			return rc;
69
70		/* HIDs are per core registers */
71		if (cpu_thread_in_core(cpu) == 0) {
72
73			rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val);
74			if (rc != 0)
75				return rc;
76
77			rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val);
78			if (rc != 0)
79				return rc;
80
81			rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val);
82			if (rc != 0)
83				return rc;
84
85			rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val);
86			if (rc != 0)
87				return rc;
88
89			rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val);
90			if (rc != 0)
91				return rc;
92		}
93	}
94
95	return 0;
96}
97
98static void pnv_alloc_idle_core_states(void)
99{
100	int i, j;
101	int nr_cores = cpu_nr_cores();
102	u32 *core_idle_state;
103
104	/*
105	 * core_idle_state - First 8 bits track the idle state of each thread
106	 * of the core. The 8th bit is the lock bit. Initially all thread bits
107	 * are set. They are cleared when the thread enters deep idle state
108	 * like sleep and winkle. Initially the lock bit is cleared.
109	 * The lock bit has 2 purposes
110	 * a. While the first thread is restoring core state, it prevents
111	 * other threads in the core from switching to process context.
112	 * b. While the last thread in the core is saving the core state, it
113	 * prevents a different thread from waking up.
114	 */
115	for (i = 0; i < nr_cores; i++) {
116		int first_cpu = i * threads_per_core;
117		int node = cpu_to_node(first_cpu);
118
119		core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node);
120		*core_idle_state = PNV_CORE_IDLE_THREAD_BITS;
121
122		for (j = 0; j < threads_per_core; j++) {
123			int cpu = first_cpu + j;
124
125			paca[cpu].core_idle_state_ptr = core_idle_state;
126			paca[cpu].thread_idle_state = PNV_THREAD_RUNNING;
127			paca[cpu].thread_mask = 1 << j;
128		}
129	}
130
131	update_subcore_sibling_mask();
132
133	if (supported_cpuidle_states & OPAL_PM_WINKLE_ENABLED)
134		pnv_save_sprs_for_winkle();
135}
136
137u32 pnv_get_supported_cpuidle_states(void)
138{
139	return supported_cpuidle_states;
140}
141EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states);
142
143
144static void pnv_fastsleep_workaround_apply(void *info)
145
146{
147	int rc;
148	int *err = info;
149
150	rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP,
151					OPAL_CONFIG_IDLE_APPLY);
152	if (rc)
153		*err = 1;
154}
155
156/*
157 * Used to store fastsleep workaround state
158 * 0 - Workaround applied/undone at fastsleep entry/exit path (Default)
159 * 1 - Workaround applied once, never undone.
160 */
161static u8 fastsleep_workaround_applyonce;
162
163static ssize_t show_fastsleep_workaround_applyonce(struct device *dev,
164		struct device_attribute *attr, char *buf)
165{
166	return sprintf(buf, "%u\n", fastsleep_workaround_applyonce);
167}
168
169static ssize_t store_fastsleep_workaround_applyonce(struct device *dev,
170		struct device_attribute *attr, const char *buf,
171		size_t count)
172{
173	cpumask_t primary_thread_mask;
174	int err;
175	u8 val;
176
177	if (kstrtou8(buf, 0, &val) || val != 1)
178		return -EINVAL;
179
180	if (fastsleep_workaround_applyonce == 1)
181		return count;
182
183	/*
184	 * fastsleep_workaround_applyonce = 1 implies
185	 * fastsleep workaround needs to be left in 'applied' state on all
186	 * the cores. Do this by-
187	 * 1. Patching out the call to 'undo' workaround in fastsleep exit path
188	 * 2. Sending ipi to all the cores which have atleast one online thread
189	 * 3. Patching out the call to 'apply' workaround in fastsleep entry
190	 * path
191	 * There is no need to send ipi to cores which have all threads
192	 * offlined, as last thread of the core entering fastsleep or deeper
193	 * state would have applied workaround.
194	 */
195	err = patch_instruction(
196		(unsigned int *)pnv_fastsleep_workaround_at_exit,
197		PPC_INST_NOP);
198	if (err) {
199		pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit");
200		goto fail;
201	}
202
203	get_online_cpus();
204	primary_thread_mask = cpu_online_cores_map();
205	on_each_cpu_mask(&primary_thread_mask,
206				pnv_fastsleep_workaround_apply,
207				&err, 1);
208	put_online_cpus();
209	if (err) {
210		pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply");
211		goto fail;
212	}
213
214	err = patch_instruction(
215		(unsigned int *)pnv_fastsleep_workaround_at_entry,
216		PPC_INST_NOP);
217	if (err) {
218		pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry");
219		goto fail;
220	}
221
222	fastsleep_workaround_applyonce = 1;
223
224	return count;
225fail:
226	return -EIO;
227}
228
229static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600,
230			show_fastsleep_workaround_applyonce,
231			store_fastsleep_workaround_applyonce);
232
233static int __init pnv_init_idle_states(void)
234{
235	struct device_node *power_mgt;
236	int dt_idle_states;
237	u32 *flags;
238	int i;
239
240	supported_cpuidle_states = 0;
241
242	if (cpuidle_disable != IDLE_NO_OVERRIDE)
243		goto out;
244
245	if (!firmware_has_feature(FW_FEATURE_OPALv3))
246		goto out;
247
248	power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
249	if (!power_mgt) {
250		pr_warn("opal: PowerMgmt Node not found\n");
251		goto out;
252	}
253	dt_idle_states = of_property_count_u32_elems(power_mgt,
254			"ibm,cpu-idle-state-flags");
255	if (dt_idle_states < 0) {
256		pr_warn("cpuidle-powernv: no idle states found in the DT\n");
257		goto out;
258	}
259
260	flags = kzalloc(sizeof(*flags) * dt_idle_states, GFP_KERNEL);
261	if (of_property_read_u32_array(power_mgt,
262			"ibm,cpu-idle-state-flags", flags, dt_idle_states)) {
263		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n");
264		goto out_free;
265	}
266
267	for (i = 0; i < dt_idle_states; i++)
268		supported_cpuidle_states |= flags[i];
269
270	if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
271		patch_instruction(
272			(unsigned int *)pnv_fastsleep_workaround_at_entry,
273			PPC_INST_NOP);
274		patch_instruction(
275			(unsigned int *)pnv_fastsleep_workaround_at_exit,
276			PPC_INST_NOP);
277	} else {
278		/*
279		 * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that
280		 * workaround is needed to use fastsleep. Provide sysfs
281		 * control to choose how this workaround has to be applied.
282		 */
283		device_create_file(cpu_subsys.dev_root,
284				&dev_attr_fastsleep_workaround_applyonce);
285	}
286
287	pnv_alloc_idle_core_states();
288out_free:
289	kfree(flags);
290out:
291	return 0;
292}
293machine_subsys_initcall(powernv, pnv_init_idle_states);
294