1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License version 2 as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 * GNU General Public License for more details.
10 *
11 * Copyright (C) 2013 ARM Limited
12 *
13 * Author: Will Deacon <will.deacon@arm.com>
14 */
15
16#define pr_fmt(fmt) "psci: " fmt
17
18#include <linux/acpi.h>
19#include <linux/init.h>
20#include <linux/of.h>
21#include <linux/smp.h>
22#include <linux/reboot.h>
23#include <linux/pm.h>
24#include <linux/delay.h>
25#include <linux/slab.h>
26#include <uapi/linux/psci.h>
27
28#include <asm/acpi.h>
29#include <asm/compiler.h>
30#include <asm/cpu_ops.h>
31#include <asm/errno.h>
32#include <asm/psci.h>
33#include <asm/smp_plat.h>
34#include <asm/suspend.h>
35#include <asm/system_misc.h>
36
37#define PSCI_POWER_STATE_TYPE_STANDBY		0
38#define PSCI_POWER_STATE_TYPE_POWER_DOWN	1
39
40struct psci_power_state {
41	u16	id;
42	u8	type;
43	u8	affinity_level;
44};
45
46struct psci_operations {
47	int (*cpu_suspend)(struct psci_power_state state,
48			   unsigned long entry_point);
49	int (*cpu_off)(struct psci_power_state state);
50	int (*cpu_on)(unsigned long cpuid, unsigned long entry_point);
51	int (*migrate)(unsigned long cpuid);
52	int (*affinity_info)(unsigned long target_affinity,
53			unsigned long lowest_affinity_level);
54	int (*migrate_info_type)(void);
55};
56
57static struct psci_operations psci_ops;
58
59static int (*invoke_psci_fn)(u64, u64, u64, u64);
60typedef int (*psci_initcall_t)(const struct device_node *);
61
62asmlinkage int __invoke_psci_fn_hvc(u64, u64, u64, u64);
63asmlinkage int __invoke_psci_fn_smc(u64, u64, u64, u64);
64
65enum psci_function {
66	PSCI_FN_CPU_SUSPEND,
67	PSCI_FN_CPU_ON,
68	PSCI_FN_CPU_OFF,
69	PSCI_FN_MIGRATE,
70	PSCI_FN_AFFINITY_INFO,
71	PSCI_FN_MIGRATE_INFO_TYPE,
72	PSCI_FN_MAX,
73};
74
75static DEFINE_PER_CPU_READ_MOSTLY(struct psci_power_state *, psci_power_state);
76
77static u32 psci_function_id[PSCI_FN_MAX];
78
79static int psci_to_linux_errno(int errno)
80{
81	switch (errno) {
82	case PSCI_RET_SUCCESS:
83		return 0;
84	case PSCI_RET_NOT_SUPPORTED:
85		return -EOPNOTSUPP;
86	case PSCI_RET_INVALID_PARAMS:
87		return -EINVAL;
88	case PSCI_RET_DENIED:
89		return -EPERM;
90	};
91
92	return -EINVAL;
93}
94
95static u32 psci_power_state_pack(struct psci_power_state state)
96{
97	return ((state.id << PSCI_0_2_POWER_STATE_ID_SHIFT)
98			& PSCI_0_2_POWER_STATE_ID_MASK) |
99		((state.type << PSCI_0_2_POWER_STATE_TYPE_SHIFT)
100		 & PSCI_0_2_POWER_STATE_TYPE_MASK) |
101		((state.affinity_level << PSCI_0_2_POWER_STATE_AFFL_SHIFT)
102		 & PSCI_0_2_POWER_STATE_AFFL_MASK);
103}
104
105static void psci_power_state_unpack(u32 power_state,
106				    struct psci_power_state *state)
107{
108	state->id = (power_state & PSCI_0_2_POWER_STATE_ID_MASK) >>
109			PSCI_0_2_POWER_STATE_ID_SHIFT;
110	state->type = (power_state & PSCI_0_2_POWER_STATE_TYPE_MASK) >>
111			PSCI_0_2_POWER_STATE_TYPE_SHIFT;
112	state->affinity_level =
113			(power_state & PSCI_0_2_POWER_STATE_AFFL_MASK) >>
114			PSCI_0_2_POWER_STATE_AFFL_SHIFT;
115}
116
117static int psci_get_version(void)
118{
119	int err;
120
121	err = invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0);
122	return err;
123}
124
125static int psci_cpu_suspend(struct psci_power_state state,
126			    unsigned long entry_point)
127{
128	int err;
129	u32 fn, power_state;
130
131	fn = psci_function_id[PSCI_FN_CPU_SUSPEND];
132	power_state = psci_power_state_pack(state);
133	err = invoke_psci_fn(fn, power_state, entry_point, 0);
134	return psci_to_linux_errno(err);
135}
136
137static int psci_cpu_off(struct psci_power_state state)
138{
139	int err;
140	u32 fn, power_state;
141
142	fn = psci_function_id[PSCI_FN_CPU_OFF];
143	power_state = psci_power_state_pack(state);
144	err = invoke_psci_fn(fn, power_state, 0, 0);
145	return psci_to_linux_errno(err);
146}
147
148static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point)
149{
150	int err;
151	u32 fn;
152
153	fn = psci_function_id[PSCI_FN_CPU_ON];
154	err = invoke_psci_fn(fn, cpuid, entry_point, 0);
155	return psci_to_linux_errno(err);
156}
157
158static int psci_migrate(unsigned long cpuid)
159{
160	int err;
161	u32 fn;
162
163	fn = psci_function_id[PSCI_FN_MIGRATE];
164	err = invoke_psci_fn(fn, cpuid, 0, 0);
165	return psci_to_linux_errno(err);
166}
167
168static int psci_affinity_info(unsigned long target_affinity,
169		unsigned long lowest_affinity_level)
170{
171	int err;
172	u32 fn;
173
174	fn = psci_function_id[PSCI_FN_AFFINITY_INFO];
175	err = invoke_psci_fn(fn, target_affinity, lowest_affinity_level, 0);
176	return err;
177}
178
179static int psci_migrate_info_type(void)
180{
181	int err;
182	u32 fn;
183
184	fn = psci_function_id[PSCI_FN_MIGRATE_INFO_TYPE];
185	err = invoke_psci_fn(fn, 0, 0, 0);
186	return err;
187}
188
189static int __maybe_unused cpu_psci_cpu_init_idle(struct device_node *cpu_node,
190						 unsigned int cpu)
191{
192	int i, ret, count = 0;
193	struct psci_power_state *psci_states;
194	struct device_node *state_node;
195
196	/*
197	 * If the PSCI cpu_suspend function hook has not been initialized
198	 * idle states must not be enabled, so bail out
199	 */
200	if (!psci_ops.cpu_suspend)
201		return -EOPNOTSUPP;
202
203	/* Count idle states */
204	while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states",
205					      count))) {
206		count++;
207		of_node_put(state_node);
208	}
209
210	if (!count)
211		return -ENODEV;
212
213	psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL);
214	if (!psci_states)
215		return -ENOMEM;
216
217	for (i = 0; i < count; i++) {
218		u32 psci_power_state;
219
220		state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i);
221
222		ret = of_property_read_u32(state_node,
223					   "arm,psci-suspend-param",
224					   &psci_power_state);
225		if (ret) {
226			pr_warn(" * %s missing arm,psci-suspend-param property\n",
227				state_node->full_name);
228			of_node_put(state_node);
229			goto free_mem;
230		}
231
232		of_node_put(state_node);
233		pr_debug("psci-power-state %#x index %d\n", psci_power_state,
234							    i);
235		psci_power_state_unpack(psci_power_state, &psci_states[i]);
236	}
237	/* Idle states parsed correctly, initialize per-cpu pointer */
238	per_cpu(psci_power_state, cpu) = psci_states;
239	return 0;
240
241free_mem:
242	kfree(psci_states);
243	return ret;
244}
245
246static int get_set_conduit_method(struct device_node *np)
247{
248	const char *method;
249
250	pr_info("probing for conduit method from DT.\n");
251
252	if (of_property_read_string(np, "method", &method)) {
253		pr_warn("missing \"method\" property\n");
254		return -ENXIO;
255	}
256
257	if (!strcmp("hvc", method)) {
258		invoke_psci_fn = __invoke_psci_fn_hvc;
259	} else if (!strcmp("smc", method)) {
260		invoke_psci_fn = __invoke_psci_fn_smc;
261	} else {
262		pr_warn("invalid \"method\" property: %s\n", method);
263		return -EINVAL;
264	}
265	return 0;
266}
267
268static void psci_sys_reset(enum reboot_mode reboot_mode, const char *cmd)
269{
270	invoke_psci_fn(PSCI_0_2_FN_SYSTEM_RESET, 0, 0, 0);
271}
272
273static void psci_sys_poweroff(void)
274{
275	invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0);
276}
277
278static void __init psci_0_2_set_functions(void)
279{
280	pr_info("Using standard PSCI v0.2 function IDs\n");
281	psci_function_id[PSCI_FN_CPU_SUSPEND] = PSCI_0_2_FN64_CPU_SUSPEND;
282	psci_ops.cpu_suspend = psci_cpu_suspend;
283
284	psci_function_id[PSCI_FN_CPU_OFF] = PSCI_0_2_FN_CPU_OFF;
285	psci_ops.cpu_off = psci_cpu_off;
286
287	psci_function_id[PSCI_FN_CPU_ON] = PSCI_0_2_FN64_CPU_ON;
288	psci_ops.cpu_on = psci_cpu_on;
289
290	psci_function_id[PSCI_FN_MIGRATE] = PSCI_0_2_FN64_MIGRATE;
291	psci_ops.migrate = psci_migrate;
292
293	psci_function_id[PSCI_FN_AFFINITY_INFO] = PSCI_0_2_FN64_AFFINITY_INFO;
294	psci_ops.affinity_info = psci_affinity_info;
295
296	psci_function_id[PSCI_FN_MIGRATE_INFO_TYPE] =
297		PSCI_0_2_FN_MIGRATE_INFO_TYPE;
298	psci_ops.migrate_info_type = psci_migrate_info_type;
299
300	arm_pm_restart = psci_sys_reset;
301
302	pm_power_off = psci_sys_poweroff;
303}
304
305/*
306 * Probe function for PSCI firmware versions >= 0.2
307 */
308static int __init psci_probe(void)
309{
310	int ver = psci_get_version();
311
312	if (ver == PSCI_RET_NOT_SUPPORTED) {
313		/*
314		 * PSCI versions >=0.2 mandates implementation of
315		 * PSCI_VERSION.
316		 */
317		pr_err("PSCI firmware does not comply with the v0.2 spec.\n");
318		return -EOPNOTSUPP;
319	} else {
320		pr_info("PSCIv%d.%d detected in firmware.\n",
321				PSCI_VERSION_MAJOR(ver),
322				PSCI_VERSION_MINOR(ver));
323
324		if (PSCI_VERSION_MAJOR(ver) == 0 &&
325				PSCI_VERSION_MINOR(ver) < 2) {
326			pr_err("Conflicting PSCI version detected.\n");
327			return -EINVAL;
328		}
329	}
330
331	psci_0_2_set_functions();
332
333	return 0;
334}
335
336/*
337 * PSCI init function for PSCI versions >=0.2
338 *
339 * Probe based on PSCI PSCI_VERSION function
340 */
341static int __init psci_0_2_init(struct device_node *np)
342{
343	int err;
344
345	err = get_set_conduit_method(np);
346
347	if (err)
348		goto out_put_node;
349	/*
350	 * Starting with v0.2, the PSCI specification introduced a call
351	 * (PSCI_VERSION) that allows probing the firmware version, so
352	 * that PSCI function IDs and version specific initialization
353	 * can be carried out according to the specific version reported
354	 * by firmware
355	 */
356	err = psci_probe();
357
358out_put_node:
359	of_node_put(np);
360	return err;
361}
362
363/*
364 * PSCI < v0.2 get PSCI Function IDs via DT.
365 */
366static int __init psci_0_1_init(struct device_node *np)
367{
368	u32 id;
369	int err;
370
371	err = get_set_conduit_method(np);
372
373	if (err)
374		goto out_put_node;
375
376	pr_info("Using PSCI v0.1 Function IDs from DT\n");
377
378	if (!of_property_read_u32(np, "cpu_suspend", &id)) {
379		psci_function_id[PSCI_FN_CPU_SUSPEND] = id;
380		psci_ops.cpu_suspend = psci_cpu_suspend;
381	}
382
383	if (!of_property_read_u32(np, "cpu_off", &id)) {
384		psci_function_id[PSCI_FN_CPU_OFF] = id;
385		psci_ops.cpu_off = psci_cpu_off;
386	}
387
388	if (!of_property_read_u32(np, "cpu_on", &id)) {
389		psci_function_id[PSCI_FN_CPU_ON] = id;
390		psci_ops.cpu_on = psci_cpu_on;
391	}
392
393	if (!of_property_read_u32(np, "migrate", &id)) {
394		psci_function_id[PSCI_FN_MIGRATE] = id;
395		psci_ops.migrate = psci_migrate;
396	}
397
398out_put_node:
399	of_node_put(np);
400	return err;
401}
402
403static const struct of_device_id psci_of_match[] __initconst = {
404	{ .compatible = "arm,psci",	.data = psci_0_1_init},
405	{ .compatible = "arm,psci-0.2",	.data = psci_0_2_init},
406	{},
407};
408
409int __init psci_dt_init(void)
410{
411	struct device_node *np;
412	const struct of_device_id *matched_np;
413	psci_initcall_t init_fn;
414
415	np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np);
416
417	if (!np)
418		return -ENODEV;
419
420	init_fn = (psci_initcall_t)matched_np->data;
421	return init_fn(np);
422}
423
424/*
425 * We use PSCI 0.2+ when ACPI is deployed on ARM64 and it's
426 * explicitly clarified in SBBR
427 */
428int __init psci_acpi_init(void)
429{
430	if (!acpi_psci_present()) {
431		pr_info("is not implemented in ACPI.\n");
432		return -EOPNOTSUPP;
433	}
434
435	pr_info("probing for conduit method from ACPI.\n");
436
437	if (acpi_psci_use_hvc())
438		invoke_psci_fn = __invoke_psci_fn_hvc;
439	else
440		invoke_psci_fn = __invoke_psci_fn_smc;
441
442	return psci_probe();
443}
444
445#ifdef CONFIG_SMP
446
447static int __init cpu_psci_cpu_init(struct device_node *dn, unsigned int cpu)
448{
449	return 0;
450}
451
452static int __init cpu_psci_cpu_prepare(unsigned int cpu)
453{
454	if (!psci_ops.cpu_on) {
455		pr_err("no cpu_on method, not booting CPU%d\n", cpu);
456		return -ENODEV;
457	}
458
459	return 0;
460}
461
462static int cpu_psci_cpu_boot(unsigned int cpu)
463{
464	int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa(secondary_entry));
465	if (err)
466		pr_err("failed to boot CPU%d (%d)\n", cpu, err);
467
468	return err;
469}
470
471#ifdef CONFIG_HOTPLUG_CPU
472static int cpu_psci_cpu_disable(unsigned int cpu)
473{
474	/* Fail early if we don't have CPU_OFF support */
475	if (!psci_ops.cpu_off)
476		return -EOPNOTSUPP;
477	return 0;
478}
479
480static void cpu_psci_cpu_die(unsigned int cpu)
481{
482	int ret;
483	/*
484	 * There are no known implementations of PSCI actually using the
485	 * power state field, pass a sensible default for now.
486	 */
487	struct psci_power_state state = {
488		.type = PSCI_POWER_STATE_TYPE_POWER_DOWN,
489	};
490
491	ret = psci_ops.cpu_off(state);
492
493	pr_crit("unable to power off CPU%u (%d)\n", cpu, ret);
494}
495
496static int cpu_psci_cpu_kill(unsigned int cpu)
497{
498	int err, i;
499
500	if (!psci_ops.affinity_info)
501		return 1;
502	/*
503	 * cpu_kill could race with cpu_die and we can
504	 * potentially end up declaring this cpu undead
505	 * while it is dying. So, try again a few times.
506	 */
507
508	for (i = 0; i < 10; i++) {
509		err = psci_ops.affinity_info(cpu_logical_map(cpu), 0);
510		if (err == PSCI_0_2_AFFINITY_LEVEL_OFF) {
511			pr_info("CPU%d killed.\n", cpu);
512			return 1;
513		}
514
515		msleep(10);
516		pr_info("Retrying again to check for CPU kill\n");
517	}
518
519	pr_warn("CPU%d may not have shut down cleanly (AFFINITY_INFO reports %d)\n",
520			cpu, err);
521	/* Make op_cpu_kill() fail. */
522	return 0;
523}
524#endif
525#endif
526
527static int psci_suspend_finisher(unsigned long index)
528{
529	struct psci_power_state *state = __this_cpu_read(psci_power_state);
530
531	return psci_ops.cpu_suspend(state[index - 1],
532				    virt_to_phys(cpu_resume));
533}
534
535static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index)
536{
537	int ret;
538	struct psci_power_state *state = __this_cpu_read(psci_power_state);
539	/*
540	 * idle state index 0 corresponds to wfi, should never be called
541	 * from the cpu_suspend operations
542	 */
543	if (WARN_ON_ONCE(!index))
544		return -EINVAL;
545
546	if (state[index - 1].type == PSCI_POWER_STATE_TYPE_STANDBY)
547		ret = psci_ops.cpu_suspend(state[index - 1], 0);
548	else
549		ret = cpu_suspend(index, psci_suspend_finisher);
550
551	return ret;
552}
553
554const struct cpu_operations cpu_psci_ops = {
555	.name		= "psci",
556#ifdef CONFIG_CPU_IDLE
557	.cpu_init_idle	= cpu_psci_cpu_init_idle,
558	.cpu_suspend	= cpu_psci_cpu_suspend,
559#endif
560#ifdef CONFIG_SMP
561	.cpu_init	= cpu_psci_cpu_init,
562	.cpu_prepare	= cpu_psci_cpu_prepare,
563	.cpu_boot	= cpu_psci_cpu_boot,
564#ifdef CONFIG_HOTPLUG_CPU
565	.cpu_disable	= cpu_psci_cpu_disable,
566	.cpu_die	= cpu_psci_cpu_die,
567	.cpu_kill	= cpu_psci_cpu_kill,
568#endif
569#endif
570};
571
572