1/*
2 * Xtensa SMP support functions.
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License.  See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright (C) 2008 - 2013 Tensilica Inc.
9 *
10 * Chris Zankel <chris@zankel.net>
11 * Joe Taylor <joe@tensilica.com>
12 * Pete Delaney <piet@tensilica.com
13 */
14
15#include <linux/cpu.h>
16#include <linux/cpumask.h>
17#include <linux/delay.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/irqdomain.h>
21#include <linux/irq.h>
22#include <linux/kdebug.h>
23#include <linux/module.h>
24#include <linux/reboot.h>
25#include <linux/seq_file.h>
26#include <linux/smp.h>
27#include <linux/thread_info.h>
28
29#include <asm/cacheflush.h>
30#include <asm/kdebug.h>
31#include <asm/mmu_context.h>
32#include <asm/mxregs.h>
33#include <asm/platform.h>
34#include <asm/tlbflush.h>
35#include <asm/traps.h>
36
37#ifdef CONFIG_SMP
38# if XCHAL_HAVE_S32C1I == 0
39#  error "The S32C1I option is required for SMP."
40# endif
41#endif
42
43static void system_invalidate_dcache_range(unsigned long start,
44		unsigned long size);
45static void system_flush_invalidate_dcache_range(unsigned long start,
46		unsigned long size);
47
48/* IPI (Inter Process Interrupt) */
49
50#define IPI_IRQ	0
51
52static irqreturn_t ipi_interrupt(int irq, void *dev_id);
53static struct irqaction ipi_irqaction = {
54	.handler =	ipi_interrupt,
55	.flags =	IRQF_PERCPU,
56	.name =		"ipi",
57};
58
59void ipi_init(void)
60{
61	unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
62	setup_irq(irq, &ipi_irqaction);
63}
64
65static inline unsigned int get_core_count(void)
66{
67	/* Bits 18..21 of SYSCFGID contain the core count minus 1. */
68	unsigned int syscfgid = get_er(SYSCFGID);
69	return ((syscfgid >> 18) & 0xf) + 1;
70}
71
72static inline int get_core_id(void)
73{
74	/* Bits 0...18 of SYSCFGID contain the core id  */
75	unsigned int core_id = get_er(SYSCFGID);
76	return core_id & 0x3fff;
77}
78
79void __init smp_prepare_cpus(unsigned int max_cpus)
80{
81	unsigned i;
82
83	for (i = 0; i < max_cpus; ++i)
84		set_cpu_present(i, true);
85}
86
87void __init smp_init_cpus(void)
88{
89	unsigned i;
90	unsigned int ncpus = get_core_count();
91	unsigned int core_id = get_core_id();
92
93	pr_info("%s: Core Count = %d\n", __func__, ncpus);
94	pr_info("%s: Core Id = %d\n", __func__, core_id);
95
96	for (i = 0; i < ncpus; ++i)
97		set_cpu_possible(i, true);
98}
99
100void __init smp_prepare_boot_cpu(void)
101{
102	unsigned int cpu = smp_processor_id();
103	BUG_ON(cpu != 0);
104	cpu_asid_cache(cpu) = ASID_USER_FIRST;
105}
106
107void __init smp_cpus_done(unsigned int max_cpus)
108{
109}
110
111static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
112static DECLARE_COMPLETION(cpu_running);
113
114void secondary_start_kernel(void)
115{
116	struct mm_struct *mm = &init_mm;
117	unsigned int cpu = smp_processor_id();
118
119	init_mmu();
120
121#ifdef CONFIG_DEBUG_KERNEL
122	if (boot_secondary_processors == 0) {
123		pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
124			__func__, boot_secondary_processors, cpu);
125		for (;;)
126			__asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
127	}
128
129	pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
130		__func__, boot_secondary_processors, cpu);
131#endif
132	/* Init EXCSAVE1 */
133
134	secondary_trap_init();
135
136	/* All kernel threads share the same mm context. */
137
138	atomic_inc(&mm->mm_users);
139	atomic_inc(&mm->mm_count);
140	current->active_mm = mm;
141	cpumask_set_cpu(cpu, mm_cpumask(mm));
142	enter_lazy_tlb(mm, current);
143
144	preempt_disable();
145	trace_hardirqs_off();
146
147	calibrate_delay();
148
149	notify_cpu_starting(cpu);
150
151	secondary_init_irq();
152	local_timer_setup(cpu);
153
154	set_cpu_online(cpu, true);
155
156	local_irq_enable();
157
158	complete(&cpu_running);
159
160	cpu_startup_entry(CPUHP_ONLINE);
161}
162
163static void mx_cpu_start(void *p)
164{
165	unsigned cpu = (unsigned)p;
166	unsigned long run_stall_mask = get_er(MPSCORE);
167
168	set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
169	pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
170			__func__, cpu, run_stall_mask, get_er(MPSCORE));
171}
172
173static void mx_cpu_stop(void *p)
174{
175	unsigned cpu = (unsigned)p;
176	unsigned long run_stall_mask = get_er(MPSCORE);
177
178	set_er(run_stall_mask | (1u << cpu), MPSCORE);
179	pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
180			__func__, cpu, run_stall_mask, get_er(MPSCORE));
181}
182
183#ifdef CONFIG_HOTPLUG_CPU
184unsigned long cpu_start_id __cacheline_aligned;
185#endif
186unsigned long cpu_start_ccount;
187
188static int boot_secondary(unsigned int cpu, struct task_struct *ts)
189{
190	unsigned long timeout = jiffies + msecs_to_jiffies(1000);
191	unsigned long ccount;
192	int i;
193
194#ifdef CONFIG_HOTPLUG_CPU
195	cpu_start_id = cpu;
196	system_flush_invalidate_dcache_range(
197			(unsigned long)&cpu_start_id, sizeof(cpu_start_id));
198#endif
199	smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
200
201	for (i = 0; i < 2; ++i) {
202		do
203			ccount = get_ccount();
204		while (!ccount);
205
206		cpu_start_ccount = ccount;
207
208		while (time_before(jiffies, timeout)) {
209			mb();
210			if (!cpu_start_ccount)
211				break;
212		}
213
214		if (cpu_start_ccount) {
215			smp_call_function_single(0, mx_cpu_stop,
216					(void *)cpu, 1);
217			cpu_start_ccount = 0;
218			return -EIO;
219		}
220	}
221	return 0;
222}
223
224int __cpu_up(unsigned int cpu, struct task_struct *idle)
225{
226	int ret = 0;
227
228	if (cpu_asid_cache(cpu) == 0)
229		cpu_asid_cache(cpu) = ASID_USER_FIRST;
230
231	start_info.stack = (unsigned long)task_pt_regs(idle);
232	wmb();
233
234	pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
235			__func__, cpu, idle, start_info.stack);
236
237	ret = boot_secondary(cpu, idle);
238	if (ret == 0) {
239		wait_for_completion_timeout(&cpu_running,
240				msecs_to_jiffies(1000));
241		if (!cpu_online(cpu))
242			ret = -EIO;
243	}
244
245	if (ret)
246		pr_err("CPU %u failed to boot\n", cpu);
247
248	return ret;
249}
250
251#ifdef CONFIG_HOTPLUG_CPU
252
253/*
254 * __cpu_disable runs on the processor to be shutdown.
255 */
256int __cpu_disable(void)
257{
258	unsigned int cpu = smp_processor_id();
259
260	/*
261	 * Take this CPU offline.  Once we clear this, we can't return,
262	 * and we must not schedule until we're ready to give up the cpu.
263	 */
264	set_cpu_online(cpu, false);
265
266	/*
267	 * OK - migrate IRQs away from this CPU
268	 */
269	migrate_irqs();
270
271	/*
272	 * Flush user cache and TLB mappings, and then remove this CPU
273	 * from the vm mask set of all processes.
274	 */
275	local_flush_cache_all();
276	local_flush_tlb_all();
277	invalidate_page_directory();
278
279	clear_tasks_mm_cpumask(cpu);
280
281	return 0;
282}
283
284static void platform_cpu_kill(unsigned int cpu)
285{
286	smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
287}
288
289/*
290 * called on the thread which is asking for a CPU to be shutdown -
291 * waits until shutdown has completed, or it is timed out.
292 */
293void __cpu_die(unsigned int cpu)
294{
295	unsigned long timeout = jiffies + msecs_to_jiffies(1000);
296	while (time_before(jiffies, timeout)) {
297		system_invalidate_dcache_range((unsigned long)&cpu_start_id,
298				sizeof(cpu_start_id));
299		if (cpu_start_id == -cpu) {
300			platform_cpu_kill(cpu);
301			return;
302		}
303	}
304	pr_err("CPU%u: unable to kill\n", cpu);
305}
306
307void arch_cpu_idle_dead(void)
308{
309	cpu_die();
310}
311/*
312 * Called from the idle thread for the CPU which has been shutdown.
313 *
314 * Note that we disable IRQs here, but do not re-enable them
315 * before returning to the caller. This is also the behaviour
316 * of the other hotplug-cpu capable cores, so presumably coming
317 * out of idle fixes this.
318 */
319void __ref cpu_die(void)
320{
321	idle_task_exit();
322	local_irq_disable();
323	__asm__ __volatile__(
324			"	movi	a2, cpu_restart\n"
325			"	jx	a2\n");
326}
327
328#endif /* CONFIG_HOTPLUG_CPU */
329
330enum ipi_msg_type {
331	IPI_RESCHEDULE = 0,
332	IPI_CALL_FUNC,
333	IPI_CPU_STOP,
334	IPI_MAX
335};
336
337static const struct {
338	const char *short_text;
339	const char *long_text;
340} ipi_text[] = {
341	{ .short_text = "RES", .long_text = "Rescheduling interrupts" },
342	{ .short_text = "CAL", .long_text = "Function call interrupts" },
343	{ .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
344};
345
346struct ipi_data {
347	unsigned long ipi_count[IPI_MAX];
348};
349
350static DEFINE_PER_CPU(struct ipi_data, ipi_data);
351
352static void send_ipi_message(const struct cpumask *callmask,
353		enum ipi_msg_type msg_id)
354{
355	int index;
356	unsigned long mask = 0;
357
358	for_each_cpu(index, callmask)
359		if (index != smp_processor_id())
360			mask |= 1 << index;
361
362	set_er(mask, MIPISET(msg_id));
363}
364
365void arch_send_call_function_ipi_mask(const struct cpumask *mask)
366{
367	send_ipi_message(mask, IPI_CALL_FUNC);
368}
369
370void arch_send_call_function_single_ipi(int cpu)
371{
372	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
373}
374
375void smp_send_reschedule(int cpu)
376{
377	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
378}
379
380void smp_send_stop(void)
381{
382	struct cpumask targets;
383
384	cpumask_copy(&targets, cpu_online_mask);
385	cpumask_clear_cpu(smp_processor_id(), &targets);
386	send_ipi_message(&targets, IPI_CPU_STOP);
387}
388
389static void ipi_cpu_stop(unsigned int cpu)
390{
391	set_cpu_online(cpu, false);
392	machine_halt();
393}
394
395irqreturn_t ipi_interrupt(int irq, void *dev_id)
396{
397	unsigned int cpu = smp_processor_id();
398	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
399	unsigned int msg;
400	unsigned i;
401
402	msg = get_er(MIPICAUSE(cpu));
403	for (i = 0; i < IPI_MAX; i++)
404		if (msg & (1 << i)) {
405			set_er(1 << i, MIPICAUSE(cpu));
406			++ipi->ipi_count[i];
407		}
408
409	if (msg & (1 << IPI_RESCHEDULE))
410		scheduler_ipi();
411	if (msg & (1 << IPI_CALL_FUNC))
412		generic_smp_call_function_interrupt();
413	if (msg & (1 << IPI_CPU_STOP))
414		ipi_cpu_stop(cpu);
415
416	return IRQ_HANDLED;
417}
418
419void show_ipi_list(struct seq_file *p, int prec)
420{
421	unsigned int cpu;
422	unsigned i;
423
424	for (i = 0; i < IPI_MAX; ++i) {
425		seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
426		for_each_online_cpu(cpu)
427			seq_printf(p, " %10lu",
428					per_cpu(ipi_data, cpu).ipi_count[i]);
429		seq_printf(p, "   %s\n", ipi_text[i].long_text);
430	}
431}
432
433int setup_profiling_timer(unsigned int multiplier)
434{
435	pr_debug("setup_profiling_timer %d\n", multiplier);
436	return 0;
437}
438
439/* TLB flush functions */
440
441struct flush_data {
442	struct vm_area_struct *vma;
443	unsigned long addr1;
444	unsigned long addr2;
445};
446
447static void ipi_flush_tlb_all(void *arg)
448{
449	local_flush_tlb_all();
450}
451
452void flush_tlb_all(void)
453{
454	on_each_cpu(ipi_flush_tlb_all, NULL, 1);
455}
456
457static void ipi_flush_tlb_mm(void *arg)
458{
459	local_flush_tlb_mm(arg);
460}
461
462void flush_tlb_mm(struct mm_struct *mm)
463{
464	on_each_cpu(ipi_flush_tlb_mm, mm, 1);
465}
466
467static void ipi_flush_tlb_page(void *arg)
468{
469	struct flush_data *fd = arg;
470	local_flush_tlb_page(fd->vma, fd->addr1);
471}
472
473void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
474{
475	struct flush_data fd = {
476		.vma = vma,
477		.addr1 = addr,
478	};
479	on_each_cpu(ipi_flush_tlb_page, &fd, 1);
480}
481
482static void ipi_flush_tlb_range(void *arg)
483{
484	struct flush_data *fd = arg;
485	local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
486}
487
488void flush_tlb_range(struct vm_area_struct *vma,
489		     unsigned long start, unsigned long end)
490{
491	struct flush_data fd = {
492		.vma = vma,
493		.addr1 = start,
494		.addr2 = end,
495	};
496	on_each_cpu(ipi_flush_tlb_range, &fd, 1);
497}
498
499static void ipi_flush_tlb_kernel_range(void *arg)
500{
501	struct flush_data *fd = arg;
502	local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
503}
504
505void flush_tlb_kernel_range(unsigned long start, unsigned long end)
506{
507	struct flush_data fd = {
508		.addr1 = start,
509		.addr2 = end,
510	};
511	on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
512}
513
514/* Cache flush functions */
515
516static void ipi_flush_cache_all(void *arg)
517{
518	local_flush_cache_all();
519}
520
521void flush_cache_all(void)
522{
523	on_each_cpu(ipi_flush_cache_all, NULL, 1);
524}
525
526static void ipi_flush_cache_page(void *arg)
527{
528	struct flush_data *fd = arg;
529	local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
530}
531
532void flush_cache_page(struct vm_area_struct *vma,
533		     unsigned long address, unsigned long pfn)
534{
535	struct flush_data fd = {
536		.vma = vma,
537		.addr1 = address,
538		.addr2 = pfn,
539	};
540	on_each_cpu(ipi_flush_cache_page, &fd, 1);
541}
542
543static void ipi_flush_cache_range(void *arg)
544{
545	struct flush_data *fd = arg;
546	local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
547}
548
549void flush_cache_range(struct vm_area_struct *vma,
550		     unsigned long start, unsigned long end)
551{
552	struct flush_data fd = {
553		.vma = vma,
554		.addr1 = start,
555		.addr2 = end,
556	};
557	on_each_cpu(ipi_flush_cache_range, &fd, 1);
558}
559
560static void ipi_flush_icache_range(void *arg)
561{
562	struct flush_data *fd = arg;
563	local_flush_icache_range(fd->addr1, fd->addr2);
564}
565
566void flush_icache_range(unsigned long start, unsigned long end)
567{
568	struct flush_data fd = {
569		.addr1 = start,
570		.addr2 = end,
571	};
572	on_each_cpu(ipi_flush_icache_range, &fd, 1);
573}
574EXPORT_SYMBOL(flush_icache_range);
575
576/* ------------------------------------------------------------------------- */
577
578static void ipi_invalidate_dcache_range(void *arg)
579{
580	struct flush_data *fd = arg;
581	__invalidate_dcache_range(fd->addr1, fd->addr2);
582}
583
584static void system_invalidate_dcache_range(unsigned long start,
585		unsigned long size)
586{
587	struct flush_data fd = {
588		.addr1 = start,
589		.addr2 = size,
590	};
591	on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
592}
593
594static void ipi_flush_invalidate_dcache_range(void *arg)
595{
596	struct flush_data *fd = arg;
597	__flush_invalidate_dcache_range(fd->addr1, fd->addr2);
598}
599
600static void system_flush_invalidate_dcache_range(unsigned long start,
601		unsigned long size)
602{
603	struct flush_data fd = {
604		.addr1 = start,
605		.addr2 = size,
606	};
607	on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
608}
609