1/*
2 * Xtensa SMP support functions.
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License.  See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright (C) 2008 - 2013 Tensilica Inc.
9 *
10 * Chris Zankel <chris@zankel.net>
11 * Joe Taylor <joe@tensilica.com>
12 * Pete Delaney <piet@tensilica.com
13 */
14
15#include <linux/cpu.h>
16#include <linux/cpumask.h>
17#include <linux/delay.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/irqdomain.h>
21#include <linux/irq.h>
22#include <linux/kdebug.h>
23#include <linux/module.h>
24#include <linux/profile.h>
25#include <linux/sched/mm.h>
26#include <linux/sched/hotplug.h>
27#include <linux/sched/task_stack.h>
28#include <linux/reboot.h>
29#include <linux/seq_file.h>
30#include <linux/smp.h>
31#include <linux/thread_info.h>
32
33#include <asm/cacheflush.h>
34#include <asm/coprocessor.h>
35#include <asm/kdebug.h>
36#include <asm/mmu_context.h>
37#include <asm/mxregs.h>
38#include <asm/platform.h>
39#include <asm/tlbflush.h>
40#include <asm/traps.h>
41
42#ifdef CONFIG_SMP
43# if XCHAL_HAVE_S32C1I == 0
44#  error "The S32C1I option is required for SMP."
45# endif
46#endif
47
48static void system_invalidate_dcache_range(unsigned long start,
49		unsigned long size);
50static void system_flush_invalidate_dcache_range(unsigned long start,
51		unsigned long size);
52
53/* IPI (Inter Process Interrupt) */
54
55#define IPI_IRQ	0
56
57static irqreturn_t ipi_interrupt(int irq, void *dev_id);
58
59void ipi_init(void)
60{
61	unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
62	if (request_irq(irq, ipi_interrupt, IRQF_PERCPU, "ipi", NULL))
63		pr_err("Failed to request irq %u (ipi)\n", irq);
64}
65
66static inline unsigned int get_core_count(void)
67{
68	/* Bits 18..21 of SYSCFGID contain the core count minus 1. */
69	unsigned int syscfgid = get_er(SYSCFGID);
70	return ((syscfgid >> 18) & 0xf) + 1;
71}
72
73static inline int get_core_id(void)
74{
75	/* Bits 0...18 of SYSCFGID contain the core id  */
76	unsigned int core_id = get_er(SYSCFGID);
77	return core_id & 0x3fff;
78}
79
80void __init smp_prepare_cpus(unsigned int max_cpus)
81{
82	unsigned i;
83
84	for_each_possible_cpu(i)
85		set_cpu_present(i, true);
86}
87
88void __init smp_init_cpus(void)
89{
90	unsigned i;
91	unsigned int ncpus = get_core_count();
92	unsigned int core_id = get_core_id();
93
94	pr_info("%s: Core Count = %d\n", __func__, ncpus);
95	pr_info("%s: Core Id = %d\n", __func__, core_id);
96
97	if (ncpus > NR_CPUS) {
98		ncpus = NR_CPUS;
99		pr_info("%s: limiting core count by %d\n", __func__, ncpus);
100	}
101
102	for (i = 0; i < ncpus; ++i)
103		set_cpu_possible(i, true);
104}
105
106void __init smp_prepare_boot_cpu(void)
107{
108	unsigned int cpu = smp_processor_id();
109	BUG_ON(cpu != 0);
110	cpu_asid_cache(cpu) = ASID_USER_FIRST;
111}
112
113void __init smp_cpus_done(unsigned int max_cpus)
114{
115}
116
117static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
118static DECLARE_COMPLETION(cpu_running);
119
120void secondary_start_kernel(void)
121{
122	struct mm_struct *mm = &init_mm;
123	unsigned int cpu = smp_processor_id();
124
125	init_mmu();
126
127#ifdef CONFIG_DEBUG_MISC
128	if (boot_secondary_processors == 0) {
129		pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
130			__func__, boot_secondary_processors, cpu);
131		for (;;)
132			__asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
133	}
134
135	pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
136		__func__, boot_secondary_processors, cpu);
137#endif
138	/* Init EXCSAVE1 */
139
140	secondary_trap_init();
141
142	/* All kernel threads share the same mm context. */
143
144	mmget(mm);
145	mmgrab(mm);
146	current->active_mm = mm;
147	cpumask_set_cpu(cpu, mm_cpumask(mm));
148	enter_lazy_tlb(mm, current);
149
150	trace_hardirqs_off();
151
152	calibrate_delay();
153
154	notify_cpu_starting(cpu);
155
156	secondary_init_irq();
157	local_timer_setup(cpu);
158
159	set_cpu_online(cpu, true);
160
161	local_irq_enable();
162
163	complete(&cpu_running);
164
165	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
166}
167
168static void mx_cpu_start(void *p)
169{
170	unsigned cpu = (unsigned)p;
171	unsigned long run_stall_mask = get_er(MPSCORE);
172
173	set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
174	pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
175			__func__, cpu, run_stall_mask, get_er(MPSCORE));
176}
177
178static void mx_cpu_stop(void *p)
179{
180	unsigned cpu = (unsigned)p;
181	unsigned long run_stall_mask = get_er(MPSCORE);
182
183	set_er(run_stall_mask | (1u << cpu), MPSCORE);
184	pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
185			__func__, cpu, run_stall_mask, get_er(MPSCORE));
186}
187
188#ifdef CONFIG_HOTPLUG_CPU
189unsigned long cpu_start_id __cacheline_aligned;
190#endif
191unsigned long cpu_start_ccount;
192
193static int boot_secondary(unsigned int cpu, struct task_struct *ts)
194{
195	unsigned long timeout = jiffies + msecs_to_jiffies(1000);
196	unsigned long ccount;
197	int i;
198
199#ifdef CONFIG_HOTPLUG_CPU
200	WRITE_ONCE(cpu_start_id, cpu);
201	/* Pairs with the third memw in the cpu_restart */
202	mb();
203	system_flush_invalidate_dcache_range((unsigned long)&cpu_start_id,
204					     sizeof(cpu_start_id));
205#endif
206	smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
207
208	for (i = 0; i < 2; ++i) {
209		do
210			ccount = get_ccount();
211		while (!ccount);
212
213		WRITE_ONCE(cpu_start_ccount, ccount);
214
215		do {
216			/*
217			 * Pairs with the first two memws in the
218			 * .Lboot_secondary.
219			 */
220			mb();
221			ccount = READ_ONCE(cpu_start_ccount);
222		} while (ccount && time_before(jiffies, timeout));
223
224		if (ccount) {
225			smp_call_function_single(0, mx_cpu_stop,
226						 (void *)cpu, 1);
227			WRITE_ONCE(cpu_start_ccount, 0);
228			return -EIO;
229		}
230	}
231	return 0;
232}
233
234int __cpu_up(unsigned int cpu, struct task_struct *idle)
235{
236	int ret = 0;
237
238	if (cpu_asid_cache(cpu) == 0)
239		cpu_asid_cache(cpu) = ASID_USER_FIRST;
240
241	start_info.stack = (unsigned long)task_pt_regs(idle);
242	wmb();
243
244	pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
245			__func__, cpu, idle, start_info.stack);
246
247	init_completion(&cpu_running);
248	ret = boot_secondary(cpu, idle);
249	if (ret == 0) {
250		wait_for_completion_timeout(&cpu_running,
251				msecs_to_jiffies(1000));
252		if (!cpu_online(cpu))
253			ret = -EIO;
254	}
255
256	if (ret)
257		pr_err("CPU %u failed to boot\n", cpu);
258
259	return ret;
260}
261
262#ifdef CONFIG_HOTPLUG_CPU
263
264/*
265 * __cpu_disable runs on the processor to be shutdown.
266 */
267int __cpu_disable(void)
268{
269	unsigned int cpu = smp_processor_id();
270
271	/*
272	 * Take this CPU offline.  Once we clear this, we can't return,
273	 * and we must not schedule until we're ready to give up the cpu.
274	 */
275	set_cpu_online(cpu, false);
276
277#if XTENSA_HAVE_COPROCESSORS
278	/*
279	 * Flush coprocessor contexts that are active on the current CPU.
280	 */
281	local_coprocessors_flush_release_all();
282#endif
283	/*
284	 * OK - migrate IRQs away from this CPU
285	 */
286	migrate_irqs();
287
288	/*
289	 * Flush user cache and TLB mappings, and then remove this CPU
290	 * from the vm mask set of all processes.
291	 */
292	local_flush_cache_all();
293	local_flush_tlb_all();
294	invalidate_page_directory();
295
296	clear_tasks_mm_cpumask(cpu);
297
298	return 0;
299}
300
301static void platform_cpu_kill(unsigned int cpu)
302{
303	smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
304}
305
306/*
307 * called on the thread which is asking for a CPU to be shutdown -
308 * waits until shutdown has completed, or it is timed out.
309 */
310void __cpu_die(unsigned int cpu)
311{
312	unsigned long timeout = jiffies + msecs_to_jiffies(1000);
313	while (time_before(jiffies, timeout)) {
314		system_invalidate_dcache_range((unsigned long)&cpu_start_id,
315					       sizeof(cpu_start_id));
316		/* Pairs with the second memw in the cpu_restart */
317		mb();
318		if (READ_ONCE(cpu_start_id) == -cpu) {
319			platform_cpu_kill(cpu);
320			return;
321		}
322	}
323	pr_err("CPU%u: unable to kill\n", cpu);
324}
325
326void __noreturn arch_cpu_idle_dead(void)
327{
328	cpu_die();
329}
330/*
331 * Called from the idle thread for the CPU which has been shutdown.
332 *
333 * Note that we disable IRQs here, but do not re-enable them
334 * before returning to the caller. This is also the behaviour
335 * of the other hotplug-cpu capable cores, so presumably coming
336 * out of idle fixes this.
337 */
338void __ref cpu_die(void)
339{
340	idle_task_exit();
341	local_irq_disable();
342	__asm__ __volatile__(
343			"	movi	a2, cpu_restart\n"
344			"	jx	a2\n");
345
346	BUG();
347}
348
349#endif /* CONFIG_HOTPLUG_CPU */
350
351enum ipi_msg_type {
352	IPI_RESCHEDULE = 0,
353	IPI_CALL_FUNC,
354	IPI_CPU_STOP,
355	IPI_MAX
356};
357
358static const struct {
359	const char *short_text;
360	const char *long_text;
361} ipi_text[] = {
362	{ .short_text = "RES", .long_text = "Rescheduling interrupts" },
363	{ .short_text = "CAL", .long_text = "Function call interrupts" },
364	{ .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
365};
366
367struct ipi_data {
368	unsigned long ipi_count[IPI_MAX];
369};
370
371static DEFINE_PER_CPU(struct ipi_data, ipi_data);
372
373static void send_ipi_message(const struct cpumask *callmask,
374		enum ipi_msg_type msg_id)
375{
376	int index;
377	unsigned long mask = 0;
378
379	for_each_cpu(index, callmask)
380		mask |= 1 << index;
381
382	set_er(mask, MIPISET(msg_id));
383}
384
385void arch_send_call_function_ipi_mask(const struct cpumask *mask)
386{
387	send_ipi_message(mask, IPI_CALL_FUNC);
388}
389
390void arch_send_call_function_single_ipi(int cpu)
391{
392	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
393}
394
395void arch_smp_send_reschedule(int cpu)
396{
397	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
398}
399
400void smp_send_stop(void)
401{
402	struct cpumask targets;
403
404	cpumask_copy(&targets, cpu_online_mask);
405	cpumask_clear_cpu(smp_processor_id(), &targets);
406	send_ipi_message(&targets, IPI_CPU_STOP);
407}
408
409static void ipi_cpu_stop(unsigned int cpu)
410{
411	set_cpu_online(cpu, false);
412	machine_halt();
413}
414
415irqreturn_t ipi_interrupt(int irq, void *dev_id)
416{
417	unsigned int cpu = smp_processor_id();
418	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
419
420	for (;;) {
421		unsigned int msg;
422
423		msg = get_er(MIPICAUSE(cpu));
424		set_er(msg, MIPICAUSE(cpu));
425
426		if (!msg)
427			break;
428
429		if (msg & (1 << IPI_CALL_FUNC)) {
430			++ipi->ipi_count[IPI_CALL_FUNC];
431			generic_smp_call_function_interrupt();
432		}
433
434		if (msg & (1 << IPI_RESCHEDULE)) {
435			++ipi->ipi_count[IPI_RESCHEDULE];
436			scheduler_ipi();
437		}
438
439		if (msg & (1 << IPI_CPU_STOP)) {
440			++ipi->ipi_count[IPI_CPU_STOP];
441			ipi_cpu_stop(cpu);
442		}
443	}
444
445	return IRQ_HANDLED;
446}
447
448void show_ipi_list(struct seq_file *p, int prec)
449{
450	unsigned int cpu;
451	unsigned i;
452
453	for (i = 0; i < IPI_MAX; ++i) {
454		seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
455		for_each_online_cpu(cpu)
456			seq_printf(p, " %10lu",
457					per_cpu(ipi_data, cpu).ipi_count[i]);
458		seq_printf(p, "   %s\n", ipi_text[i].long_text);
459	}
460}
461
462int setup_profiling_timer(unsigned int multiplier)
463{
464	pr_debug("setup_profiling_timer %d\n", multiplier);
465	return 0;
466}
467
468/* TLB flush functions */
469
470struct flush_data {
471	struct vm_area_struct *vma;
472	unsigned long addr1;
473	unsigned long addr2;
474};
475
476static void ipi_flush_tlb_all(void *arg)
477{
478	local_flush_tlb_all();
479}
480
481void flush_tlb_all(void)
482{
483	on_each_cpu(ipi_flush_tlb_all, NULL, 1);
484}
485
486static void ipi_flush_tlb_mm(void *arg)
487{
488	local_flush_tlb_mm(arg);
489}
490
491void flush_tlb_mm(struct mm_struct *mm)
492{
493	on_each_cpu(ipi_flush_tlb_mm, mm, 1);
494}
495
496static void ipi_flush_tlb_page(void *arg)
497{
498	struct flush_data *fd = arg;
499	local_flush_tlb_page(fd->vma, fd->addr1);
500}
501
502void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
503{
504	struct flush_data fd = {
505		.vma = vma,
506		.addr1 = addr,
507	};
508	on_each_cpu(ipi_flush_tlb_page, &fd, 1);
509}
510
511static void ipi_flush_tlb_range(void *arg)
512{
513	struct flush_data *fd = arg;
514	local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
515}
516
517void flush_tlb_range(struct vm_area_struct *vma,
518		     unsigned long start, unsigned long end)
519{
520	struct flush_data fd = {
521		.vma = vma,
522		.addr1 = start,
523		.addr2 = end,
524	};
525	on_each_cpu(ipi_flush_tlb_range, &fd, 1);
526}
527
528static void ipi_flush_tlb_kernel_range(void *arg)
529{
530	struct flush_data *fd = arg;
531	local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
532}
533
534void flush_tlb_kernel_range(unsigned long start, unsigned long end)
535{
536	struct flush_data fd = {
537		.addr1 = start,
538		.addr2 = end,
539	};
540	on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
541}
542
543/* Cache flush functions */
544
545static void ipi_flush_cache_all(void *arg)
546{
547	local_flush_cache_all();
548}
549
550void flush_cache_all(void)
551{
552	on_each_cpu(ipi_flush_cache_all, NULL, 1);
553}
554
555static void ipi_flush_cache_page(void *arg)
556{
557	struct flush_data *fd = arg;
558	local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
559}
560
561void flush_cache_page(struct vm_area_struct *vma,
562		     unsigned long address, unsigned long pfn)
563{
564	struct flush_data fd = {
565		.vma = vma,
566		.addr1 = address,
567		.addr2 = pfn,
568	};
569	on_each_cpu(ipi_flush_cache_page, &fd, 1);
570}
571
572static void ipi_flush_cache_range(void *arg)
573{
574	struct flush_data *fd = arg;
575	local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
576}
577
578void flush_cache_range(struct vm_area_struct *vma,
579		     unsigned long start, unsigned long end)
580{
581	struct flush_data fd = {
582		.vma = vma,
583		.addr1 = start,
584		.addr2 = end,
585	};
586	on_each_cpu(ipi_flush_cache_range, &fd, 1);
587}
588
589static void ipi_flush_icache_range(void *arg)
590{
591	struct flush_data *fd = arg;
592	local_flush_icache_range(fd->addr1, fd->addr2);
593}
594
595void flush_icache_range(unsigned long start, unsigned long end)
596{
597	struct flush_data fd = {
598		.addr1 = start,
599		.addr2 = end,
600	};
601	on_each_cpu(ipi_flush_icache_range, &fd, 1);
602}
603EXPORT_SYMBOL(flush_icache_range);
604
605/* ------------------------------------------------------------------------- */
606
607static void ipi_invalidate_dcache_range(void *arg)
608{
609	struct flush_data *fd = arg;
610	__invalidate_dcache_range(fd->addr1, fd->addr2);
611}
612
613static void system_invalidate_dcache_range(unsigned long start,
614		unsigned long size)
615{
616	struct flush_data fd = {
617		.addr1 = start,
618		.addr2 = size,
619	};
620	on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
621}
622
623static void ipi_flush_invalidate_dcache_range(void *arg)
624{
625	struct flush_data *fd = arg;
626	__flush_invalidate_dcache_range(fd->addr1, fd->addr2);
627}
628
629static void system_flush_invalidate_dcache_range(unsigned long start,
630		unsigned long size)
631{
632	struct flush_data fd = {
633		.addr1 = start,
634		.addr2 = size,
635	};
636	on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
637}
638