1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * machine_kexec.c for kexec
4 *
5 * Copyright (C) 2022 Loongson Technology Corporation Limited
6 */
7#include <linux/compiler.h>
8#include <linux/cpu.h>
9#include <linux/kexec.h>
10#include <linux/crash_dump.h>
11#include <linux/delay.h>
12#include <linux/irq.h>
13#include <linux/libfdt.h>
14#include <linux/mm.h>
15#include <linux/of_fdt.h>
16#include <linux/reboot.h>
17#include <linux/sched.h>
18#include <linux/sched/task_stack.h>
19
20#include <asm/bootinfo.h>
21#include <asm/cacheflush.h>
22#include <asm/page.h>
23
24/* 0x100000 ~ 0x200000 is safe */
25#define KEXEC_CONTROL_CODE	TO_CACHE(0x100000UL)
26#define KEXEC_CMDLINE_ADDR	TO_CACHE(0x108000UL)
27
28static unsigned long reboot_code_buffer;
29static cpumask_t cpus_in_crash = CPU_MASK_NONE;
30
31#ifdef CONFIG_SMP
32static void (*relocated_kexec_smp_wait)(void *);
33atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0);
34#endif
35
36static unsigned long efi_boot;
37static unsigned long cmdline_ptr;
38static unsigned long systable_ptr;
39static unsigned long start_addr;
40static unsigned long first_ind_entry;
41
42static void kexec_image_info(const struct kimage *kimage)
43{
44	unsigned long i;
45
46	pr_debug("kexec kimage info:\n");
47	pr_debug("\ttype:        %d\n", kimage->type);
48	pr_debug("\tstart:       %lx\n", kimage->start);
49	pr_debug("\thead:        %lx\n", kimage->head);
50	pr_debug("\tnr_segments: %lu\n", kimage->nr_segments);
51
52	for (i = 0; i < kimage->nr_segments; i++) {
53		pr_debug("\t    segment[%lu]: %016lx - %016lx", i,
54			kimage->segment[i].mem,
55			kimage->segment[i].mem + kimage->segment[i].memsz);
56		pr_debug("\t\t0x%lx bytes, %lu pages\n",
57			(unsigned long)kimage->segment[i].memsz,
58			(unsigned long)kimage->segment[i].memsz /  PAGE_SIZE);
59	}
60}
61
62int machine_kexec_prepare(struct kimage *kimage)
63{
64	int i;
65	char *bootloader = "kexec";
66	void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR;
67
68	kexec_image_info(kimage);
69
70	kimage->arch.efi_boot = fw_arg0;
71	kimage->arch.systable_ptr = fw_arg2;
72
73	/* Find the command line */
74	for (i = 0; i < kimage->nr_segments; i++) {
75		if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) {
76			if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE))
77				kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr;
78			break;
79		}
80	}
81
82	if (!kimage->arch.cmdline_ptr) {
83		pr_err("Command line not included in the provided image\n");
84		return -EINVAL;
85	}
86
87	/* kexec/kdump need a safe page to save reboot_code_buffer */
88	kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE);
89
90	reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page);
91	memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size);
92
93#ifdef CONFIG_SMP
94	/* All secondary cpus now may jump to kexec_smp_wait cycle */
95	relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel);
96#endif
97
98	return 0;
99}
100
101void machine_kexec_cleanup(struct kimage *kimage)
102{
103}
104
105void kexec_reboot(void)
106{
107	do_kexec_t do_kexec = NULL;
108
109	/*
110	 * We know we were online, and there will be no incoming IPIs at
111	 * this point. Mark online again before rebooting so that the crash
112	 * analysis tool will see us correctly.
113	 */
114	set_cpu_online(smp_processor_id(), true);
115
116	/* Ensure remote CPUs observe that we're online before rebooting. */
117	smp_mb__after_atomic();
118
119	/*
120	 * Make sure we get correct instructions written by the
121	 * machine_kexec_prepare() CPU.
122	 */
123	__asm__ __volatile__ ("\tibar 0\n"::);
124
125#ifdef CONFIG_SMP
126	/* All secondary cpus go to kexec_smp_wait */
127	if (smp_processor_id() > 0) {
128		relocated_kexec_smp_wait(NULL);
129		unreachable();
130	}
131#endif
132
133	do_kexec = (void *)reboot_code_buffer;
134	do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry);
135
136	unreachable();
137}
138
139
140#ifdef CONFIG_SMP
141static void kexec_shutdown_secondary(void *regs)
142{
143	int cpu = smp_processor_id();
144
145	if (!cpu_online(cpu))
146		return;
147
148	/* We won't be sent IPIs any more. */
149	set_cpu_online(cpu, false);
150
151	local_irq_disable();
152	while (!atomic_read(&kexec_ready_to_reboot))
153		cpu_relax();
154
155	kexec_reboot();
156}
157
158static void crash_shutdown_secondary(void *passed_regs)
159{
160	int cpu = smp_processor_id();
161	struct pt_regs *regs = passed_regs;
162
163	/*
164	 * If we are passed registers, use those. Otherwise get the
165	 * regs from the last interrupt, which should be correct, as
166	 * we are in an interrupt. But if the regs are not there,
167	 * pull them from the top of the stack. They are probably
168	 * wrong, but we need something to keep from crashing again.
169	 */
170	if (!regs)
171		regs = get_irq_regs();
172	if (!regs)
173		regs = task_pt_regs(current);
174
175	if (!cpu_online(cpu))
176		return;
177
178	/* We won't be sent IPIs any more. */
179	set_cpu_online(cpu, false);
180
181	local_irq_disable();
182	if (!cpumask_test_cpu(cpu, &cpus_in_crash))
183		crash_save_cpu(regs, cpu);
184	cpumask_set_cpu(cpu, &cpus_in_crash);
185
186	while (!atomic_read(&kexec_ready_to_reboot))
187		cpu_relax();
188
189	kexec_reboot();
190}
191
192void crash_smp_send_stop(void)
193{
194	unsigned int ncpus;
195	unsigned long timeout;
196	static int cpus_stopped;
197
198	/*
199	 * This function can be called twice in panic path, but obviously
200	 * we should execute this only once.
201	 */
202	if (cpus_stopped)
203		return;
204
205	cpus_stopped = 1;
206
207	 /* Excluding the panic cpu */
208	ncpus = num_online_cpus() - 1;
209
210	smp_call_function(crash_shutdown_secondary, NULL, 0);
211	smp_wmb();
212
213	/*
214	 * The crash CPU sends an IPI and wait for other CPUs to
215	 * respond. Delay of at least 10 seconds.
216	 */
217	timeout = MSEC_PER_SEC * 10;
218	pr_emerg("Sending IPI to other cpus...\n");
219	while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) {
220		mdelay(1);
221		cpu_relax();
222	}
223}
224#endif /* defined(CONFIG_SMP) */
225
226void machine_shutdown(void)
227{
228	int cpu;
229
230	/* All CPUs go to reboot_code_buffer */
231	for_each_possible_cpu(cpu)
232		if (!cpu_online(cpu))
233			cpu_device_up(get_cpu_device(cpu));
234
235#ifdef CONFIG_SMP
236	smp_call_function(kexec_shutdown_secondary, NULL, 0);
237#endif
238}
239
240void machine_crash_shutdown(struct pt_regs *regs)
241{
242	int crashing_cpu;
243
244	local_irq_disable();
245
246	crashing_cpu = smp_processor_id();
247	crash_save_cpu(regs, crashing_cpu);
248
249#ifdef CONFIG_SMP
250	crash_smp_send_stop();
251#endif
252	cpumask_set_cpu(crashing_cpu, &cpus_in_crash);
253
254	pr_info("Starting crashdump kernel...\n");
255}
256
257void machine_kexec(struct kimage *image)
258{
259	unsigned long entry, *ptr;
260	struct kimage_arch *internal = &image->arch;
261
262	efi_boot = internal->efi_boot;
263	cmdline_ptr = internal->cmdline_ptr;
264	systable_ptr = internal->systable_ptr;
265
266	start_addr = (unsigned long)phys_to_virt(image->start);
267
268	first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ?
269		(unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0;
270
271	/*
272	 * The generic kexec code builds a page list with physical
273	 * addresses. they are directly accessible through XKPRANGE
274	 * hence the phys_to_virt() call.
275	 */
276	for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE);
277	     ptr = (entry & IND_INDIRECTION) ?
278	       phys_to_virt(entry & PAGE_MASK) : ptr + 1) {
279		if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION ||
280		    *ptr & IND_DESTINATION)
281			*ptr = (unsigned long) phys_to_virt(*ptr);
282	}
283
284	/* Mark offline before disabling local irq. */
285	set_cpu_online(smp_processor_id(), false);
286
287	/* We do not want to be bothered. */
288	local_irq_disable();
289
290	pr_notice("EFI boot flag 0x%lx\n", efi_boot);
291	pr_notice("Command line at 0x%lx\n", cmdline_ptr);
292	pr_notice("System table at 0x%lx\n", systable_ptr);
293	pr_notice("We will call new kernel at 0x%lx\n", start_addr);
294	pr_notice("Bye ...\n");
295
296	/* Make reboot code buffer available to the boot CPU. */
297	flush_cache_all();
298
299#ifdef CONFIG_SMP
300	atomic_set(&kexec_ready_to_reboot, 1);
301#endif
302
303	kexec_reboot();
304}
305