1// SPDX-License-Identifier: GPL-2.0+
2
3#define pr_fmt(fmt) "kprobes: " fmt
4
5#include <linux/kprobes.h>
6#include <linux/extable.h>
7#include <linux/slab.h>
8#include <linux/stop_machine.h>
9#include <asm/ptrace.h>
10#include <linux/uaccess.h>
11#include <asm/sections.h>
12#include <asm/cacheflush.h>
13#include <asm/bug.h>
14#include <asm/patch.h>
15
16#include "decode-insn.h"
17
18DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
19DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
20
21static void __kprobes
22post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *);
23
24static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
25{
26	u32 insn = __BUG_INSN_32;
27	unsigned long offset = GET_INSN_LENGTH(p->opcode);
28
29	p->ainsn.api.restore = (unsigned long)p->addr + offset;
30
31	patch_text(p->ainsn.api.insn, &p->opcode, 1);
32	patch_text((void *)((unsigned long)(p->ainsn.api.insn) + offset),
33		   &insn, 1);
34}
35
36static void __kprobes arch_prepare_simulate(struct kprobe *p)
37{
38	p->ainsn.api.restore = 0;
39}
40
41static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
42{
43	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
44
45	if (p->ainsn.api.handler)
46		p->ainsn.api.handler((u32)p->opcode,
47					(unsigned long)p->addr, regs);
48
49	post_kprobe_handler(p, kcb, regs);
50}
51
52static bool __kprobes arch_check_kprobe(struct kprobe *p)
53{
54	unsigned long tmp  = (unsigned long)p->addr - p->offset;
55	unsigned long addr = (unsigned long)p->addr;
56
57	while (tmp <= addr) {
58		if (tmp == addr)
59			return true;
60
61		tmp += GET_INSN_LENGTH(*(u16 *)tmp);
62	}
63
64	return false;
65}
66
67int __kprobes arch_prepare_kprobe(struct kprobe *p)
68{
69	u16 *insn = (u16 *)p->addr;
70
71	if ((unsigned long)insn & 0x1)
72		return -EILSEQ;
73
74	if (!arch_check_kprobe(p))
75		return -EILSEQ;
76
77	/* copy instruction */
78	p->opcode = (kprobe_opcode_t)(*insn++);
79	if (GET_INSN_LENGTH(p->opcode) == 4)
80		p->opcode |= (kprobe_opcode_t)(*insn) << 16;
81
82	/* decode instruction */
83	switch (riscv_probe_decode_insn(p->addr, &p->ainsn.api)) {
84	case INSN_REJECTED:	/* insn not supported */
85		return -EINVAL;
86
87	case INSN_GOOD_NO_SLOT:	/* insn need simulation */
88		p->ainsn.api.insn = NULL;
89		break;
90
91	case INSN_GOOD:	/* instruction uses slot */
92		p->ainsn.api.insn = get_insn_slot();
93		if (!p->ainsn.api.insn)
94			return -ENOMEM;
95		break;
96	}
97
98	/* prepare the instruction */
99	if (p->ainsn.api.insn)
100		arch_prepare_ss_slot(p);
101	else
102		arch_prepare_simulate(p);
103
104	return 0;
105}
106
107#ifdef CONFIG_MMU
108void *alloc_insn_page(void)
109{
110	return  __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, VMALLOC_END,
111				     GFP_KERNEL, PAGE_KERNEL_READ_EXEC,
112				     VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
113				     __builtin_return_address(0));
114}
115#endif
116
117/* install breakpoint in text */
118void __kprobes arch_arm_kprobe(struct kprobe *p)
119{
120	u32 insn = (p->opcode & __INSN_LENGTH_MASK) == __INSN_LENGTH_32 ?
121		   __BUG_INSN_32 : __BUG_INSN_16;
122
123	patch_text(p->addr, &insn, 1);
124}
125
126/* remove breakpoint from text */
127void __kprobes arch_disarm_kprobe(struct kprobe *p)
128{
129	patch_text(p->addr, &p->opcode, 1);
130}
131
132void __kprobes arch_remove_kprobe(struct kprobe *p)
133{
134}
135
136static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
137{
138	kcb->prev_kprobe.kp = kprobe_running();
139	kcb->prev_kprobe.status = kcb->kprobe_status;
140}
141
142static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
143{
144	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
145	kcb->kprobe_status = kcb->prev_kprobe.status;
146}
147
148static void __kprobes set_current_kprobe(struct kprobe *p)
149{
150	__this_cpu_write(current_kprobe, p);
151}
152
153/*
154 * Interrupts need to be disabled before single-step mode is set, and not
155 * reenabled until after single-step mode ends.
156 * Without disabling interrupt on local CPU, there is a chance of
157 * interrupt occurrence in the period of exception return and  start of
158 * out-of-line single-step, that result in wrongly single stepping
159 * into the interrupt handler.
160 */
161static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
162						struct pt_regs *regs)
163{
164	kcb->saved_status = regs->status;
165	regs->status &= ~SR_SPIE;
166}
167
168static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
169						struct pt_regs *regs)
170{
171	regs->status = kcb->saved_status;
172}
173
174static void __kprobes setup_singlestep(struct kprobe *p,
175				       struct pt_regs *regs,
176				       struct kprobe_ctlblk *kcb, int reenter)
177{
178	unsigned long slot;
179
180	if (reenter) {
181		save_previous_kprobe(kcb);
182		set_current_kprobe(p);
183		kcb->kprobe_status = KPROBE_REENTER;
184	} else {
185		kcb->kprobe_status = KPROBE_HIT_SS;
186	}
187
188	if (p->ainsn.api.insn) {
189		/* prepare for single stepping */
190		slot = (unsigned long)p->ainsn.api.insn;
191
192		/* IRQs and single stepping do not mix well. */
193		kprobes_save_local_irqflag(kcb, regs);
194
195		instruction_pointer_set(regs, slot);
196	} else {
197		/* insn simulation */
198		arch_simulate_insn(p, regs);
199	}
200}
201
202static int __kprobes reenter_kprobe(struct kprobe *p,
203				    struct pt_regs *regs,
204				    struct kprobe_ctlblk *kcb)
205{
206	switch (kcb->kprobe_status) {
207	case KPROBE_HIT_SSDONE:
208	case KPROBE_HIT_ACTIVE:
209		kprobes_inc_nmissed_count(p);
210		setup_singlestep(p, regs, kcb, 1);
211		break;
212	case KPROBE_HIT_SS:
213	case KPROBE_REENTER:
214		pr_warn("Failed to recover from reentered kprobes.\n");
215		dump_kprobe(p);
216		BUG();
217		break;
218	default:
219		WARN_ON(1);
220		return 0;
221	}
222
223	return 1;
224}
225
226static void __kprobes
227post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_regs *regs)
228{
229	/* return addr restore if non-branching insn */
230	if (cur->ainsn.api.restore != 0)
231		regs->epc = cur->ainsn.api.restore;
232
233	/* restore back original saved kprobe variables and continue */
234	if (kcb->kprobe_status == KPROBE_REENTER) {
235		restore_previous_kprobe(kcb);
236		return;
237	}
238
239	/* call post handler */
240	kcb->kprobe_status = KPROBE_HIT_SSDONE;
241	if (cur->post_handler)	{
242		/* post_handler can hit breakpoint and single step
243		 * again, so we enable D-flag for recursive exception.
244		 */
245		cur->post_handler(cur, regs, 0);
246	}
247
248	reset_current_kprobe();
249}
250
251int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr)
252{
253	struct kprobe *cur = kprobe_running();
254	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
255
256	switch (kcb->kprobe_status) {
257	case KPROBE_HIT_SS:
258	case KPROBE_REENTER:
259		/*
260		 * We are here because the instruction being single
261		 * stepped caused a page fault. We reset the current
262		 * kprobe and the ip points back to the probe address
263		 * and allow the page fault handler to continue as a
264		 * normal page fault.
265		 */
266		regs->epc = (unsigned long) cur->addr;
267		BUG_ON(!instruction_pointer(regs));
268
269		if (kcb->kprobe_status == KPROBE_REENTER)
270			restore_previous_kprobe(kcb);
271		else {
272			kprobes_restore_local_irqflag(kcb, regs);
273			reset_current_kprobe();
274		}
275
276		break;
277	case KPROBE_HIT_ACTIVE:
278	case KPROBE_HIT_SSDONE:
279		/*
280		 * In case the user-specified fault handler returned
281		 * zero, try to fix up.
282		 */
283		if (fixup_exception(regs))
284			return 1;
285	}
286	return 0;
287}
288
289bool __kprobes
290kprobe_breakpoint_handler(struct pt_regs *regs)
291{
292	struct kprobe *p, *cur_kprobe;
293	struct kprobe_ctlblk *kcb;
294	unsigned long addr = instruction_pointer(regs);
295
296	kcb = get_kprobe_ctlblk();
297	cur_kprobe = kprobe_running();
298
299	p = get_kprobe((kprobe_opcode_t *) addr);
300
301	if (p) {
302		if (cur_kprobe) {
303			if (reenter_kprobe(p, regs, kcb))
304				return true;
305		} else {
306			/* Probe hit */
307			set_current_kprobe(p);
308			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
309
310			/*
311			 * If we have no pre-handler or it returned 0, we
312			 * continue with normal processing.  If we have a
313			 * pre-handler and it returned non-zero, it will
314			 * modify the execution path and no need to single
315			 * stepping. Let's just reset current kprobe and exit.
316			 *
317			 * pre_handler can hit a breakpoint and can step thru
318			 * before return.
319			 */
320			if (!p->pre_handler || !p->pre_handler(p, regs))
321				setup_singlestep(p, regs, kcb, 0);
322			else
323				reset_current_kprobe();
324		}
325		return true;
326	}
327
328	/*
329	 * The breakpoint instruction was removed right
330	 * after we hit it.  Another cpu has removed
331	 * either a probepoint or a debugger breakpoint
332	 * at this address.  In either case, no further
333	 * handling of this interrupt is appropriate.
334	 * Return back to original instruction, and continue.
335	 */
336	return false;
337}
338
339bool __kprobes
340kprobe_single_step_handler(struct pt_regs *regs)
341{
342	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
343	unsigned long addr = instruction_pointer(regs);
344	struct kprobe *cur = kprobe_running();
345
346	if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) &&
347	    ((unsigned long)&cur->ainsn.api.insn[0] + GET_INSN_LENGTH(cur->opcode) == addr)) {
348		kprobes_restore_local_irqflag(kcb, regs);
349		post_kprobe_handler(cur, kcb, regs);
350		return true;
351	}
352	/* not ours, kprobes should ignore it */
353	return false;
354}
355
356/*
357 * Provide a blacklist of symbols identifying ranges which cannot be kprobed.
358 * This blacklist is exposed to userspace via debugfs (kprobes/blacklist).
359 */
360int __init arch_populate_kprobe_blacklist(void)
361{
362	int ret;
363
364	ret = kprobe_add_area_blacklist((unsigned long)__irqentry_text_start,
365					(unsigned long)__irqentry_text_end);
366	return ret;
367}
368
369int __kprobes arch_trampoline_kprobe(struct kprobe *p)
370{
371	return 0;
372}
373
374int __init arch_init_kprobes(void)
375{
376	return 0;
377}
378