1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 *  Kernel Probes (KProbes)
4 *
5 * Copyright (C) IBM Corporation, 2002, 2004
6 *
7 * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
8 *		Probes initial implementation ( includes contributions from
9 *		Rusty Russell).
10 * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
11 *		interface to access function arguments.
12 * 2004-Nov	Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
13 *		for PPC64
14 */
15
16#include <linux/kprobes.h>
17#include <linux/ptrace.h>
18#include <linux/preempt.h>
19#include <linux/extable.h>
20#include <linux/kdebug.h>
21#include <linux/slab.h>
22#include <linux/moduleloader.h>
23#include <linux/set_memory.h>
24#include <asm/code-patching.h>
25#include <asm/cacheflush.h>
26#include <asm/sstep.h>
27#include <asm/sections.h>
28#include <asm/inst.h>
29#include <linux/uaccess.h>
30
31DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
32DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
33
34struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
35
36bool arch_within_kprobe_blacklist(unsigned long addr)
37{
38	return  (addr >= (unsigned long)__kprobes_text_start &&
39		 addr < (unsigned long)__kprobes_text_end) ||
40		(addr >= (unsigned long)_stext &&
41		 addr < (unsigned long)__head_end);
42}
43
44kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
45{
46	kprobe_opcode_t *addr = NULL;
47
48#ifdef CONFIG_PPC64_ELF_ABI_V2
49	/* PPC64 ABIv2 needs local entry point */
50	addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
51	if (addr && !offset) {
52#ifdef CONFIG_KPROBES_ON_FTRACE
53		unsigned long faddr;
54		/*
55		 * Per livepatch.h, ftrace location is always within the first
56		 * 16 bytes of a function on powerpc with -mprofile-kernel.
57		 */
58		faddr = ftrace_location_range((unsigned long)addr,
59					      (unsigned long)addr + 16);
60		if (faddr)
61			addr = (kprobe_opcode_t *)faddr;
62		else
63#endif
64			addr = (kprobe_opcode_t *)ppc_function_entry(addr);
65	}
66#elif defined(CONFIG_PPC64_ELF_ABI_V1)
67	/*
68	 * 64bit powerpc ABIv1 uses function descriptors:
69	 * - Check for the dot variant of the symbol first.
70	 * - If that fails, try looking up the symbol provided.
71	 *
72	 * This ensures we always get to the actual symbol and not
73	 * the descriptor.
74	 *
75	 * Also handle <module:symbol> format.
76	 */
77	char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
78	bool dot_appended = false;
79	const char *c;
80	ssize_t ret = 0;
81	int len = 0;
82
83	if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
84		c++;
85		len = c - name;
86		memcpy(dot_name, name, len);
87	} else
88		c = name;
89
90	if (*c != '\0' && *c != '.') {
91		dot_name[len++] = '.';
92		dot_appended = true;
93	}
94	ret = strscpy(dot_name + len, c, KSYM_NAME_LEN);
95	if (ret > 0)
96		addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
97
98	/* Fallback to the original non-dot symbol lookup */
99	if (!addr && dot_appended)
100		addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
101#else
102	addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
103#endif
104
105	return addr;
106}
107
108static bool arch_kprobe_on_func_entry(unsigned long offset)
109{
110#ifdef CONFIG_PPC64_ELF_ABI_V2
111#ifdef CONFIG_KPROBES_ON_FTRACE
112	return offset <= 16;
113#else
114	return offset <= 8;
115#endif
116#else
117	return !offset;
118#endif
119}
120
121/* XXX try and fold the magic of kprobe_lookup_name() in this */
122kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
123					 bool *on_func_entry)
124{
125	*on_func_entry = arch_kprobe_on_func_entry(offset);
126	return (kprobe_opcode_t *)(addr + offset);
127}
128
129void *alloc_insn_page(void)
130{
131	void *page;
132
133	page = module_alloc(PAGE_SIZE);
134	if (!page)
135		return NULL;
136
137	if (strict_module_rwx_enabled()) {
138		int err = set_memory_rox((unsigned long)page, 1);
139
140		if (err)
141			goto error;
142	}
143	return page;
144error:
145	module_memfree(page);
146	return NULL;
147}
148
149int arch_prepare_kprobe(struct kprobe *p)
150{
151	int ret = 0;
152	struct kprobe *prev;
153	ppc_inst_t insn = ppc_inst_read(p->addr);
154
155	if ((unsigned long)p->addr & 0x03) {
156		printk("Attempt to register kprobe at an unaligned address\n");
157		ret = -EINVAL;
158	} else if (!can_single_step(ppc_inst_val(insn))) {
159		printk("Cannot register a kprobe on instructions that can't be single stepped\n");
160		ret = -EINVAL;
161	} else if ((unsigned long)p->addr & ~PAGE_MASK &&
162		   ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) {
163		printk("Cannot register a kprobe on the second word of prefixed instruction\n");
164		ret = -EINVAL;
165	}
166	prev = get_kprobe(p->addr - 1);
167
168	/*
169	 * When prev is a ftrace-based kprobe, we don't have an insn, and it
170	 * doesn't probe for prefixed instruction.
171	 */
172	if (prev && !kprobe_ftrace(prev) &&
173	    ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) {
174		printk("Cannot register a kprobe on the second word of prefixed instruction\n");
175		ret = -EINVAL;
176	}
177
178	/* insn must be on a special executable page on ppc64.  This is
179	 * not explicitly required on ppc32 (right now), but it doesn't hurt */
180	if (!ret) {
181		p->ainsn.insn = get_insn_slot();
182		if (!p->ainsn.insn)
183			ret = -ENOMEM;
184	}
185
186	if (!ret) {
187		patch_instruction(p->ainsn.insn, insn);
188		p->opcode = ppc_inst_val(insn);
189	}
190
191	p->ainsn.boostable = 0;
192	return ret;
193}
194NOKPROBE_SYMBOL(arch_prepare_kprobe);
195
196void arch_arm_kprobe(struct kprobe *p)
197{
198	WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION)));
199}
200NOKPROBE_SYMBOL(arch_arm_kprobe);
201
202void arch_disarm_kprobe(struct kprobe *p)
203{
204	WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode)));
205}
206NOKPROBE_SYMBOL(arch_disarm_kprobe);
207
208void arch_remove_kprobe(struct kprobe *p)
209{
210	if (p->ainsn.insn) {
211		free_insn_slot(p->ainsn.insn, 0);
212		p->ainsn.insn = NULL;
213	}
214}
215NOKPROBE_SYMBOL(arch_remove_kprobe);
216
217static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
218{
219	enable_single_step(regs);
220
221	/*
222	 * On powerpc we should single step on the original
223	 * instruction even if the probed insn is a trap
224	 * variant as values in regs could play a part in
225	 * if the trap is taken or not
226	 */
227	regs_set_return_ip(regs, (unsigned long)p->ainsn.insn);
228}
229
230static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
231{
232	kcb->prev_kprobe.kp = kprobe_running();
233	kcb->prev_kprobe.status = kcb->kprobe_status;
234	kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
235}
236
237static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
238{
239	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
240	kcb->kprobe_status = kcb->prev_kprobe.status;
241	kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
242}
243
244static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
245				struct kprobe_ctlblk *kcb)
246{
247	__this_cpu_write(current_kprobe, p);
248	kcb->kprobe_saved_msr = regs->msr;
249}
250
251void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
252{
253	ri->ret_addr = (kprobe_opcode_t *)regs->link;
254	ri->fp = NULL;
255
256	/* Replace the return addr with trampoline addr */
257	regs->link = (unsigned long)__kretprobe_trampoline;
258}
259NOKPROBE_SYMBOL(arch_prepare_kretprobe);
260
261static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
262{
263	int ret;
264	ppc_inst_t insn = ppc_inst_read(p->ainsn.insn);
265
266	/* regs->nip is also adjusted if emulate_step returns 1 */
267	ret = emulate_step(regs, insn);
268	if (ret > 0) {
269		/*
270		 * Once this instruction has been boosted
271		 * successfully, set the boostable flag
272		 */
273		if (unlikely(p->ainsn.boostable == 0))
274			p->ainsn.boostable = 1;
275	} else if (ret < 0) {
276		/*
277		 * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
278		 * So, we should never get here... but, its still
279		 * good to catch them, just in case...
280		 */
281		printk("Can't step on instruction %08lx\n", ppc_inst_as_ulong(insn));
282		BUG();
283	} else {
284		/*
285		 * If we haven't previously emulated this instruction, then it
286		 * can't be boosted. Note it down so we don't try to do so again.
287		 *
288		 * If, however, we had emulated this instruction in the past,
289		 * then this is just an error with the current run (for
290		 * instance, exceptions due to a load/store). We return 0 so
291		 * that this is now single-stepped, but continue to try
292		 * emulating it in subsequent probe hits.
293		 */
294		if (unlikely(p->ainsn.boostable != 1))
295			p->ainsn.boostable = -1;
296	}
297
298	return ret;
299}
300NOKPROBE_SYMBOL(try_to_emulate);
301
302int kprobe_handler(struct pt_regs *regs)
303{
304	struct kprobe *p;
305	int ret = 0;
306	unsigned int *addr = (unsigned int *)regs->nip;
307	struct kprobe_ctlblk *kcb;
308
309	if (user_mode(regs))
310		return 0;
311
312	if (!IS_ENABLED(CONFIG_BOOKE) &&
313	    (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR)))
314		return 0;
315
316	/*
317	 * We don't want to be preempted for the entire
318	 * duration of kprobe processing
319	 */
320	preempt_disable();
321	kcb = get_kprobe_ctlblk();
322
323	p = get_kprobe(addr);
324	if (!p) {
325		unsigned int instr;
326
327		if (get_kernel_nofault(instr, addr))
328			goto no_kprobe;
329
330		if (instr != BREAKPOINT_INSTRUCTION) {
331			/*
332			 * PowerPC has multiple variants of the "trap"
333			 * instruction. If the current instruction is a
334			 * trap variant, it could belong to someone else
335			 */
336			if (is_trap(instr))
337				goto no_kprobe;
338			/*
339			 * The breakpoint instruction was removed right
340			 * after we hit it.  Another cpu has removed
341			 * either a probepoint or a debugger breakpoint
342			 * at this address.  In either case, no further
343			 * handling of this interrupt is appropriate.
344			 */
345			ret = 1;
346		}
347		/* Not one of ours: let kernel handle it */
348		goto no_kprobe;
349	}
350
351	/* Check we're not actually recursing */
352	if (kprobe_running()) {
353		kprobe_opcode_t insn = *p->ainsn.insn;
354		if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) {
355			/* Turn off 'trace' bits */
356			regs_set_return_msr(regs,
357				(regs->msr & ~MSR_SINGLESTEP) |
358				kcb->kprobe_saved_msr);
359			goto no_kprobe;
360		}
361
362		/*
363		 * We have reentered the kprobe_handler(), since another probe
364		 * was hit while within the handler. We here save the original
365		 * kprobes variables and just single step on the instruction of
366		 * the new probe without calling any user handlers.
367		 */
368		save_previous_kprobe(kcb);
369		set_current_kprobe(p, regs, kcb);
370		kprobes_inc_nmissed_count(p);
371		kcb->kprobe_status = KPROBE_REENTER;
372		if (p->ainsn.boostable >= 0) {
373			ret = try_to_emulate(p, regs);
374
375			if (ret > 0) {
376				restore_previous_kprobe(kcb);
377				preempt_enable();
378				return 1;
379			}
380		}
381		prepare_singlestep(p, regs);
382		return 1;
383	}
384
385	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
386	set_current_kprobe(p, regs, kcb);
387	if (p->pre_handler && p->pre_handler(p, regs)) {
388		/* handler changed execution path, so skip ss setup */
389		reset_current_kprobe();
390		preempt_enable();
391		return 1;
392	}
393
394	if (p->ainsn.boostable >= 0) {
395		ret = try_to_emulate(p, regs);
396
397		if (ret > 0) {
398			if (p->post_handler)
399				p->post_handler(p, regs, 0);
400
401			kcb->kprobe_status = KPROBE_HIT_SSDONE;
402			reset_current_kprobe();
403			preempt_enable();
404			return 1;
405		}
406	}
407	prepare_singlestep(p, regs);
408	kcb->kprobe_status = KPROBE_HIT_SS;
409	return 1;
410
411no_kprobe:
412	preempt_enable();
413	return ret;
414}
415NOKPROBE_SYMBOL(kprobe_handler);
416
417/*
418 * Function return probe trampoline:
419 * 	- init_kprobes() establishes a probepoint here
420 * 	- When the probed function returns, this probe
421 * 		causes the handlers to fire
422 */
423asm(".global __kretprobe_trampoline\n"
424	".type __kretprobe_trampoline, @function\n"
425	"__kretprobe_trampoline:\n"
426	"nop\n"
427	"blr\n"
428	".size __kretprobe_trampoline, .-__kretprobe_trampoline\n");
429
430/*
431 * Called when the probe at kretprobe trampoline is hit
432 */
433static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
434{
435	unsigned long orig_ret_address;
436
437	orig_ret_address = __kretprobe_trampoline_handler(regs, NULL);
438	/*
439	 * We get here through one of two paths:
440	 * 1. by taking a trap -> kprobe_handler() -> here
441	 * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here
442	 *
443	 * When going back through (1), we need regs->nip to be setup properly
444	 * as it is used to determine the return address from the trap.
445	 * For (2), since nip is not honoured with optprobes, we instead setup
446	 * the link register properly so that the subsequent 'blr' in
447	 * __kretprobe_trampoline jumps back to the right instruction.
448	 *
449	 * For nip, we should set the address to the previous instruction since
450	 * we end up emulating it in kprobe_handler(), which increments the nip
451	 * again.
452	 */
453	regs_set_return_ip(regs, orig_ret_address - 4);
454	regs->link = orig_ret_address;
455
456	return 0;
457}
458NOKPROBE_SYMBOL(trampoline_probe_handler);
459
460/*
461 * Called after single-stepping.  p->addr is the address of the
462 * instruction whose first byte has been replaced by the "breakpoint"
463 * instruction.  To avoid the SMP problems that can occur when we
464 * temporarily put back the original opcode to single-step, we
465 * single-stepped a copy of the instruction.  The address of this
466 * copy is p->ainsn.insn.
467 */
468int kprobe_post_handler(struct pt_regs *regs)
469{
470	int len;
471	struct kprobe *cur = kprobe_running();
472	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
473
474	if (!cur || user_mode(regs))
475		return 0;
476
477	len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn));
478	/* make sure we got here for instruction we have a kprobe on */
479	if (((unsigned long)cur->ainsn.insn + len) != regs->nip)
480		return 0;
481
482	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
483		kcb->kprobe_status = KPROBE_HIT_SSDONE;
484		cur->post_handler(cur, regs, 0);
485	}
486
487	/* Adjust nip to after the single-stepped instruction */
488	regs_set_return_ip(regs, (unsigned long)cur->addr + len);
489	regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr);
490
491	/*Restore back the original saved kprobes variables and continue. */
492	if (kcb->kprobe_status == KPROBE_REENTER) {
493		restore_previous_kprobe(kcb);
494		goto out;
495	}
496	reset_current_kprobe();
497out:
498	preempt_enable();
499
500	/*
501	 * if somebody else is singlestepping across a probe point, msr
502	 * will have DE/SE set, in which case, continue the remaining processing
503	 * of do_debug, as if this is not a probe hit.
504	 */
505	if (regs->msr & MSR_SINGLESTEP)
506		return 0;
507
508	return 1;
509}
510NOKPROBE_SYMBOL(kprobe_post_handler);
511
512int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
513{
514	struct kprobe *cur = kprobe_running();
515	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
516	const struct exception_table_entry *entry;
517
518	switch(kcb->kprobe_status) {
519	case KPROBE_HIT_SS:
520	case KPROBE_REENTER:
521		/*
522		 * We are here because the instruction being single
523		 * stepped caused a page fault. We reset the current
524		 * kprobe and the nip points back to the probe address
525		 * and allow the page fault handler to continue as a
526		 * normal page fault.
527		 */
528		regs_set_return_ip(regs, (unsigned long)cur->addr);
529		/* Turn off 'trace' bits */
530		regs_set_return_msr(regs,
531			(regs->msr & ~MSR_SINGLESTEP) |
532			kcb->kprobe_saved_msr);
533		if (kcb->kprobe_status == KPROBE_REENTER)
534			restore_previous_kprobe(kcb);
535		else
536			reset_current_kprobe();
537		preempt_enable();
538		break;
539	case KPROBE_HIT_ACTIVE:
540	case KPROBE_HIT_SSDONE:
541		/*
542		 * In case the user-specified fault handler returned
543		 * zero, try to fix up.
544		 */
545		if ((entry = search_exception_tables(regs->nip)) != NULL) {
546			regs_set_return_ip(regs, extable_fixup(entry));
547			return 1;
548		}
549
550		/*
551		 * fixup_exception() could not handle it,
552		 * Let do_page_fault() fix it.
553		 */
554		break;
555	default:
556		break;
557	}
558	return 0;
559}
560NOKPROBE_SYMBOL(kprobe_fault_handler);
561
562static struct kprobe trampoline_p = {
563	.addr = (kprobe_opcode_t *) &__kretprobe_trampoline,
564	.pre_handler = trampoline_probe_handler
565};
566
567int __init arch_init_kprobes(void)
568{
569	return register_kprobe(&trampoline_p);
570}
571
572int arch_trampoline_kprobe(struct kprobe *p)
573{
574	if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline)
575		return 1;
576
577	return 0;
578}
579NOKPROBE_SYMBOL(arch_trampoline_kprobe);
580