1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fprobe - Simple ftrace probe wrapper for function entry.
4 */
5#define pr_fmt(fmt) "fprobe: " fmt
6
7#include <linux/err.h>
8#include <linux/fprobe.h>
9#include <linux/kallsyms.h>
10#include <linux/kprobes.h>
11#include <linux/rethook.h>
12#include <linux/slab.h>
13#include <linux/sort.h>
14
15#include "trace.h"
16
17struct fprobe_rethook_node {
18	struct rethook_node node;
19	unsigned long entry_ip;
20	unsigned long entry_parent_ip;
21	char data[];
22};
23
24static inline void __fprobe_handler(unsigned long ip, unsigned long parent_ip,
25			struct ftrace_ops *ops, struct ftrace_regs *fregs)
26{
27	struct fprobe_rethook_node *fpr;
28	struct rethook_node *rh = NULL;
29	struct fprobe *fp;
30	void *entry_data = NULL;
31	int ret = 0;
32
33	fp = container_of(ops, struct fprobe, ops);
34
35	if (fp->exit_handler) {
36		rh = rethook_try_get(fp->rethook);
37		if (!rh) {
38			fp->nmissed++;
39			return;
40		}
41		fpr = container_of(rh, struct fprobe_rethook_node, node);
42		fpr->entry_ip = ip;
43		fpr->entry_parent_ip = parent_ip;
44		if (fp->entry_data_size)
45			entry_data = fpr->data;
46	}
47
48	if (fp->entry_handler)
49		ret = fp->entry_handler(fp, ip, parent_ip, ftrace_get_regs(fregs), entry_data);
50
51	/* If entry_handler returns !0, nmissed is not counted. */
52	if (rh) {
53		if (ret)
54			rethook_recycle(rh);
55		else
56			rethook_hook(rh, ftrace_get_regs(fregs), true);
57	}
58}
59
60static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
61		struct ftrace_ops *ops, struct ftrace_regs *fregs)
62{
63	struct fprobe *fp;
64	int bit;
65
66	fp = container_of(ops, struct fprobe, ops);
67	if (fprobe_disabled(fp))
68		return;
69
70	/* recursion detection has to go before any traceable function and
71	 * all functions before this point should be marked as notrace
72	 */
73	bit = ftrace_test_recursion_trylock(ip, parent_ip);
74	if (bit < 0) {
75		fp->nmissed++;
76		return;
77	}
78	__fprobe_handler(ip, parent_ip, ops, fregs);
79	ftrace_test_recursion_unlock(bit);
80
81}
82NOKPROBE_SYMBOL(fprobe_handler);
83
84static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
85				  struct ftrace_ops *ops, struct ftrace_regs *fregs)
86{
87	struct fprobe *fp;
88	int bit;
89
90	fp = container_of(ops, struct fprobe, ops);
91	if (fprobe_disabled(fp))
92		return;
93
94	/* recursion detection has to go before any traceable function and
95	 * all functions called before this point should be marked as notrace
96	 */
97	bit = ftrace_test_recursion_trylock(ip, parent_ip);
98	if (bit < 0) {
99		fp->nmissed++;
100		return;
101	}
102
103	/*
104	 * This user handler is shared with other kprobes and is not expected to be
105	 * called recursively. So if any other kprobe handler is running, this will
106	 * exit as kprobe does. See the section 'Share the callbacks with kprobes'
107	 * in Documentation/trace/fprobe.rst for more information.
108	 */
109	if (unlikely(kprobe_running())) {
110		fp->nmissed++;
111		goto recursion_unlock;
112	}
113
114	kprobe_busy_begin();
115	__fprobe_handler(ip, parent_ip, ops, fregs);
116	kprobe_busy_end();
117
118recursion_unlock:
119	ftrace_test_recursion_unlock(bit);
120}
121
122static void fprobe_exit_handler(struct rethook_node *rh, void *data,
123				unsigned long ret_ip, struct pt_regs *regs)
124{
125	struct fprobe *fp = (struct fprobe *)data;
126	struct fprobe_rethook_node *fpr;
127	int bit;
128
129	if (!fp || fprobe_disabled(fp))
130		return;
131
132	fpr = container_of(rh, struct fprobe_rethook_node, node);
133
134	/*
135	 * we need to assure no calls to traceable functions in-between the
136	 * end of fprobe_handler and the beginning of fprobe_exit_handler.
137	 */
138	bit = ftrace_test_recursion_trylock(fpr->entry_ip, fpr->entry_parent_ip);
139	if (bit < 0) {
140		fp->nmissed++;
141		return;
142	}
143
144	fp->exit_handler(fp, fpr->entry_ip, ret_ip, regs,
145			 fp->entry_data_size ? (void *)fpr->data : NULL);
146	ftrace_test_recursion_unlock(bit);
147}
148NOKPROBE_SYMBOL(fprobe_exit_handler);
149
150static int symbols_cmp(const void *a, const void *b)
151{
152	const char **str_a = (const char **) a;
153	const char **str_b = (const char **) b;
154
155	return strcmp(*str_a, *str_b);
156}
157
158/* Convert ftrace location address from symbols */
159static unsigned long *get_ftrace_locations(const char **syms, int num)
160{
161	unsigned long *addrs;
162
163	/* Convert symbols to symbol address */
164	addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
165	if (!addrs)
166		return ERR_PTR(-ENOMEM);
167
168	/* ftrace_lookup_symbols expects sorted symbols */
169	sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
170
171	if (!ftrace_lookup_symbols(syms, num, addrs))
172		return addrs;
173
174	kfree(addrs);
175	return ERR_PTR(-ENOENT);
176}
177
178static void fprobe_init(struct fprobe *fp)
179{
180	fp->nmissed = 0;
181	if (fprobe_shared_with_kprobes(fp))
182		fp->ops.func = fprobe_kprobe_handler;
183	else
184		fp->ops.func = fprobe_handler;
185	fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS;
186}
187
188static int fprobe_init_rethook(struct fprobe *fp, int num)
189{
190	int size;
191
192	if (!fp->exit_handler) {
193		fp->rethook = NULL;
194		return 0;
195	}
196
197	/* Initialize rethook if needed */
198	if (fp->nr_maxactive)
199		num = fp->nr_maxactive;
200	else
201		num *= num_possible_cpus() * 2;
202	if (num <= 0)
203		return -EINVAL;
204
205	size = sizeof(struct fprobe_rethook_node) + fp->entry_data_size;
206
207	/* Initialize rethook */
208	fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler, size, num);
209	if (IS_ERR(fp->rethook))
210		return PTR_ERR(fp->rethook);
211
212	return 0;
213}
214
215static void fprobe_fail_cleanup(struct fprobe *fp)
216{
217	if (!IS_ERR_OR_NULL(fp->rethook)) {
218		/* Don't need to cleanup rethook->handler because this is not used. */
219		rethook_free(fp->rethook);
220		fp->rethook = NULL;
221	}
222	ftrace_free_filter(&fp->ops);
223}
224
225/**
226 * register_fprobe() - Register fprobe to ftrace by pattern.
227 * @fp: A fprobe data structure to be registered.
228 * @filter: A wildcard pattern of probed symbols.
229 * @notfilter: A wildcard pattern of NOT probed symbols.
230 *
231 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
232 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
233 *
234 * Return 0 if @fp is registered successfully, -errno if not.
235 */
236int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
237{
238	struct ftrace_hash *hash;
239	unsigned char *str;
240	int ret, len;
241
242	if (!fp || !filter)
243		return -EINVAL;
244
245	fprobe_init(fp);
246
247	len = strlen(filter);
248	str = kstrdup(filter, GFP_KERNEL);
249	ret = ftrace_set_filter(&fp->ops, str, len, 0);
250	kfree(str);
251	if (ret)
252		return ret;
253
254	if (notfilter) {
255		len = strlen(notfilter);
256		str = kstrdup(notfilter, GFP_KERNEL);
257		ret = ftrace_set_notrace(&fp->ops, str, len, 0);
258		kfree(str);
259		if (ret)
260			goto out;
261	}
262
263	/* TODO:
264	 * correctly calculate the total number of filtered symbols
265	 * from both filter and notfilter.
266	 */
267	hash = rcu_access_pointer(fp->ops.local_hash.filter_hash);
268	if (WARN_ON_ONCE(!hash))
269		goto out;
270
271	ret = fprobe_init_rethook(fp, (int)hash->count);
272	if (!ret)
273		ret = register_ftrace_function(&fp->ops);
274
275out:
276	if (ret)
277		fprobe_fail_cleanup(fp);
278	return ret;
279}
280EXPORT_SYMBOL_GPL(register_fprobe);
281
282/**
283 * register_fprobe_ips() - Register fprobe to ftrace by address.
284 * @fp: A fprobe data structure to be registered.
285 * @addrs: An array of target ftrace location addresses.
286 * @num: The number of entries of @addrs.
287 *
288 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
289 * The @addrs must be the addresses of ftrace location address, which may be
290 * the symbol address + arch-dependent offset.
291 * If you unsure what this mean, please use other registration functions.
292 *
293 * Return 0 if @fp is registered successfully, -errno if not.
294 */
295int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
296{
297	int ret;
298
299	if (!fp || !addrs || num <= 0)
300		return -EINVAL;
301
302	fprobe_init(fp);
303
304	ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0);
305	if (ret)
306		return ret;
307
308	ret = fprobe_init_rethook(fp, num);
309	if (!ret)
310		ret = register_ftrace_function(&fp->ops);
311
312	if (ret)
313		fprobe_fail_cleanup(fp);
314	return ret;
315}
316EXPORT_SYMBOL_GPL(register_fprobe_ips);
317
318/**
319 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
320 * @fp: A fprobe data structure to be registered.
321 * @syms: An array of target symbols.
322 * @num: The number of entries of @syms.
323 *
324 * Register @fp to the symbols given by @syms array. This will be useful if
325 * you are sure the symbols exist in the kernel.
326 *
327 * Return 0 if @fp is registered successfully, -errno if not.
328 */
329int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
330{
331	unsigned long *addrs;
332	int ret;
333
334	if (!fp || !syms || num <= 0)
335		return -EINVAL;
336
337	addrs = get_ftrace_locations(syms, num);
338	if (IS_ERR(addrs))
339		return PTR_ERR(addrs);
340
341	ret = register_fprobe_ips(fp, addrs, num);
342
343	kfree(addrs);
344
345	return ret;
346}
347EXPORT_SYMBOL_GPL(register_fprobe_syms);
348
349bool fprobe_is_registered(struct fprobe *fp)
350{
351	if (!fp || (fp->ops.saved_func != fprobe_handler &&
352		    fp->ops.saved_func != fprobe_kprobe_handler))
353		return false;
354	return true;
355}
356
357/**
358 * unregister_fprobe() - Unregister fprobe from ftrace
359 * @fp: A fprobe data structure to be unregistered.
360 *
361 * Unregister fprobe (and remove ftrace hooks from the function entries).
362 *
363 * Return 0 if @fp is unregistered successfully, -errno if not.
364 */
365int unregister_fprobe(struct fprobe *fp)
366{
367	int ret;
368
369	if (!fprobe_is_registered(fp))
370		return -EINVAL;
371
372	if (!IS_ERR_OR_NULL(fp->rethook))
373		rethook_stop(fp->rethook);
374
375	ret = unregister_ftrace_function(&fp->ops);
376	if (ret < 0)
377		return ret;
378
379	if (!IS_ERR_OR_NULL(fp->rethook))
380		rethook_free(fp->rethook);
381
382	ftrace_free_filter(&fp->ops);
383
384	return ret;
385}
386EXPORT_SYMBOL_GPL(unregister_fprobe);
387