1// SPDX-License-Identifier: GPL-2.0-only
2
3#define pr_fmt(fmt)       KBUILD_MODNAME ": " fmt
4
5#include <linux/ftrace.h>
6#include <linux/ktime.h>
7#include <linux/module.h>
8
9#include <asm/barrier.h>
10
11/*
12 * Arbitrary large value chosen to be sufficiently large to minimize noise but
13 * sufficiently small to complete quickly.
14 */
15static unsigned int nr_function_calls = 100000;
16module_param(nr_function_calls, uint, 0);
17MODULE_PARM_DESC(nr_function_calls, "How many times to call the relevant tracee");
18
19/*
20 * The number of ops associated with a call site affects whether a tracer can
21 * be called directly or whether it's necessary to go via the list func, which
22 * can be significantly more expensive.
23 */
24static unsigned int nr_ops_relevant = 1;
25module_param(nr_ops_relevant, uint, 0);
26MODULE_PARM_DESC(nr_ops_relevant, "How many ftrace_ops to associate with the relevant tracee");
27
28/*
29 * On architectures where all call sites share the same trampoline, having
30 * tracers enabled for distinct functions can force the use of the list func
31 * and incur overhead for all call sites.
32 */
33static unsigned int nr_ops_irrelevant;
34module_param(nr_ops_irrelevant, uint, 0);
35MODULE_PARM_DESC(nr_ops_irrelevant, "How many ftrace_ops to associate with the irrelevant tracee");
36
37/*
38 * On architectures with DYNAMIC_FTRACE_WITH_REGS, saving the full pt_regs can
39 * be more expensive than only saving the minimal necessary regs.
40 */
41static bool save_regs;
42module_param(save_regs, bool, 0);
43MODULE_PARM_DESC(save_regs, "Register ops with FTRACE_OPS_FL_SAVE_REGS (save all registers in the trampoline)");
44
45static bool assist_recursion;
46module_param(assist_recursion, bool, 0);
47MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RECURSION");
48
49static bool assist_rcu;
50module_param(assist_rcu, bool, 0);
51MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RCU");
52
53/*
54 * By default, a trivial tracer is used which immediately returns to mimimize
55 * overhead. Sometimes a consistency check using a more expensive tracer is
56 * desireable.
57 */
58static bool check_count;
59module_param(check_count, bool, 0);
60MODULE_PARM_DESC(check_count, "Check that tracers are called the expected number of times\n");
61
62/*
63 * Usually it's not interesting to leave the ops registered after the test
64 * runs, but sometimes it can be useful to leave them registered so that they
65 * can be inspected through the tracefs 'enabled_functions' file.
66 */
67static bool persist;
68module_param(persist, bool, 0);
69MODULE_PARM_DESC(persist, "Successfully load module and leave ftrace ops registered after test completes\n");
70
71/*
72 * Marked as noinline to ensure that an out-of-line traceable copy is
73 * generated by the compiler.
74 *
75 * The barrier() ensures the compiler won't elide calls by determining there
76 * are no side-effects.
77 */
78static noinline void tracee_relevant(void)
79{
80	barrier();
81}
82
83/*
84 * Marked as noinline to ensure that an out-of-line traceable copy is
85 * generated by the compiler.
86 *
87 * The barrier() ensures the compiler won't elide calls by determining there
88 * are no side-effects.
89 */
90static noinline void tracee_irrelevant(void)
91{
92	barrier();
93}
94
95struct sample_ops {
96	struct ftrace_ops ops;
97	unsigned int count;
98};
99
100static void ops_func_nop(unsigned long ip, unsigned long parent_ip,
101			 struct ftrace_ops *op,
102			 struct ftrace_regs *fregs)
103{
104	/* do nothing */
105}
106
107static void ops_func_count(unsigned long ip, unsigned long parent_ip,
108			   struct ftrace_ops *op,
109			   struct ftrace_regs *fregs)
110{
111	struct sample_ops *self;
112
113	self = container_of(op, struct sample_ops, ops);
114	self->count++;
115}
116
117static struct sample_ops *ops_relevant;
118static struct sample_ops *ops_irrelevant;
119
120static struct sample_ops *ops_alloc_init(void *tracee, ftrace_func_t func,
121					 unsigned long flags, int nr)
122{
123	struct sample_ops *ops;
124
125	ops = kcalloc(nr, sizeof(*ops), GFP_KERNEL);
126	if (WARN_ON_ONCE(!ops))
127		return NULL;
128
129	for (unsigned int i = 0; i < nr; i++) {
130		ops[i].ops.func = func;
131		ops[i].ops.flags = flags;
132		WARN_ON_ONCE(ftrace_set_filter_ip(&ops[i].ops, (unsigned long)tracee, 0, 0));
133		WARN_ON_ONCE(register_ftrace_function(&ops[i].ops));
134	}
135
136	return ops;
137}
138
139static void ops_destroy(struct sample_ops *ops, int nr)
140{
141	if (!ops)
142		return;
143
144	for (unsigned int i = 0; i < nr; i++) {
145		WARN_ON_ONCE(unregister_ftrace_function(&ops[i].ops));
146		ftrace_free_filter(&ops[i].ops);
147	}
148
149	kfree(ops);
150}
151
152static void ops_check(struct sample_ops *ops, int nr,
153		      unsigned int expected_count)
154{
155	if (!ops || !check_count)
156		return;
157
158	for (unsigned int i = 0; i < nr; i++) {
159		if (ops->count == expected_count)
160			continue;
161		pr_warn("Counter called %u times (expected %u)\n",
162			ops->count, expected_count);
163	}
164}
165
166static ftrace_func_t tracer_relevant = ops_func_nop;
167static ftrace_func_t tracer_irrelevant = ops_func_nop;
168
169static int __init ftrace_ops_sample_init(void)
170{
171	unsigned long flags = 0;
172	ktime_t start, end;
173	u64 period;
174
175	if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && save_regs) {
176		pr_info("this kernel does not support saving registers\n");
177		save_regs = false;
178	} else if (save_regs) {
179		flags |= FTRACE_OPS_FL_SAVE_REGS;
180	}
181
182	if (assist_recursion)
183		flags |= FTRACE_OPS_FL_RECURSION;
184
185	if (assist_rcu)
186		flags |= FTRACE_OPS_FL_RCU;
187
188	if (check_count) {
189		tracer_relevant = ops_func_count;
190		tracer_irrelevant = ops_func_count;
191	}
192
193	pr_info("registering:\n"
194		"  relevant ops: %u\n"
195		"    tracee: %ps\n"
196		"    tracer: %ps\n"
197		"  irrelevant ops: %u\n"
198		"    tracee: %ps\n"
199		"    tracer: %ps\n"
200		"  saving registers: %s\n"
201		"  assist recursion: %s\n"
202		"  assist RCU: %s\n",
203		nr_ops_relevant, tracee_relevant, tracer_relevant,
204		nr_ops_irrelevant, tracee_irrelevant, tracer_irrelevant,
205		save_regs ? "YES" : "NO",
206		assist_recursion ? "YES" : "NO",
207		assist_rcu ? "YES" : "NO");
208
209	ops_relevant = ops_alloc_init(tracee_relevant, tracer_relevant,
210				      flags, nr_ops_relevant);
211	ops_irrelevant = ops_alloc_init(tracee_irrelevant, tracer_irrelevant,
212					flags, nr_ops_irrelevant);
213
214	start = ktime_get();
215	for (unsigned int i = 0; i < nr_function_calls; i++)
216		tracee_relevant();
217	end = ktime_get();
218
219	ops_check(ops_relevant, nr_ops_relevant, nr_function_calls);
220	ops_check(ops_irrelevant, nr_ops_irrelevant, 0);
221
222	period = ktime_to_ns(ktime_sub(end, start));
223
224	pr_info("Attempted %u calls to %ps in %lluns (%lluns / call)\n",
225		nr_function_calls, tracee_relevant,
226		period, div_u64(period, nr_function_calls));
227
228	if (persist)
229		return 0;
230
231	ops_destroy(ops_relevant, nr_ops_relevant);
232	ops_destroy(ops_irrelevant, nr_ops_irrelevant);
233
234	/*
235	 * The benchmark completed sucessfully, but there's no reason to keep
236	 * the module around. Return an error do the user doesn't have to
237	 * manually unload the module.
238	 */
239	return -EINVAL;
240}
241module_init(ftrace_ops_sample_init);
242
243static void __exit ftrace_ops_sample_exit(void)
244{
245	ops_destroy(ops_relevant, nr_ops_relevant);
246	ops_destroy(ops_irrelevant, nr_ops_irrelevant);
247}
248module_exit(ftrace_ops_sample_exit);
249
250MODULE_AUTHOR("Mark Rutland");
251MODULE_DESCRIPTION("Example of using custom ftrace_ops");
252MODULE_LICENSE("GPL");
253