1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Stage 1 of the trace events.
4 *
5 * Override the macros in the event tracepoint header <trace/events/XXX.h>
6 * to include the following:
7 *
8 * struct trace_event_raw_<call> {
9 *	struct trace_entry		ent;
10 *	<type>				<item>;
11 *	<type2>				<item2>[<len>];
12 *	[...]
13 * };
14 *
15 * The <type> <item> is created by the __field(type, item) macro or
16 * the __array(type2, item2, len) macro.
17 * We simply do "type item;", and that will create the fields
18 * in the structure.
19 */
20
21#include <linux/trace_events.h>
22
23#ifndef TRACE_SYSTEM_VAR
24#define TRACE_SYSTEM_VAR TRACE_SYSTEM
25#endif
26
27#include "stages/init.h"
28
29/*
30 * DECLARE_EVENT_CLASS can be used to add a generic function
31 * handlers for events. That is, if all events have the same
32 * parameters and just have distinct trace points.
33 * Each tracepoint can be defined with DEFINE_EVENT and that
34 * will map the DECLARE_EVENT_CLASS to the tracepoint.
35 *
36 * TRACE_EVENT is a one to one mapping between tracepoint and template.
37 */
38#undef TRACE_EVENT
39#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
40	DECLARE_EVENT_CLASS(name,			       \
41			     PARAMS(proto),		       \
42			     PARAMS(args),		       \
43			     PARAMS(tstruct),		       \
44			     PARAMS(assign),		       \
45			     PARAMS(print));		       \
46	DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args));
47
48#include "stages/stage1_struct_define.h"
49
50#undef DECLARE_EVENT_CLASS
51#define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print)	\
52	struct trace_event_raw_##name {					\
53		struct trace_entry	ent;				\
54		tstruct							\
55		char			__data[];			\
56	};								\
57									\
58	static struct trace_event_class event_class_##name;
59
60#undef DEFINE_EVENT
61#define DEFINE_EVENT(template, name, proto, args)	\
62	static struct trace_event_call	__used		\
63	__attribute__((__aligned__(4))) event_##name
64
65#undef DEFINE_EVENT_FN
66#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg)	\
67	DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
68
69#undef DEFINE_EVENT_PRINT
70#define DEFINE_EVENT_PRINT(template, name, proto, args, print)	\
71	DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
72
73/* Callbacks are meaningless to ftrace. */
74#undef TRACE_EVENT_FN
75#define TRACE_EVENT_FN(name, proto, args, tstruct,			\
76		assign, print, reg, unreg)				\
77	TRACE_EVENT(name, PARAMS(proto), PARAMS(args),			\
78		PARAMS(tstruct), PARAMS(assign), PARAMS(print))		\
79
80#undef TRACE_EVENT_FN_COND
81#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct,	\
82		assign, print, reg, unreg)				\
83	TRACE_EVENT_CONDITION(name, PARAMS(proto), PARAMS(args), PARAMS(cond),		\
84		PARAMS(tstruct), PARAMS(assign), PARAMS(print))		\
85
86#undef TRACE_EVENT_FLAGS
87#define TRACE_EVENT_FLAGS(name, value)					\
88	__TRACE_EVENT_FLAGS(name, value)
89
90#undef TRACE_EVENT_PERF_PERM
91#define TRACE_EVENT_PERF_PERM(name, expr...)				\
92	__TRACE_EVENT_PERF_PERM(name, expr)
93
94#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
95
96/*
97 * Stage 2 of the trace events.
98 *
99 * Include the following:
100 *
101 * struct trace_event_data_offsets_<call> {
102 *	u32				<item1>;
103 *	u32				<item2>;
104 *	[...]
105 * };
106 *
107 * The __dynamic_array() macro will create each u32 <item>, this is
108 * to keep the offset of each array from the beginning of the event.
109 * The size of an array is also encoded, in the higher 16 bits of <item>.
110 */
111
112#include "stages/stage2_data_offsets.h"
113
114#undef DECLARE_EVENT_CLASS
115#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print)	\
116	struct trace_event_data_offsets_##call {			\
117		tstruct;						\
118	};
119
120#undef DEFINE_EVENT
121#define DEFINE_EVENT(template, name, proto, args)
122
123#undef DEFINE_EVENT_PRINT
124#define DEFINE_EVENT_PRINT(template, name, proto, args, print)
125
126#undef TRACE_EVENT_FLAGS
127#define TRACE_EVENT_FLAGS(event, flag)
128
129#undef TRACE_EVENT_PERF_PERM
130#define TRACE_EVENT_PERF_PERM(event, expr...)
131
132#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
133
134/*
135 * Stage 3 of the trace events.
136 *
137 * Override the macros in the event tracepoint header <trace/events/XXX.h>
138 * to include the following:
139 *
140 * enum print_line_t
141 * trace_raw_output_<call>(struct trace_iterator *iter, int flags)
142 * {
143 *	struct trace_seq *s = &iter->seq;
144 *	struct trace_event_raw_<call> *field; <-- defined in stage 1
145 *	struct trace_seq *p = &iter->tmp_seq;
146 *
147 * -------(for event)-------
148 *
149 *	struct trace_entry *entry;
150 *
151 *	entry = iter->ent;
152 *
153 *	if (entry->type != event_<call>->event.type) {
154 *		WARN_ON_ONCE(1);
155 *		return TRACE_TYPE_UNHANDLED;
156 *	}
157 *
158 *	field = (typeof(field))entry;
159 *
160 *	trace_seq_init(p);
161 *	return trace_output_call(iter, <call>, <TP_printk> "\n");
162 *
163 * ------(or, for event class)------
164 *
165 *	int ret;
166 *
167 *	field = (typeof(field))iter->ent;
168 *
169 *	ret = trace_raw_output_prep(iter, trace_event);
170 *	if (ret != TRACE_TYPE_HANDLED)
171 *		return ret;
172 *
173 *	trace_event_printf(iter, <TP_printk> "\n");
174 *
175 *	return trace_handle_return(s);
176 * -------
177 *  }
178 *
179 * This is the method used to print the raw event to the trace
180 * output format. Note, this is not needed if the data is read
181 * in binary.
182 */
183
184#include "stages/stage3_trace_output.h"
185
186#undef DECLARE_EVENT_CLASS
187#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print)	\
188static notrace enum print_line_t					\
189trace_raw_output_##call(struct trace_iterator *iter, int flags,		\
190			struct trace_event *trace_event)		\
191{									\
192	struct trace_seq *s = &iter->seq;				\
193	struct trace_seq __maybe_unused *p = &iter->tmp_seq;		\
194	struct trace_event_raw_##call *field;				\
195	int ret;							\
196									\
197	field = (typeof(field))iter->ent;				\
198									\
199	ret = trace_raw_output_prep(iter, trace_event);			\
200	if (ret != TRACE_TYPE_HANDLED)					\
201		return ret;						\
202									\
203	trace_event_printf(iter, print);				\
204									\
205	return trace_handle_return(s);					\
206}									\
207static struct trace_event_functions trace_event_type_funcs_##call = {	\
208	.trace			= trace_raw_output_##call,		\
209};
210
211#undef DEFINE_EVENT_PRINT
212#define DEFINE_EVENT_PRINT(template, call, proto, args, print)		\
213static notrace enum print_line_t					\
214trace_raw_output_##call(struct trace_iterator *iter, int flags,		\
215			 struct trace_event *event)			\
216{									\
217	struct trace_event_raw_##template *field;			\
218	struct trace_entry *entry;					\
219	struct trace_seq *p = &iter->tmp_seq;				\
220									\
221	entry = iter->ent;						\
222									\
223	if (entry->type != event_##call.event.type) {			\
224		WARN_ON_ONCE(1);					\
225		return TRACE_TYPE_UNHANDLED;				\
226	}								\
227									\
228	field = (typeof(field))entry;					\
229									\
230	trace_seq_init(p);						\
231	return trace_output_call(iter, #call, print);			\
232}									\
233static struct trace_event_functions trace_event_type_funcs_##call = {	\
234	.trace			= trace_raw_output_##call,		\
235};
236
237#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
238
239#include "stages/stage4_event_fields.h"
240
241#undef DECLARE_EVENT_CLASS
242#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print)	\
243static struct trace_event_fields trace_event_fields_##call[] = {	\
244	tstruct								\
245	{} };
246
247#undef DEFINE_EVENT_PRINT
248#define DEFINE_EVENT_PRINT(template, name, proto, args, print)
249
250#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
251
252#include "stages/stage5_get_offsets.h"
253
254#undef DECLARE_EVENT_CLASS
255#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print)	\
256static inline notrace int trace_event_get_offsets_##call(		\
257	struct trace_event_data_offsets_##call *__data_offsets, proto)	\
258{									\
259	int __data_size = 0;						\
260	int __maybe_unused __item_length;				\
261	struct trace_event_raw_##call __maybe_unused *entry;		\
262									\
263	tstruct;							\
264									\
265	return __data_size;						\
266}
267
268#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
269
270/*
271 * Stage 4 of the trace events.
272 *
273 * Override the macros in the event tracepoint header <trace/events/XXX.h>
274 * to include the following:
275 *
276 * For those macros defined with TRACE_EVENT:
277 *
278 * static struct trace_event_call event_<call>;
279 *
280 * static void trace_event_raw_event_<call>(void *__data, proto)
281 * {
282 *	struct trace_event_file *trace_file = __data;
283 *	struct trace_event_call *event_call = trace_file->event_call;
284 *	struct trace_event_data_offsets_<call> __maybe_unused __data_offsets;
285 *	unsigned long eflags = trace_file->flags;
286 *	enum event_trigger_type __tt = ETT_NONE;
287 *	struct ring_buffer_event *event;
288 *	struct trace_event_raw_<call> *entry; <-- defined in stage 1
289 *	struct trace_buffer *buffer;
290 *	unsigned long irq_flags;
291 *	int __data_size;
292 *	int pc;
293 *
294 *	if (!(eflags & EVENT_FILE_FL_TRIGGER_COND)) {
295 *		if (eflags & EVENT_FILE_FL_TRIGGER_MODE)
296 *			event_triggers_call(trace_file, NULL);
297 *		if (eflags & EVENT_FILE_FL_SOFT_DISABLED)
298 *			return;
299 *	}
300 *
301 *	local_save_flags(irq_flags);
302 *	pc = preempt_count();
303 *
304 *	__data_size = trace_event_get_offsets_<call>(&__data_offsets, args);
305 *
306 *	event = trace_event_buffer_lock_reserve(&buffer, trace_file,
307 *				  event_<call>->event.type,
308 *				  sizeof(*entry) + __data_size,
309 *				  irq_flags, pc);
310 *	if (!event)
311 *		return;
312 *	entry	= ring_buffer_event_data(event);
313 *
314 *	{ <assign>; }  <-- Here we assign the entries by the __field and
315 *			   __array macros.
316 *
317 *	if (eflags & EVENT_FILE_FL_TRIGGER_COND)
318 *		__tt = event_triggers_call(trace_file, entry);
319 *
320 *	if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT,
321 *		     &trace_file->flags))
322 *		ring_buffer_discard_commit(buffer, event);
323 *	else if (!filter_check_discard(trace_file, entry, buffer, event))
324 *		trace_buffer_unlock_commit(buffer, event, irq_flags, pc);
325 *
326 *	if (__tt)
327 *		event_triggers_post_call(trace_file, __tt);
328 * }
329 *
330 * static struct trace_event ftrace_event_type_<call> = {
331 *	.trace			= trace_raw_output_<call>, <-- stage 2
332 * };
333 *
334 * static char print_fmt_<call>[] = <TP_printk>;
335 *
336 * static struct trace_event_class __used event_class_<template> = {
337 *	.system			= "<system>",
338 *	.fields_array		= trace_event_fields_<call>,
339 *	.fields			= LIST_HEAD_INIT(event_class_##call.fields),
340 *	.raw_init		= trace_event_raw_init,
341 *	.probe			= trace_event_raw_event_##call,
342 *	.reg			= trace_event_reg,
343 * };
344 *
345 * static struct trace_event_call event_<call> = {
346 *	.class			= event_class_<template>,
347 *	{
348 *		.tp			= &__tracepoint_<call>,
349 *	},
350 *	.event			= &ftrace_event_type_<call>,
351 *	.print_fmt		= print_fmt_<call>,
352 *	.flags			= TRACE_EVENT_FL_TRACEPOINT,
353 * };
354 * // its only safe to use pointers when doing linker tricks to
355 * // create an array.
356 * static struct trace_event_call __used
357 * __section("_ftrace_events") *__event_<call> = &event_<call>;
358 *
359 */
360
361#ifdef CONFIG_PERF_EVENTS
362
363#define _TRACE_PERF_PROTO(call, proto)					\
364	static notrace void						\
365	perf_trace_##call(void *__data, proto);
366
367#define _TRACE_PERF_INIT(call)						\
368	.perf_probe		= perf_trace_##call,
369
370#else
371#define _TRACE_PERF_PROTO(call, proto)
372#define _TRACE_PERF_INIT(call)
373#endif /* CONFIG_PERF_EVENTS */
374
375#include "stages/stage6_event_callback.h"
376
377#undef DECLARE_EVENT_CLASS
378#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print)	\
379									\
380static notrace void							\
381trace_event_raw_event_##call(void *__data, proto)			\
382{									\
383	struct trace_event_file *trace_file = __data;			\
384	struct trace_event_data_offsets_##call __maybe_unused __data_offsets;\
385	struct trace_event_buffer fbuffer;				\
386	struct trace_event_raw_##call *entry;				\
387	int __data_size;						\
388									\
389	if (trace_trigger_soft_disabled(trace_file))			\
390		return;							\
391									\
392	__data_size = trace_event_get_offsets_##call(&__data_offsets, args); \
393									\
394	entry = trace_event_buffer_reserve(&fbuffer, trace_file,	\
395				 sizeof(*entry) + __data_size);		\
396									\
397	if (!entry)							\
398		return;							\
399									\
400	tstruct								\
401									\
402	{ assign; }							\
403									\
404	trace_event_buffer_commit(&fbuffer);				\
405}
406/*
407 * The ftrace_test_probe is compiled out, it is only here as a build time check
408 * to make sure that if the tracepoint handling changes, the ftrace probe will
409 * fail to compile unless it too is updated.
410 */
411
412#undef DEFINE_EVENT
413#define DEFINE_EVENT(template, call, proto, args)			\
414static inline void ftrace_test_probe_##call(void)			\
415{									\
416	check_trace_callback_type_##call(trace_event_raw_event_##template); \
417}
418
419#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
420
421#include "stages/stage7_class_define.h"
422
423#undef DECLARE_EVENT_CLASS
424#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print)	\
425_TRACE_PERF_PROTO(call, PARAMS(proto));					\
426static char print_fmt_##call[] = print;					\
427static struct trace_event_class __used __refdata event_class_##call = { \
428	.system			= TRACE_SYSTEM_STRING,			\
429	.fields_array		= trace_event_fields_##call,		\
430	.fields			= LIST_HEAD_INIT(event_class_##call.fields),\
431	.raw_init		= trace_event_raw_init,			\
432	.probe			= trace_event_raw_event_##call,		\
433	.reg			= trace_event_reg,			\
434	_TRACE_PERF_INIT(call)						\
435};
436
437#undef DEFINE_EVENT
438#define DEFINE_EVENT(template, call, proto, args)			\
439									\
440static struct trace_event_call __used event_##call = {			\
441	.class			= &event_class_##template,		\
442	{								\
443		.tp			= &__tracepoint_##call,		\
444	},								\
445	.event.funcs		= &trace_event_type_funcs_##template,	\
446	.print_fmt		= print_fmt_##template,			\
447	.flags			= TRACE_EVENT_FL_TRACEPOINT,		\
448};									\
449static struct trace_event_call __used					\
450__section("_ftrace_events") *__event_##call = &event_##call
451
452#undef DEFINE_EVENT_PRINT
453#define DEFINE_EVENT_PRINT(template, call, proto, args, print)		\
454									\
455static char print_fmt_##call[] = print;					\
456									\
457static struct trace_event_call __used event_##call = {			\
458	.class			= &event_class_##template,		\
459	{								\
460		.tp			= &__tracepoint_##call,		\
461	},								\
462	.event.funcs		= &trace_event_type_funcs_##call,	\
463	.print_fmt		= print_fmt_##call,			\
464	.flags			= TRACE_EVENT_FL_TRACEPOINT,		\
465};									\
466static struct trace_event_call __used					\
467__section("_ftrace_events") *__event_##call = &event_##call
468
469#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
470