1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_TRACE_RECURSION_H
3#define _LINUX_TRACE_RECURSION_H
4
5#include <linux/interrupt.h>
6#include <linux/sched.h>
7
8#ifdef CONFIG_TRACING
9
10/* Only current can touch trace_recursion */
11
12/*
13 * For function tracing recursion:
14 *  The order of these bits are important.
15 *
16 *  When function tracing occurs, the following steps are made:
17 *   If arch does not support a ftrace feature:
18 *    call internal function (uses INTERNAL bits) which calls...
19 *   The function callback, which can use the FTRACE bits to
20 *    check for recursion.
21 */
22enum {
23	/* Function recursion bits */
24	TRACE_FTRACE_BIT,
25	TRACE_FTRACE_NMI_BIT,
26	TRACE_FTRACE_IRQ_BIT,
27	TRACE_FTRACE_SIRQ_BIT,
28	TRACE_FTRACE_TRANSITION_BIT,
29
30	/* Internal use recursion bits */
31	TRACE_INTERNAL_BIT,
32	TRACE_INTERNAL_NMI_BIT,
33	TRACE_INTERNAL_IRQ_BIT,
34	TRACE_INTERNAL_SIRQ_BIT,
35	TRACE_INTERNAL_TRANSITION_BIT,
36
37	TRACE_BRANCH_BIT,
38/*
39 * Abuse of the trace_recursion.
40 * As we need a way to maintain state if we are tracing the function
41 * graph in irq because we want to trace a particular function that
42 * was called in irq context but we have irq tracing off. Since this
43 * can only be modified by current, we can reuse trace_recursion.
44 */
45	TRACE_IRQ_BIT,
46
47	/* Set if the function is in the set_graph_function file */
48	TRACE_GRAPH_BIT,
49
50	/*
51	 * In the very unlikely case that an interrupt came in
52	 * at a start of graph tracing, and we want to trace
53	 * the function in that interrupt, the depth can be greater
54	 * than zero, because of the preempted start of a previous
55	 * trace. In an even more unlikely case, depth could be 2
56	 * if a softirq interrupted the start of graph tracing,
57	 * followed by an interrupt preempting a start of graph
58	 * tracing in the softirq, and depth can even be 3
59	 * if an NMI came in at the start of an interrupt function
60	 * that preempted a softirq start of a function that
61	 * preempted normal context!!!! Luckily, it can't be
62	 * greater than 3, so the next two bits are a mask
63	 * of what the depth is when we set TRACE_GRAPH_BIT
64	 */
65
66	TRACE_GRAPH_DEPTH_START_BIT,
67	TRACE_GRAPH_DEPTH_END_BIT,
68
69	/*
70	 * To implement set_graph_notrace, if this bit is set, we ignore
71	 * function graph tracing of called functions, until the return
72	 * function is called to clear it.
73	 */
74	TRACE_GRAPH_NOTRACE_BIT,
75
76	/* Used to prevent recursion recording from recursing. */
77	TRACE_RECORD_RECURSION_BIT,
78};
79
80#define trace_recursion_set(bit)	do { (current)->trace_recursion |= (1<<(bit)); } while (0)
81#define trace_recursion_clear(bit)	do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
82#define trace_recursion_test(bit)	((current)->trace_recursion & (1<<(bit)))
83
84#define trace_recursion_depth() \
85	(((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
86#define trace_recursion_set_depth(depth) \
87	do {								\
88		current->trace_recursion &=				\
89			~(3 << TRACE_GRAPH_DEPTH_START_BIT);		\
90		current->trace_recursion |=				\
91			((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT;	\
92	} while (0)
93
94#define TRACE_CONTEXT_BITS	4
95
96#define TRACE_FTRACE_START	TRACE_FTRACE_BIT
97
98#define TRACE_LIST_START	TRACE_INTERNAL_BIT
99
100#define TRACE_CONTEXT_MASK	((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
101
102/*
103 * Used for setting context
104 *  NMI     = 0
105 *  IRQ     = 1
106 *  SOFTIRQ = 2
107 *  NORMAL  = 3
108 */
109enum {
110	TRACE_CTX_NMI,
111	TRACE_CTX_IRQ,
112	TRACE_CTX_SOFTIRQ,
113	TRACE_CTX_NORMAL,
114	TRACE_CTX_TRANSITION,
115};
116
117static __always_inline int trace_get_context_bit(void)
118{
119	unsigned char bit = interrupt_context_level();
120
121	return TRACE_CTX_NORMAL - bit;
122}
123
124#ifdef CONFIG_FTRACE_RECORD_RECURSION
125extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
126# define do_ftrace_record_recursion(ip, pip)				\
127	do {								\
128		if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
129			trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
130			ftrace_record_recursion(ip, pip);		\
131			trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
132		}							\
133	} while (0)
134#else
135# define do_ftrace_record_recursion(ip, pip)	do { } while (0)
136#endif
137
138#ifdef CONFIG_ARCH_WANTS_NO_INSTR
139# define trace_warn_on_no_rcu(ip)					\
140	({								\
141		bool __ret = !rcu_is_watching();			\
142		if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
143			trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
144			WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \
145			trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
146		}							\
147		__ret;							\
148	})
149#else
150# define trace_warn_on_no_rcu(ip)	false
151#endif
152
153/*
154 * Preemption is promised to be disabled when return bit >= 0.
155 */
156static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
157							int start)
158{
159	unsigned int val = READ_ONCE(current->trace_recursion);
160	int bit;
161
162	if (trace_warn_on_no_rcu(ip))
163		return -1;
164
165	bit = trace_get_context_bit() + start;
166	if (unlikely(val & (1 << bit))) {
167		/*
168		 * If an interrupt occurs during a trace, and another trace
169		 * happens in that interrupt but before the preempt_count is
170		 * updated to reflect the new interrupt context, then this
171		 * will think a recursion occurred, and the event will be dropped.
172		 * Let a single instance happen via the TRANSITION_BIT to
173		 * not drop those events.
174		 */
175		bit = TRACE_CTX_TRANSITION + start;
176		if (val & (1 << bit)) {
177			do_ftrace_record_recursion(ip, pip);
178			return -1;
179		}
180	}
181
182	val |= 1 << bit;
183	current->trace_recursion = val;
184	barrier();
185
186	preempt_disable_notrace();
187
188	return bit;
189}
190
191/*
192 * Preemption will be enabled (if it was previously enabled).
193 */
194static __always_inline void trace_clear_recursion(int bit)
195{
196	preempt_enable_notrace();
197	barrier();
198	trace_recursion_clear(bit);
199}
200
201/**
202 * ftrace_test_recursion_trylock - tests for recursion in same context
203 *
204 * Use this for ftrace callbacks. This will detect if the function
205 * tracing recursed in the same context (normal vs interrupt),
206 *
207 * Returns: -1 if a recursion happened.
208 *           >= 0 if no recursion.
209 */
210static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
211							 unsigned long parent_ip)
212{
213	return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
214}
215
216/**
217 * ftrace_test_recursion_unlock - called when function callback is complete
218 * @bit: The return of a successful ftrace_test_recursion_trylock()
219 *
220 * This is used at the end of a ftrace callback.
221 */
222static __always_inline void ftrace_test_recursion_unlock(int bit)
223{
224	trace_clear_recursion(bit);
225}
226
227#endif /* CONFIG_TRACING */
228#endif /* _LINUX_TRACE_RECURSION_H */
229