1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _LINUX_TRACE_RECURSION_H 3#define _LINUX_TRACE_RECURSION_H 4 5#include <linux/interrupt.h> 6#include <linux/sched.h> 7 8#ifdef CONFIG_TRACING 9 10/* Only current can touch trace_recursion */ 11 12/* 13 * For function tracing recursion: 14 * The order of these bits are important. 15 * 16 * When function tracing occurs, the following steps are made: 17 * If arch does not support a ftrace feature: 18 * call internal function (uses INTERNAL bits) which calls... 19 * The function callback, which can use the FTRACE bits to 20 * check for recursion. 21 */ 22enum { 23 /* Function recursion bits */ 24 TRACE_FTRACE_BIT, 25 TRACE_FTRACE_NMI_BIT, 26 TRACE_FTRACE_IRQ_BIT, 27 TRACE_FTRACE_SIRQ_BIT, 28 TRACE_FTRACE_TRANSITION_BIT, 29 30 /* Internal use recursion bits */ 31 TRACE_INTERNAL_BIT, 32 TRACE_INTERNAL_NMI_BIT, 33 TRACE_INTERNAL_IRQ_BIT, 34 TRACE_INTERNAL_SIRQ_BIT, 35 TRACE_INTERNAL_TRANSITION_BIT, 36 37 TRACE_BRANCH_BIT, 38/* 39 * Abuse of the trace_recursion. 40 * As we need a way to maintain state if we are tracing the function 41 * graph in irq because we want to trace a particular function that 42 * was called in irq context but we have irq tracing off. Since this 43 * can only be modified by current, we can reuse trace_recursion. 44 */ 45 TRACE_IRQ_BIT, 46 47 /* Set if the function is in the set_graph_function file */ 48 TRACE_GRAPH_BIT, 49 50 /* 51 * In the very unlikely case that an interrupt came in 52 * at a start of graph tracing, and we want to trace 53 * the function in that interrupt, the depth can be greater 54 * than zero, because of the preempted start of a previous 55 * trace. In an even more unlikely case, depth could be 2 56 * if a softirq interrupted the start of graph tracing, 57 * followed by an interrupt preempting a start of graph 58 * tracing in the softirq, and depth can even be 3 59 * if an NMI came in at the start of an interrupt function 60 * that preempted a softirq start of a function that 61 * preempted normal context!!!! Luckily, it can't be 62 * greater than 3, so the next two bits are a mask 63 * of what the depth is when we set TRACE_GRAPH_BIT 64 */ 65 66 TRACE_GRAPH_DEPTH_START_BIT, 67 TRACE_GRAPH_DEPTH_END_BIT, 68 69 /* 70 * To implement set_graph_notrace, if this bit is set, we ignore 71 * function graph tracing of called functions, until the return 72 * function is called to clear it. 73 */ 74 TRACE_GRAPH_NOTRACE_BIT, 75 76 /* Used to prevent recursion recording from recursing. */ 77 TRACE_RECORD_RECURSION_BIT, 78}; 79 80#define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0) 81#define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0) 82#define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit))) 83 84#define trace_recursion_depth() \ 85 (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3) 86#define trace_recursion_set_depth(depth) \ 87 do { \ 88 current->trace_recursion &= \ 89 ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \ 90 current->trace_recursion |= \ 91 ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \ 92 } while (0) 93 94#define TRACE_CONTEXT_BITS 4 95 96#define TRACE_FTRACE_START TRACE_FTRACE_BIT 97 98#define TRACE_LIST_START TRACE_INTERNAL_BIT 99 100#define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1) 101 102/* 103 * Used for setting context 104 * NMI = 0 105 * IRQ = 1 106 * SOFTIRQ = 2 107 * NORMAL = 3 108 */ 109enum { 110 TRACE_CTX_NMI, 111 TRACE_CTX_IRQ, 112 TRACE_CTX_SOFTIRQ, 113 TRACE_CTX_NORMAL, 114 TRACE_CTX_TRANSITION, 115}; 116 117static __always_inline int trace_get_context_bit(void) 118{ 119 unsigned char bit = interrupt_context_level(); 120 121 return TRACE_CTX_NORMAL - bit; 122} 123 124#ifdef CONFIG_FTRACE_RECORD_RECURSION 125extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip); 126# define do_ftrace_record_recursion(ip, pip) \ 127 do { \ 128 if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ 129 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ 130 ftrace_record_recursion(ip, pip); \ 131 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ 132 } \ 133 } while (0) 134#else 135# define do_ftrace_record_recursion(ip, pip) do { } while (0) 136#endif 137 138#ifdef CONFIG_ARCH_WANTS_NO_INSTR 139# define trace_warn_on_no_rcu(ip) \ 140 ({ \ 141 bool __ret = !rcu_is_watching(); \ 142 if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ 143 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ 144 WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \ 145 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ 146 } \ 147 __ret; \ 148 }) 149#else 150# define trace_warn_on_no_rcu(ip) false 151#endif 152 153/* 154 * Preemption is promised to be disabled when return bit >= 0. 155 */ 156static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, 157 int start) 158{ 159 unsigned int val = READ_ONCE(current->trace_recursion); 160 int bit; 161 162 if (trace_warn_on_no_rcu(ip)) 163 return -1; 164 165 bit = trace_get_context_bit() + start; 166 if (unlikely(val & (1 << bit))) { 167 /* 168 * If an interrupt occurs during a trace, and another trace 169 * happens in that interrupt but before the preempt_count is 170 * updated to reflect the new interrupt context, then this 171 * will think a recursion occurred, and the event will be dropped. 172 * Let a single instance happen via the TRANSITION_BIT to 173 * not drop those events. 174 */ 175 bit = TRACE_CTX_TRANSITION + start; 176 if (val & (1 << bit)) { 177 do_ftrace_record_recursion(ip, pip); 178 return -1; 179 } 180 } 181 182 val |= 1 << bit; 183 current->trace_recursion = val; 184 barrier(); 185 186 preempt_disable_notrace(); 187 188 return bit; 189} 190 191/* 192 * Preemption will be enabled (if it was previously enabled). 193 */ 194static __always_inline void trace_clear_recursion(int bit) 195{ 196 preempt_enable_notrace(); 197 barrier(); 198 trace_recursion_clear(bit); 199} 200 201/** 202 * ftrace_test_recursion_trylock - tests for recursion in same context 203 * 204 * Use this for ftrace callbacks. This will detect if the function 205 * tracing recursed in the same context (normal vs interrupt), 206 * 207 * Returns: -1 if a recursion happened. 208 * >= 0 if no recursion. 209 */ 210static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, 211 unsigned long parent_ip) 212{ 213 return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START); 214} 215 216/** 217 * ftrace_test_recursion_unlock - called when function callback is complete 218 * @bit: The return of a successful ftrace_test_recursion_trylock() 219 * 220 * This is used at the end of a ftrace callback. 221 */ 222static __always_inline void ftrace_test_recursion_unlock(int bit) 223{ 224 trace_clear_recursion(bit); 225} 226 227#endif /* CONFIG_TRACING */ 228#endif /* _LINUX_TRACE_RECURSION_H */ 229