1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _LINUX_CONTEXT_TRACKING_H 3#define _LINUX_CONTEXT_TRACKING_H 4 5#include <linux/sched.h> 6#include <linux/vtime.h> 7#include <linux/context_tracking_state.h> 8#include <linux/instrumentation.h> 9 10#include <asm/ptrace.h> 11 12 13#ifdef CONFIG_CONTEXT_TRACKING_USER 14extern void ct_cpu_track_user(int cpu); 15 16/* Called with interrupts disabled. */ 17extern void __ct_user_enter(enum ctx_state state); 18extern void __ct_user_exit(enum ctx_state state); 19 20extern void ct_user_enter(enum ctx_state state); 21extern void ct_user_exit(enum ctx_state state); 22 23extern void user_enter_callable(void); 24extern void user_exit_callable(void); 25 26static inline void user_enter(void) 27{ 28 if (context_tracking_enabled()) 29 ct_user_enter(CONTEXT_USER); 30 31} 32static inline void user_exit(void) 33{ 34 if (context_tracking_enabled()) 35 ct_user_exit(CONTEXT_USER); 36} 37 38/* Called with interrupts disabled. */ 39static __always_inline void user_enter_irqoff(void) 40{ 41 if (context_tracking_enabled()) 42 __ct_user_enter(CONTEXT_USER); 43 44} 45static __always_inline void user_exit_irqoff(void) 46{ 47 if (context_tracking_enabled()) 48 __ct_user_exit(CONTEXT_USER); 49} 50 51static inline enum ctx_state exception_enter(void) 52{ 53 enum ctx_state prev_ctx; 54 55 if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) || 56 !context_tracking_enabled()) 57 return 0; 58 59 prev_ctx = __ct_state(); 60 if (prev_ctx != CONTEXT_KERNEL) 61 ct_user_exit(prev_ctx); 62 63 return prev_ctx; 64} 65 66static inline void exception_exit(enum ctx_state prev_ctx) 67{ 68 if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) && 69 context_tracking_enabled()) { 70 if (prev_ctx != CONTEXT_KERNEL) 71 ct_user_enter(prev_ctx); 72 } 73} 74 75static __always_inline bool context_tracking_guest_enter(void) 76{ 77 if (context_tracking_enabled()) 78 __ct_user_enter(CONTEXT_GUEST); 79 80 return context_tracking_enabled_this_cpu(); 81} 82 83static __always_inline void context_tracking_guest_exit(void) 84{ 85 if (context_tracking_enabled()) 86 __ct_user_exit(CONTEXT_GUEST); 87} 88 89#define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) 90 91#else 92static inline void user_enter(void) { } 93static inline void user_exit(void) { } 94static inline void user_enter_irqoff(void) { } 95static inline void user_exit_irqoff(void) { } 96static inline int exception_enter(void) { return 0; } 97static inline void exception_exit(enum ctx_state prev_ctx) { } 98static inline int ct_state(void) { return -1; } 99static inline int __ct_state(void) { return -1; } 100static __always_inline bool context_tracking_guest_enter(void) { return false; } 101static __always_inline void context_tracking_guest_exit(void) { } 102#define CT_WARN_ON(cond) do { } while (0) 103#endif /* !CONFIG_CONTEXT_TRACKING_USER */ 104 105#ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE 106extern void context_tracking_init(void); 107#else 108static inline void context_tracking_init(void) { } 109#endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */ 110 111#ifdef CONFIG_CONTEXT_TRACKING_IDLE 112extern void ct_idle_enter(void); 113extern void ct_idle_exit(void); 114 115/* 116 * Is the current CPU in an extended quiescent state? 117 * 118 * No ordering, as we are sampling CPU-local information. 119 */ 120static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) 121{ 122 return !(raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX); 123} 124 125/* 126 * Increment the current CPU's context_tracking structure's ->state field 127 * with ordering. Return the new value. 128 */ 129static __always_inline unsigned long ct_state_inc(int incby) 130{ 131 return raw_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state)); 132} 133 134static __always_inline bool warn_rcu_enter(void) 135{ 136 bool ret = false; 137 138 /* 139 * Horrible hack to shut up recursive RCU isn't watching fail since 140 * lots of the actual reporting also relies on RCU. 141 */ 142 preempt_disable_notrace(); 143 if (rcu_dynticks_curr_cpu_in_eqs()) { 144 ret = true; 145 ct_state_inc(RCU_DYNTICKS_IDX); 146 } 147 148 return ret; 149} 150 151static __always_inline void warn_rcu_exit(bool rcu) 152{ 153 if (rcu) 154 ct_state_inc(RCU_DYNTICKS_IDX); 155 preempt_enable_notrace(); 156} 157 158#else 159static inline void ct_idle_enter(void) { } 160static inline void ct_idle_exit(void) { } 161 162static __always_inline bool warn_rcu_enter(void) { return false; } 163static __always_inline void warn_rcu_exit(bool rcu) { } 164#endif /* !CONFIG_CONTEXT_TRACKING_IDLE */ 165 166#endif 167