1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _LINUX_CONTEXT_TRACKING_STATE_H 3#define _LINUX_CONTEXT_TRACKING_STATE_H 4 5#include <linux/percpu.h> 6#include <linux/static_key.h> 7#include <linux/context_tracking_irq.h> 8 9/* Offset to allow distinguishing irq vs. task-based idle entry/exit. */ 10#define DYNTICK_IRQ_NONIDLE ((LONG_MAX / 2) + 1) 11 12enum ctx_state { 13 CONTEXT_DISABLED = -1, /* returned by ct_state() if unknown */ 14 CONTEXT_KERNEL = 0, 15 CONTEXT_IDLE = 1, 16 CONTEXT_USER = 2, 17 CONTEXT_GUEST = 3, 18 CONTEXT_MAX = 4, 19}; 20 21/* Even value for idle, else odd. */ 22#define RCU_DYNTICKS_IDX CONTEXT_MAX 23 24#define CT_STATE_MASK (CONTEXT_MAX - 1) 25#define CT_DYNTICKS_MASK (~CT_STATE_MASK) 26 27struct context_tracking { 28#ifdef CONFIG_CONTEXT_TRACKING_USER 29 /* 30 * When active is false, probes are unset in order 31 * to minimize overhead: TIF flags are cleared 32 * and calls to user_enter/exit are ignored. This 33 * may be further optimized using static keys. 34 */ 35 bool active; 36 int recursion; 37#endif 38#ifdef CONFIG_CONTEXT_TRACKING 39 atomic_t state; 40#endif 41#ifdef CONFIG_CONTEXT_TRACKING_IDLE 42 long dynticks_nesting; /* Track process nesting level. */ 43 long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ 44#endif 45}; 46 47#ifdef CONFIG_CONTEXT_TRACKING 48DECLARE_PER_CPU(struct context_tracking, context_tracking); 49#endif 50 51#ifdef CONFIG_CONTEXT_TRACKING_USER 52static __always_inline int __ct_state(void) 53{ 54 return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; 55} 56#endif 57 58#ifdef CONFIG_CONTEXT_TRACKING_IDLE 59static __always_inline int ct_dynticks(void) 60{ 61 return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_DYNTICKS_MASK; 62} 63 64static __always_inline int ct_dynticks_cpu(int cpu) 65{ 66 struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); 67 68 return atomic_read(&ct->state) & CT_DYNTICKS_MASK; 69} 70 71static __always_inline int ct_dynticks_cpu_acquire(int cpu) 72{ 73 struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); 74 75 return atomic_read_acquire(&ct->state) & CT_DYNTICKS_MASK; 76} 77 78static __always_inline long ct_dynticks_nesting(void) 79{ 80 return __this_cpu_read(context_tracking.dynticks_nesting); 81} 82 83static __always_inline long ct_dynticks_nesting_cpu(int cpu) 84{ 85 struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); 86 87 return ct->dynticks_nesting; 88} 89 90static __always_inline long ct_dynticks_nmi_nesting(void) 91{ 92 return __this_cpu_read(context_tracking.dynticks_nmi_nesting); 93} 94 95static __always_inline long ct_dynticks_nmi_nesting_cpu(int cpu) 96{ 97 struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu); 98 99 return ct->dynticks_nmi_nesting; 100} 101#endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */ 102 103#ifdef CONFIG_CONTEXT_TRACKING_USER 104extern struct static_key_false context_tracking_key; 105 106static __always_inline bool context_tracking_enabled(void) 107{ 108 return static_branch_unlikely(&context_tracking_key); 109} 110 111static __always_inline bool context_tracking_enabled_cpu(int cpu) 112{ 113 return context_tracking_enabled() && per_cpu(context_tracking.active, cpu); 114} 115 116static inline bool context_tracking_enabled_this_cpu(void) 117{ 118 return context_tracking_enabled() && __this_cpu_read(context_tracking.active); 119} 120 121/** 122 * ct_state() - return the current context tracking state if known 123 * 124 * Returns the current cpu's context tracking state if context tracking 125 * is enabled. If context tracking is disabled, returns 126 * CONTEXT_DISABLED. This should be used primarily for debugging. 127 */ 128static __always_inline int ct_state(void) 129{ 130 int ret; 131 132 if (!context_tracking_enabled()) 133 return CONTEXT_DISABLED; 134 135 preempt_disable(); 136 ret = __ct_state(); 137 preempt_enable(); 138 139 return ret; 140} 141 142#else 143static __always_inline bool context_tracking_enabled(void) { return false; } 144static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; } 145static __always_inline bool context_tracking_enabled_this_cpu(void) { return false; } 146#endif /* CONFIG_CONTEXT_TRACKING_USER */ 147 148#endif 149