1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2#ifndef _ASM_POWERPC_PARAVIRT_H 3#define _ASM_POWERPC_PARAVIRT_H 4 5#include <linux/jump_label.h> 6#include <asm/smp.h> 7#ifdef CONFIG_PPC64 8#include <asm/paca.h> 9#include <asm/lppaca.h> 10#include <asm/hvcall.h> 11#endif 12 13#ifdef CONFIG_PPC_SPLPAR 14#include <linux/smp.h> 15#include <asm/kvm_guest.h> 16#include <asm/cputhreads.h> 17 18DECLARE_STATIC_KEY_FALSE(shared_processor); 19 20static inline bool is_shared_processor(void) 21{ 22 return static_branch_unlikely(&shared_processor); 23} 24 25#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 26extern struct static_key paravirt_steal_enabled; 27extern struct static_key paravirt_steal_rq_enabled; 28 29u64 pseries_paravirt_steal_clock(int cpu); 30 31static inline u64 paravirt_steal_clock(int cpu) 32{ 33 return pseries_paravirt_steal_clock(cpu); 34} 35#endif 36 37/* If bit 0 is set, the cpu has been ceded, conferred, or preempted */ 38static inline u32 yield_count_of(int cpu) 39{ 40 __be32 yield_count = READ_ONCE(lppaca_of(cpu).yield_count); 41 return be32_to_cpu(yield_count); 42} 43 44/* 45 * Spinlock code confers and prods, so don't trace the hcalls because the 46 * tracing code takes spinlocks which can cause recursion deadlocks. 47 * 48 * These calls are made while the lock is not held: the lock slowpath yields if 49 * it can not acquire the lock, and unlock slow path might prod if a waiter has 50 * yielded). So this may not be a problem for simple spin locks because the 51 * tracing does not technically recurse on the lock, but we avoid it anyway. 52 * 53 * However the queued spin lock contended path is more strictly ordered: the 54 * H_CONFER hcall is made after the task has queued itself on the lock, so then 55 * recursing on that lock will cause the task to then queue up again behind the 56 * first instance (or worse: queued spinlocks use tricks that assume a context 57 * never waits on more than one spinlock, so such recursion may cause random 58 * corruption in the lock code). 59 */ 60static inline void yield_to_preempted(int cpu, u32 yield_count) 61{ 62 plpar_hcall_norets_notrace(H_CONFER, get_hard_smp_processor_id(cpu), yield_count); 63} 64 65static inline void prod_cpu(int cpu) 66{ 67 plpar_hcall_norets_notrace(H_PROD, get_hard_smp_processor_id(cpu)); 68} 69 70static inline void yield_to_any(void) 71{ 72 plpar_hcall_norets_notrace(H_CONFER, -1, 0); 73} 74 75static inline bool is_vcpu_idle(int vcpu) 76{ 77 return lppaca_of(vcpu).idle; 78} 79 80static inline bool vcpu_is_dispatched(int vcpu) 81{ 82 /* 83 * This is the yield_count. An "odd" value (low bit on) means that 84 * the processor is yielded (either because of an OS yield or a 85 * hypervisor preempt). An even value implies that the processor is 86 * currently executing. 87 */ 88 return (!(yield_count_of(vcpu) & 1)); 89} 90#else 91static inline bool is_shared_processor(void) 92{ 93 return false; 94} 95 96static inline u32 yield_count_of(int cpu) 97{ 98 return 0; 99} 100 101extern void ___bad_yield_to_preempted(void); 102static inline void yield_to_preempted(int cpu, u32 yield_count) 103{ 104 ___bad_yield_to_preempted(); /* This would be a bug */ 105} 106 107extern void ___bad_yield_to_any(void); 108static inline void yield_to_any(void) 109{ 110 ___bad_yield_to_any(); /* This would be a bug */ 111} 112 113extern void ___bad_prod_cpu(void); 114static inline void prod_cpu(int cpu) 115{ 116 ___bad_prod_cpu(); /* This would be a bug */ 117} 118 119static inline bool is_vcpu_idle(int vcpu) 120{ 121 return false; 122} 123static inline bool vcpu_is_dispatched(int vcpu) 124{ 125 return true; 126} 127#endif 128 129#define vcpu_is_preempted vcpu_is_preempted 130static inline bool vcpu_is_preempted(int cpu) 131{ 132 /* 133 * The dispatch/yield bit alone is an imperfect indicator of 134 * whether the hypervisor has dispatched @cpu to run on a physical 135 * processor. When it is clear, @cpu is definitely not preempted. 136 * But when it is set, it means only that it *might* be, subject to 137 * other conditions. So we check other properties of the VM and 138 * @cpu first, resorting to the yield count last. 139 */ 140 141 /* 142 * Hypervisor preemption isn't possible in dedicated processor 143 * mode by definition. 144 */ 145 if (!is_shared_processor()) 146 return false; 147 148 /* 149 * If the hypervisor has dispatched the target CPU on a physical 150 * processor, then the target CPU is definitely not preempted. 151 */ 152 if (vcpu_is_dispatched(cpu)) 153 return false; 154 155 /* 156 * if the target CPU is not dispatched and the guest OS 157 * has not marked the CPU idle, then it is hypervisor preempted. 158 */ 159 if (!is_vcpu_idle(cpu)) 160 return true; 161 162#ifdef CONFIG_PPC_SPLPAR 163 if (!is_kvm_guest()) { 164 int first_cpu, i; 165 166 /* 167 * The result of vcpu_is_preempted() is used in a 168 * speculative way, and is always subject to invalidation 169 * by events internal and external to Linux. While we can 170 * be called in preemptable context (in the Linux sense), 171 * we're not accessing per-cpu resources in a way that can 172 * race destructively with Linux scheduler preemption and 173 * migration, and callers can tolerate the potential for 174 * error introduced by sampling the CPU index without 175 * pinning the task to it. So it is permissible to use 176 * raw_smp_processor_id() here to defeat the preempt debug 177 * warnings that can arise from using smp_processor_id() 178 * in arbitrary contexts. 179 */ 180 first_cpu = cpu_first_thread_sibling(raw_smp_processor_id()); 181 182 /* 183 * The PowerVM hypervisor dispatches VMs on a whole core 184 * basis. So we know that a thread sibling of the executing CPU 185 * cannot have been preempted by the hypervisor, even if it 186 * has called H_CONFER, which will set the yield bit. 187 */ 188 if (cpu_first_thread_sibling(cpu) == first_cpu) 189 return false; 190 191 /* 192 * The specific target CPU was marked by guest OS as idle, but 193 * then also check all other cpus in the core for PowerVM 194 * because it does core scheduling and one of the vcpu 195 * of the core getting preempted by hypervisor implies 196 * other vcpus can also be considered preempted. 197 */ 198 first_cpu = cpu_first_thread_sibling(cpu); 199 for (i = first_cpu; i < first_cpu + threads_per_core; i++) { 200 if (i == cpu) 201 continue; 202 if (vcpu_is_dispatched(i)) 203 return false; 204 if (!is_vcpu_idle(i)) 205 return true; 206 } 207 } 208#endif 209 210 /* 211 * None of the threads in target CPU's core are running but none of 212 * them were preempted too. Hence assume the target CPU to be 213 * non-preempted. 214 */ 215 return false; 216} 217 218static inline bool pv_is_native_spin_unlock(void) 219{ 220 return !is_shared_processor(); 221} 222 223#endif /* _ASM_POWERPC_PARAVIRT_H */ 224