1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Author: Huacai Chen <chenhuacai@loongson.cn> 4 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 5 */ 6#ifndef _ASM_FPU_H 7#define _ASM_FPU_H 8 9#include <linux/sched.h> 10#include <linux/sched/task_stack.h> 11#include <linux/ptrace.h> 12#include <linux/thread_info.h> 13#include <linux/bitops.h> 14 15#include <asm/cpu.h> 16#include <asm/cpu-features.h> 17#include <asm/current.h> 18#include <asm/loongarch.h> 19#include <asm/processor.h> 20#include <asm/ptrace.h> 21 22struct sigcontext; 23 24extern void kernel_fpu_begin(void); 25extern void kernel_fpu_end(void); 26 27extern void _init_fpu(unsigned int); 28extern void _save_fp(struct loongarch_fpu *); 29extern void _restore_fp(struct loongarch_fpu *); 30 31extern void _save_lsx(struct loongarch_fpu *fpu); 32extern void _restore_lsx(struct loongarch_fpu *fpu); 33extern void _init_lsx_upper(void); 34extern void _restore_lsx_upper(struct loongarch_fpu *fpu); 35 36extern void _save_lasx(struct loongarch_fpu *fpu); 37extern void _restore_lasx(struct loongarch_fpu *fpu); 38extern void _init_lasx_upper(void); 39extern void _restore_lasx_upper(struct loongarch_fpu *fpu); 40 41static inline void enable_lsx(void); 42static inline void disable_lsx(void); 43static inline void save_lsx(struct task_struct *t); 44static inline void restore_lsx(struct task_struct *t); 45 46static inline void enable_lasx(void); 47static inline void disable_lasx(void); 48static inline void save_lasx(struct task_struct *t); 49static inline void restore_lasx(struct task_struct *t); 50 51/* 52 * Mask the FCSR Cause bits according to the Enable bits, observing 53 * that Unimplemented is always enabled. 54 */ 55static inline unsigned long mask_fcsr_x(unsigned long fcsr) 56{ 57 return fcsr & ((fcsr & FPU_CSR_ALL_E) << 58 (ffs(FPU_CSR_ALL_X) - ffs(FPU_CSR_ALL_E))); 59} 60 61static inline int is_fp_enabled(void) 62{ 63 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_FPEN) ? 64 1 : 0; 65} 66 67static inline int is_lsx_enabled(void) 68{ 69 if (!cpu_has_lsx) 70 return 0; 71 72 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LSXEN) ? 73 1 : 0; 74} 75 76static inline int is_lasx_enabled(void) 77{ 78 if (!cpu_has_lasx) 79 return 0; 80 81 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LASXEN) ? 82 1 : 0; 83} 84 85static inline int is_simd_enabled(void) 86{ 87 return is_lsx_enabled() | is_lasx_enabled(); 88} 89 90#define enable_fpu() set_csr_euen(CSR_EUEN_FPEN) 91 92#define disable_fpu() clear_csr_euen(CSR_EUEN_FPEN) 93 94#define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU) 95 96static inline int is_fpu_owner(void) 97{ 98 return test_thread_flag(TIF_USEDFPU); 99} 100 101static inline void __own_fpu(void) 102{ 103 enable_fpu(); 104 set_thread_flag(TIF_USEDFPU); 105 KSTK_EUEN(current) |= CSR_EUEN_FPEN; 106} 107 108static inline void own_fpu_inatomic(int restore) 109{ 110 if (cpu_has_fpu && !is_fpu_owner()) { 111 __own_fpu(); 112 if (restore) 113 _restore_fp(¤t->thread.fpu); 114 } 115} 116 117static inline void own_fpu(int restore) 118{ 119 preempt_disable(); 120 own_fpu_inatomic(restore); 121 preempt_enable(); 122} 123 124static inline void lose_fpu_inatomic(int save, struct task_struct *tsk) 125{ 126 if (is_fpu_owner()) { 127 if (!is_simd_enabled()) { 128 if (save) 129 _save_fp(&tsk->thread.fpu); 130 disable_fpu(); 131 } else { 132 if (save) { 133 if (!is_lasx_enabled()) 134 save_lsx(tsk); 135 else 136 save_lasx(tsk); 137 } 138 disable_fpu(); 139 disable_lsx(); 140 disable_lasx(); 141 clear_tsk_thread_flag(tsk, TIF_USEDSIMD); 142 } 143 clear_tsk_thread_flag(tsk, TIF_USEDFPU); 144 } 145 KSTK_EUEN(tsk) &= ~(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 146} 147 148static inline void lose_fpu(int save) 149{ 150 preempt_disable(); 151 lose_fpu_inatomic(save, current); 152 preempt_enable(); 153} 154 155static inline void init_fpu(void) 156{ 157 unsigned int fcsr = current->thread.fpu.fcsr; 158 159 __own_fpu(); 160 _init_fpu(fcsr); 161 set_used_math(); 162} 163 164static inline void save_fp(struct task_struct *tsk) 165{ 166 if (cpu_has_fpu) 167 _save_fp(&tsk->thread.fpu); 168} 169 170static inline void restore_fp(struct task_struct *tsk) 171{ 172 if (cpu_has_fpu) 173 _restore_fp(&tsk->thread.fpu); 174} 175 176static inline void save_fpu_regs(struct task_struct *tsk) 177{ 178 unsigned int euen; 179 180 if (tsk == current) { 181 preempt_disable(); 182 183 euen = csr_read32(LOONGARCH_CSR_EUEN); 184 185#ifdef CONFIG_CPU_HAS_LASX 186 if (euen & CSR_EUEN_LASXEN) 187 _save_lasx(¤t->thread.fpu); 188 else 189#endif 190#ifdef CONFIG_CPU_HAS_LSX 191 if (euen & CSR_EUEN_LSXEN) 192 _save_lsx(¤t->thread.fpu); 193 else 194#endif 195 if (euen & CSR_EUEN_FPEN) 196 _save_fp(¤t->thread.fpu); 197 198 preempt_enable(); 199 } 200} 201 202static inline int is_simd_owner(void) 203{ 204 return test_thread_flag(TIF_USEDSIMD); 205} 206 207#ifdef CONFIG_CPU_HAS_LSX 208 209static inline void enable_lsx(void) 210{ 211 if (cpu_has_lsx) 212 csr_xchg32(CSR_EUEN_LSXEN, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); 213} 214 215static inline void disable_lsx(void) 216{ 217 if (cpu_has_lsx) 218 csr_xchg32(0, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); 219} 220 221static inline void save_lsx(struct task_struct *t) 222{ 223 if (cpu_has_lsx) 224 _save_lsx(&t->thread.fpu); 225} 226 227static inline void restore_lsx(struct task_struct *t) 228{ 229 if (cpu_has_lsx) 230 _restore_lsx(&t->thread.fpu); 231} 232 233static inline void init_lsx_upper(void) 234{ 235 if (cpu_has_lsx) 236 _init_lsx_upper(); 237} 238 239static inline void restore_lsx_upper(struct task_struct *t) 240{ 241 if (cpu_has_lsx) 242 _restore_lsx_upper(&t->thread.fpu); 243} 244 245#else 246static inline void enable_lsx(void) {} 247static inline void disable_lsx(void) {} 248static inline void save_lsx(struct task_struct *t) {} 249static inline void restore_lsx(struct task_struct *t) {} 250static inline void init_lsx_upper(void) {} 251static inline void restore_lsx_upper(struct task_struct *t) {} 252#endif 253 254#ifdef CONFIG_CPU_HAS_LASX 255 256static inline void enable_lasx(void) 257{ 258 259 if (cpu_has_lasx) 260 csr_xchg32(CSR_EUEN_LASXEN, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); 261} 262 263static inline void disable_lasx(void) 264{ 265 if (cpu_has_lasx) 266 csr_xchg32(0, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); 267} 268 269static inline void save_lasx(struct task_struct *t) 270{ 271 if (cpu_has_lasx) 272 _save_lasx(&t->thread.fpu); 273} 274 275static inline void restore_lasx(struct task_struct *t) 276{ 277 if (cpu_has_lasx) 278 _restore_lasx(&t->thread.fpu); 279} 280 281static inline void init_lasx_upper(void) 282{ 283 if (cpu_has_lasx) 284 _init_lasx_upper(); 285} 286 287static inline void restore_lasx_upper(struct task_struct *t) 288{ 289 if (cpu_has_lasx) 290 _restore_lasx_upper(&t->thread.fpu); 291} 292 293#else 294static inline void enable_lasx(void) {} 295static inline void disable_lasx(void) {} 296static inline void save_lasx(struct task_struct *t) {} 297static inline void restore_lasx(struct task_struct *t) {} 298static inline void init_lasx_upper(void) {} 299static inline void restore_lasx_upper(struct task_struct *t) {} 300#endif 301 302static inline int thread_lsx_context_live(void) 303{ 304 if (!cpu_has_lsx) 305 return 0; 306 307 return test_thread_flag(TIF_LSX_CTX_LIVE); 308} 309 310static inline int thread_lasx_context_live(void) 311{ 312 if (!cpu_has_lasx) 313 return 0; 314 315 return test_thread_flag(TIF_LASX_CTX_LIVE); 316} 317 318#endif /* _ASM_FPU_H */ 319