1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2023 Loongson Technology Corporation Limited
4 */
5
6#include <linux/cpu.h>
7#include <linux/init.h>
8#include <asm/fpu.h>
9#include <asm/smp.h>
10
11static unsigned int euen_mask = CSR_EUEN_FPEN;
12
13/*
14 * The critical section between kernel_fpu_begin() and kernel_fpu_end()
15 * is non-reentrant. It is the caller's responsibility to avoid reentrance.
16 * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example.
17 */
18static DEFINE_PER_CPU(bool, in_kernel_fpu);
19static DEFINE_PER_CPU(unsigned int, euen_current);
20
21void kernel_fpu_begin(void)
22{
23	unsigned int *euen_curr;
24
25	preempt_disable();
26
27	WARN_ON(this_cpu_read(in_kernel_fpu));
28
29	this_cpu_write(in_kernel_fpu, true);
30	euen_curr = this_cpu_ptr(&euen_current);
31
32	*euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN);
33
34#ifdef CONFIG_CPU_HAS_LASX
35	if (*euen_curr & CSR_EUEN_LASXEN)
36		_save_lasx(&current->thread.fpu);
37	else
38#endif
39#ifdef CONFIG_CPU_HAS_LSX
40	if (*euen_curr & CSR_EUEN_LSXEN)
41		_save_lsx(&current->thread.fpu);
42	else
43#endif
44	if (*euen_curr & CSR_EUEN_FPEN)
45		_save_fp(&current->thread.fpu);
46
47	write_fcsr(LOONGARCH_FCSR0, 0);
48}
49EXPORT_SYMBOL_GPL(kernel_fpu_begin);
50
51void kernel_fpu_end(void)
52{
53	unsigned int *euen_curr;
54
55	WARN_ON(!this_cpu_read(in_kernel_fpu));
56
57	euen_curr = this_cpu_ptr(&euen_current);
58
59#ifdef CONFIG_CPU_HAS_LASX
60	if (*euen_curr & CSR_EUEN_LASXEN)
61		_restore_lasx(&current->thread.fpu);
62	else
63#endif
64#ifdef CONFIG_CPU_HAS_LSX
65	if (*euen_curr & CSR_EUEN_LSXEN)
66		_restore_lsx(&current->thread.fpu);
67	else
68#endif
69	if (*euen_curr & CSR_EUEN_FPEN)
70		_restore_fp(&current->thread.fpu);
71
72	*euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN);
73
74	this_cpu_write(in_kernel_fpu, false);
75
76	preempt_enable();
77}
78EXPORT_SYMBOL_GPL(kernel_fpu_end);
79
80static int __init init_euen_mask(void)
81{
82	if (cpu_has_lsx)
83		euen_mask |= CSR_EUEN_LSXEN;
84
85	if (cpu_has_lasx)
86		euen_mask |= CSR_EUEN_LASXEN;
87
88	return 0;
89}
90arch_initcall(init_euen_mask);
91