1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef _ASM_ASMMACRO_H
6#define _ASM_ASMMACRO_H
7
8#include <asm/asm-offsets.h>
9#include <asm/regdef.h>
10#include <asm/fpregdef.h>
11#include <asm/loongarch.h>
12
13	.macro	cpu_save_nonscratch thread
14	stptr.d	s0, \thread, THREAD_REG23
15	stptr.d	s1, \thread, THREAD_REG24
16	stptr.d	s2, \thread, THREAD_REG25
17	stptr.d	s3, \thread, THREAD_REG26
18	stptr.d	s4, \thread, THREAD_REG27
19	stptr.d	s5, \thread, THREAD_REG28
20	stptr.d	s6, \thread, THREAD_REG29
21	stptr.d	s7, \thread, THREAD_REG30
22	stptr.d	s8, \thread, THREAD_REG31
23	stptr.d	sp, \thread, THREAD_REG03
24	stptr.d	fp, \thread, THREAD_REG22
25	.endm
26
27	.macro	cpu_restore_nonscratch thread
28	ldptr.d	s0, \thread, THREAD_REG23
29	ldptr.d	s1, \thread, THREAD_REG24
30	ldptr.d	s2, \thread, THREAD_REG25
31	ldptr.d	s3, \thread, THREAD_REG26
32	ldptr.d	s4, \thread, THREAD_REG27
33	ldptr.d	s5, \thread, THREAD_REG28
34	ldptr.d	s6, \thread, THREAD_REG29
35	ldptr.d	s7, \thread, THREAD_REG30
36	ldptr.d	s8, \thread, THREAD_REG31
37	ldptr.d	ra, \thread, THREAD_REG01
38	ldptr.d	sp, \thread, THREAD_REG03
39	ldptr.d	fp, \thread, THREAD_REG22
40	.endm
41
42	.macro fpu_save_csr thread tmp
43	movfcsr2gr	\tmp, fcsr0
44	stptr.w		\tmp, \thread, THREAD_FCSR
45#ifdef CONFIG_CPU_HAS_LBT
46	/* TM bit is always 0 if LBT not supported */
47	andi		\tmp, \tmp, FPU_CSR_TM
48	beqz		\tmp, 1f
49	/* Save FTOP */
50	x86mftop	\tmp
51	stptr.w		\tmp, \thread, THREAD_FTOP
52	/* Turn off TM to ensure the order of FPR in memory independent of TM */
53	x86clrtm
541:
55#endif
56	.endm
57
58	.macro fpu_restore_csr thread tmp0 tmp1
59	ldptr.w		\tmp0, \thread, THREAD_FCSR
60	movgr2fcsr	fcsr0, \tmp0
61#ifdef CONFIG_CPU_HAS_LBT
62	/* TM bit is always 0 if LBT not supported */
63	andi		\tmp0, \tmp0, FPU_CSR_TM
64	beqz		\tmp0, 2f
65	/* Restore FTOP */
66	ldptr.w		\tmp0, \thread, THREAD_FTOP
67	andi		\tmp0, \tmp0, 0x7
68	la.pcrel	\tmp1, 1f
69	alsl.d		\tmp1, \tmp0, \tmp1, 3
70	jr		\tmp1
711:
72	x86mttop	0
73	b	2f
74	x86mttop	1
75	b	2f
76	x86mttop	2
77	b	2f
78	x86mttop	3
79	b	2f
80	x86mttop	4
81	b	2f
82	x86mttop	5
83	b	2f
84	x86mttop	6
85	b	2f
86	x86mttop	7
872:
88#endif
89	.endm
90
91	.macro fpu_save_cc thread tmp0 tmp1
92	movcf2gr	\tmp0, $fcc0
93	move	\tmp1, \tmp0
94	movcf2gr	\tmp0, $fcc1
95	bstrins.d	\tmp1, \tmp0, 15, 8
96	movcf2gr	\tmp0, $fcc2
97	bstrins.d	\tmp1, \tmp0, 23, 16
98	movcf2gr	\tmp0, $fcc3
99	bstrins.d	\tmp1, \tmp0, 31, 24
100	movcf2gr	\tmp0, $fcc4
101	bstrins.d	\tmp1, \tmp0, 39, 32
102	movcf2gr	\tmp0, $fcc5
103	bstrins.d	\tmp1, \tmp0, 47, 40
104	movcf2gr	\tmp0, $fcc6
105	bstrins.d	\tmp1, \tmp0, 55, 48
106	movcf2gr	\tmp0, $fcc7
107	bstrins.d	\tmp1, \tmp0, 63, 56
108	stptr.d		\tmp1, \thread, THREAD_FCC
109	.endm
110
111	.macro fpu_restore_cc thread tmp0 tmp1
112	ldptr.d	\tmp0, \thread, THREAD_FCC
113	bstrpick.d	\tmp1, \tmp0, 7, 0
114	movgr2cf	$fcc0, \tmp1
115	bstrpick.d	\tmp1, \tmp0, 15, 8
116	movgr2cf	$fcc1, \tmp1
117	bstrpick.d	\tmp1, \tmp0, 23, 16
118	movgr2cf	$fcc2, \tmp1
119	bstrpick.d	\tmp1, \tmp0, 31, 24
120	movgr2cf	$fcc3, \tmp1
121	bstrpick.d	\tmp1, \tmp0, 39, 32
122	movgr2cf	$fcc4, \tmp1
123	bstrpick.d	\tmp1, \tmp0, 47, 40
124	movgr2cf	$fcc5, \tmp1
125	bstrpick.d	\tmp1, \tmp0, 55, 48
126	movgr2cf	$fcc6, \tmp1
127	bstrpick.d	\tmp1, \tmp0, 63, 56
128	movgr2cf	$fcc7, \tmp1
129	.endm
130
131	.macro	fpu_save_double thread tmp
132	li.w	\tmp, THREAD_FPR0
133	PTR_ADD \tmp, \tmp, \thread
134	fst.d	$f0, \tmp, THREAD_FPR0  - THREAD_FPR0
135	fst.d	$f1, \tmp, THREAD_FPR1  - THREAD_FPR0
136	fst.d	$f2, \tmp, THREAD_FPR2  - THREAD_FPR0
137	fst.d	$f3, \tmp, THREAD_FPR3  - THREAD_FPR0
138	fst.d	$f4, \tmp, THREAD_FPR4  - THREAD_FPR0
139	fst.d	$f5, \tmp, THREAD_FPR5  - THREAD_FPR0
140	fst.d	$f6, \tmp, THREAD_FPR6  - THREAD_FPR0
141	fst.d	$f7, \tmp, THREAD_FPR7  - THREAD_FPR0
142	fst.d	$f8, \tmp, THREAD_FPR8  - THREAD_FPR0
143	fst.d	$f9, \tmp, THREAD_FPR9  - THREAD_FPR0
144	fst.d	$f10, \tmp, THREAD_FPR10 - THREAD_FPR0
145	fst.d	$f11, \tmp, THREAD_FPR11 - THREAD_FPR0
146	fst.d	$f12, \tmp, THREAD_FPR12 - THREAD_FPR0
147	fst.d	$f13, \tmp, THREAD_FPR13 - THREAD_FPR0
148	fst.d	$f14, \tmp, THREAD_FPR14 - THREAD_FPR0
149	fst.d	$f15, \tmp, THREAD_FPR15 - THREAD_FPR0
150	fst.d	$f16, \tmp, THREAD_FPR16 - THREAD_FPR0
151	fst.d	$f17, \tmp, THREAD_FPR17 - THREAD_FPR0
152	fst.d	$f18, \tmp, THREAD_FPR18 - THREAD_FPR0
153	fst.d	$f19, \tmp, THREAD_FPR19 - THREAD_FPR0
154	fst.d	$f20, \tmp, THREAD_FPR20 - THREAD_FPR0
155	fst.d	$f21, \tmp, THREAD_FPR21 - THREAD_FPR0
156	fst.d	$f22, \tmp, THREAD_FPR22 - THREAD_FPR0
157	fst.d	$f23, \tmp, THREAD_FPR23 - THREAD_FPR0
158	fst.d	$f24, \tmp, THREAD_FPR24 - THREAD_FPR0
159	fst.d	$f25, \tmp, THREAD_FPR25 - THREAD_FPR0
160	fst.d	$f26, \tmp, THREAD_FPR26 - THREAD_FPR0
161	fst.d	$f27, \tmp, THREAD_FPR27 - THREAD_FPR0
162	fst.d	$f28, \tmp, THREAD_FPR28 - THREAD_FPR0
163	fst.d	$f29, \tmp, THREAD_FPR29 - THREAD_FPR0
164	fst.d	$f30, \tmp, THREAD_FPR30 - THREAD_FPR0
165	fst.d	$f31, \tmp, THREAD_FPR31 - THREAD_FPR0
166	.endm
167
168	.macro	fpu_restore_double thread tmp
169	li.w	\tmp, THREAD_FPR0
170	PTR_ADD \tmp, \tmp, \thread
171	fld.d	$f0, \tmp, THREAD_FPR0  - THREAD_FPR0
172	fld.d	$f1, \tmp, THREAD_FPR1  - THREAD_FPR0
173	fld.d	$f2, \tmp, THREAD_FPR2  - THREAD_FPR0
174	fld.d	$f3, \tmp, THREAD_FPR3  - THREAD_FPR0
175	fld.d	$f4, \tmp, THREAD_FPR4  - THREAD_FPR0
176	fld.d	$f5, \tmp, THREAD_FPR5  - THREAD_FPR0
177	fld.d	$f6, \tmp, THREAD_FPR6  - THREAD_FPR0
178	fld.d	$f7, \tmp, THREAD_FPR7  - THREAD_FPR0
179	fld.d	$f8, \tmp, THREAD_FPR8  - THREAD_FPR0
180	fld.d	$f9, \tmp, THREAD_FPR9  - THREAD_FPR0
181	fld.d	$f10, \tmp, THREAD_FPR10 - THREAD_FPR0
182	fld.d	$f11, \tmp, THREAD_FPR11 - THREAD_FPR0
183	fld.d	$f12, \tmp, THREAD_FPR12 - THREAD_FPR0
184	fld.d	$f13, \tmp, THREAD_FPR13 - THREAD_FPR0
185	fld.d	$f14, \tmp, THREAD_FPR14 - THREAD_FPR0
186	fld.d	$f15, \tmp, THREAD_FPR15 - THREAD_FPR0
187	fld.d	$f16, \tmp, THREAD_FPR16 - THREAD_FPR0
188	fld.d	$f17, \tmp, THREAD_FPR17 - THREAD_FPR0
189	fld.d	$f18, \tmp, THREAD_FPR18 - THREAD_FPR0
190	fld.d	$f19, \tmp, THREAD_FPR19 - THREAD_FPR0
191	fld.d	$f20, \tmp, THREAD_FPR20 - THREAD_FPR0
192	fld.d	$f21, \tmp, THREAD_FPR21 - THREAD_FPR0
193	fld.d	$f22, \tmp, THREAD_FPR22 - THREAD_FPR0
194	fld.d	$f23, \tmp, THREAD_FPR23 - THREAD_FPR0
195	fld.d	$f24, \tmp, THREAD_FPR24 - THREAD_FPR0
196	fld.d	$f25, \tmp, THREAD_FPR25 - THREAD_FPR0
197	fld.d	$f26, \tmp, THREAD_FPR26 - THREAD_FPR0
198	fld.d	$f27, \tmp, THREAD_FPR27 - THREAD_FPR0
199	fld.d	$f28, \tmp, THREAD_FPR28 - THREAD_FPR0
200	fld.d	$f29, \tmp, THREAD_FPR29 - THREAD_FPR0
201	fld.d	$f30, \tmp, THREAD_FPR30 - THREAD_FPR0
202	fld.d	$f31, \tmp, THREAD_FPR31 - THREAD_FPR0
203	.endm
204
205	.macro	lsx_save_data thread tmp
206	li.w	\tmp, THREAD_FPR0
207	PTR_ADD \tmp, \thread, \tmp
208	vst	$vr0, \tmp, THREAD_FPR0  - THREAD_FPR0
209	vst	$vr1, \tmp, THREAD_FPR1  - THREAD_FPR0
210	vst	$vr2, \tmp, THREAD_FPR2  - THREAD_FPR0
211	vst	$vr3, \tmp, THREAD_FPR3  - THREAD_FPR0
212	vst	$vr4, \tmp, THREAD_FPR4  - THREAD_FPR0
213	vst	$vr5, \tmp, THREAD_FPR5  - THREAD_FPR0
214	vst	$vr6, \tmp, THREAD_FPR6  - THREAD_FPR0
215	vst	$vr7, \tmp, THREAD_FPR7  - THREAD_FPR0
216	vst	$vr8, \tmp, THREAD_FPR8  - THREAD_FPR0
217	vst	$vr9, \tmp, THREAD_FPR9  - THREAD_FPR0
218	vst	$vr10, \tmp, THREAD_FPR10 - THREAD_FPR0
219	vst	$vr11, \tmp, THREAD_FPR11 - THREAD_FPR0
220	vst	$vr12, \tmp, THREAD_FPR12 - THREAD_FPR0
221	vst	$vr13, \tmp, THREAD_FPR13 - THREAD_FPR0
222	vst	$vr14, \tmp, THREAD_FPR14 - THREAD_FPR0
223	vst	$vr15, \tmp, THREAD_FPR15 - THREAD_FPR0
224	vst	$vr16, \tmp, THREAD_FPR16 - THREAD_FPR0
225	vst	$vr17, \tmp, THREAD_FPR17 - THREAD_FPR0
226	vst	$vr18, \tmp, THREAD_FPR18 - THREAD_FPR0
227	vst	$vr19, \tmp, THREAD_FPR19 - THREAD_FPR0
228	vst	$vr20, \tmp, THREAD_FPR20 - THREAD_FPR0
229	vst	$vr21, \tmp, THREAD_FPR21 - THREAD_FPR0
230	vst	$vr22, \tmp, THREAD_FPR22 - THREAD_FPR0
231	vst	$vr23, \tmp, THREAD_FPR23 - THREAD_FPR0
232	vst	$vr24, \tmp, THREAD_FPR24 - THREAD_FPR0
233	vst	$vr25, \tmp, THREAD_FPR25 - THREAD_FPR0
234	vst	$vr26, \tmp, THREAD_FPR26 - THREAD_FPR0
235	vst	$vr27, \tmp, THREAD_FPR27 - THREAD_FPR0
236	vst	$vr28, \tmp, THREAD_FPR28 - THREAD_FPR0
237	vst	$vr29, \tmp, THREAD_FPR29 - THREAD_FPR0
238	vst	$vr30, \tmp, THREAD_FPR30 - THREAD_FPR0
239	vst	$vr31, \tmp, THREAD_FPR31 - THREAD_FPR0
240	.endm
241
242	.macro	lsx_restore_data thread tmp
243	li.w	\tmp, THREAD_FPR0
244	PTR_ADD	\tmp, \thread, \tmp
245	vld	$vr0, \tmp, THREAD_FPR0  - THREAD_FPR0
246	vld	$vr1, \tmp, THREAD_FPR1  - THREAD_FPR0
247	vld	$vr2, \tmp, THREAD_FPR2  - THREAD_FPR0
248	vld	$vr3, \tmp, THREAD_FPR3  - THREAD_FPR0
249	vld	$vr4, \tmp, THREAD_FPR4  - THREAD_FPR0
250	vld	$vr5, \tmp, THREAD_FPR5  - THREAD_FPR0
251	vld	$vr6, \tmp, THREAD_FPR6  - THREAD_FPR0
252	vld	$vr7, \tmp, THREAD_FPR7  - THREAD_FPR0
253	vld	$vr8, \tmp, THREAD_FPR8  - THREAD_FPR0
254	vld	$vr9, \tmp, THREAD_FPR9  - THREAD_FPR0
255	vld	$vr10, \tmp, THREAD_FPR10 - THREAD_FPR0
256	vld	$vr11, \tmp, THREAD_FPR11 - THREAD_FPR0
257	vld	$vr12, \tmp, THREAD_FPR12 - THREAD_FPR0
258	vld	$vr13, \tmp, THREAD_FPR13 - THREAD_FPR0
259	vld	$vr14, \tmp, THREAD_FPR14 - THREAD_FPR0
260	vld	$vr15, \tmp, THREAD_FPR15 - THREAD_FPR0
261	vld	$vr16, \tmp, THREAD_FPR16 - THREAD_FPR0
262	vld	$vr17, \tmp, THREAD_FPR17 - THREAD_FPR0
263	vld	$vr18, \tmp, THREAD_FPR18 - THREAD_FPR0
264	vld	$vr19, \tmp, THREAD_FPR19 - THREAD_FPR0
265	vld	$vr20, \tmp, THREAD_FPR20 - THREAD_FPR0
266	vld	$vr21, \tmp, THREAD_FPR21 - THREAD_FPR0
267	vld	$vr22, \tmp, THREAD_FPR22 - THREAD_FPR0
268	vld	$vr23, \tmp, THREAD_FPR23 - THREAD_FPR0
269	vld	$vr24, \tmp, THREAD_FPR24 - THREAD_FPR0
270	vld	$vr25, \tmp, THREAD_FPR25 - THREAD_FPR0
271	vld	$vr26, \tmp, THREAD_FPR26 - THREAD_FPR0
272	vld	$vr27, \tmp, THREAD_FPR27 - THREAD_FPR0
273	vld	$vr28, \tmp, THREAD_FPR28 - THREAD_FPR0
274	vld	$vr29, \tmp, THREAD_FPR29 - THREAD_FPR0
275	vld	$vr30, \tmp, THREAD_FPR30 - THREAD_FPR0
276	vld	$vr31, \tmp, THREAD_FPR31 - THREAD_FPR0
277	.endm
278
279	.macro	lsx_save_all	thread tmp0 tmp1
280	fpu_save_cc		\thread, \tmp0, \tmp1
281	fpu_save_csr		\thread, \tmp0
282	lsx_save_data		\thread, \tmp0
283	.endm
284
285	.macro	lsx_restore_all	thread tmp0 tmp1
286	lsx_restore_data	\thread, \tmp0
287	fpu_restore_cc		\thread, \tmp0, \tmp1
288	fpu_restore_csr		\thread, \tmp0, \tmp1
289	.endm
290
291	.macro	lsx_save_upper vd base tmp off
292	vpickve2gr.d	\tmp, \vd, 1
293	st.d		\tmp, \base, (\off+8)
294	.endm
295
296	.macro	lsx_save_all_upper thread base tmp
297	li.w		\tmp, THREAD_FPR0
298	PTR_ADD		\base, \thread, \tmp
299	lsx_save_upper	$vr0,  \base, \tmp, (THREAD_FPR0-THREAD_FPR0)
300	lsx_save_upper	$vr1,  \base, \tmp, (THREAD_FPR1-THREAD_FPR0)
301	lsx_save_upper	$vr2,  \base, \tmp, (THREAD_FPR2-THREAD_FPR0)
302	lsx_save_upper	$vr3,  \base, \tmp, (THREAD_FPR3-THREAD_FPR0)
303	lsx_save_upper	$vr4,  \base, \tmp, (THREAD_FPR4-THREAD_FPR0)
304	lsx_save_upper	$vr5,  \base, \tmp, (THREAD_FPR5-THREAD_FPR0)
305	lsx_save_upper	$vr6,  \base, \tmp, (THREAD_FPR6-THREAD_FPR0)
306	lsx_save_upper	$vr7,  \base, \tmp, (THREAD_FPR7-THREAD_FPR0)
307	lsx_save_upper	$vr8,  \base, \tmp, (THREAD_FPR8-THREAD_FPR0)
308	lsx_save_upper	$vr9,  \base, \tmp, (THREAD_FPR9-THREAD_FPR0)
309	lsx_save_upper	$vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0)
310	lsx_save_upper	$vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0)
311	lsx_save_upper	$vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0)
312	lsx_save_upper	$vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0)
313	lsx_save_upper	$vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0)
314	lsx_save_upper	$vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0)
315	lsx_save_upper	$vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0)
316	lsx_save_upper	$vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0)
317	lsx_save_upper	$vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0)
318	lsx_save_upper	$vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0)
319	lsx_save_upper	$vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0)
320	lsx_save_upper	$vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0)
321	lsx_save_upper	$vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0)
322	lsx_save_upper	$vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0)
323	lsx_save_upper	$vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0)
324	lsx_save_upper	$vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0)
325	lsx_save_upper	$vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0)
326	lsx_save_upper	$vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0)
327	lsx_save_upper	$vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0)
328	lsx_save_upper	$vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0)
329	lsx_save_upper	$vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0)
330	lsx_save_upper	$vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0)
331	.endm
332
333	.macro	lsx_restore_upper vd base tmp off
334	ld.d		\tmp, \base, (\off+8)
335	vinsgr2vr.d	\vd,  \tmp, 1
336	.endm
337
338	.macro	lsx_restore_all_upper thread base tmp
339	li.w		  \tmp, THREAD_FPR0
340	PTR_ADD		  \base, \thread, \tmp
341	lsx_restore_upper $vr0,  \base, \tmp, (THREAD_FPR0-THREAD_FPR0)
342	lsx_restore_upper $vr1,  \base, \tmp, (THREAD_FPR1-THREAD_FPR0)
343	lsx_restore_upper $vr2,  \base, \tmp, (THREAD_FPR2-THREAD_FPR0)
344	lsx_restore_upper $vr3,  \base, \tmp, (THREAD_FPR3-THREAD_FPR0)
345	lsx_restore_upper $vr4,  \base, \tmp, (THREAD_FPR4-THREAD_FPR0)
346	lsx_restore_upper $vr5,  \base, \tmp, (THREAD_FPR5-THREAD_FPR0)
347	lsx_restore_upper $vr6,  \base, \tmp, (THREAD_FPR6-THREAD_FPR0)
348	lsx_restore_upper $vr7,  \base, \tmp, (THREAD_FPR7-THREAD_FPR0)
349	lsx_restore_upper $vr8,  \base, \tmp, (THREAD_FPR8-THREAD_FPR0)
350	lsx_restore_upper $vr9,  \base, \tmp, (THREAD_FPR9-THREAD_FPR0)
351	lsx_restore_upper $vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0)
352	lsx_restore_upper $vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0)
353	lsx_restore_upper $vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0)
354	lsx_restore_upper $vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0)
355	lsx_restore_upper $vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0)
356	lsx_restore_upper $vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0)
357	lsx_restore_upper $vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0)
358	lsx_restore_upper $vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0)
359	lsx_restore_upper $vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0)
360	lsx_restore_upper $vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0)
361	lsx_restore_upper $vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0)
362	lsx_restore_upper $vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0)
363	lsx_restore_upper $vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0)
364	lsx_restore_upper $vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0)
365	lsx_restore_upper $vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0)
366	lsx_restore_upper $vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0)
367	lsx_restore_upper $vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0)
368	lsx_restore_upper $vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0)
369	lsx_restore_upper $vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0)
370	lsx_restore_upper $vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0)
371	lsx_restore_upper $vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0)
372	lsx_restore_upper $vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0)
373	.endm
374
375	.macro	lsx_init_upper vd tmp
376	vinsgr2vr.d	\vd, \tmp, 1
377	.endm
378
379	.macro	lsx_init_all_upper tmp
380	not		\tmp, zero
381	lsx_init_upper	$vr0 \tmp
382	lsx_init_upper	$vr1 \tmp
383	lsx_init_upper	$vr2 \tmp
384	lsx_init_upper	$vr3 \tmp
385	lsx_init_upper	$vr4 \tmp
386	lsx_init_upper	$vr5 \tmp
387	lsx_init_upper	$vr6 \tmp
388	lsx_init_upper	$vr7 \tmp
389	lsx_init_upper	$vr8 \tmp
390	lsx_init_upper	$vr9 \tmp
391	lsx_init_upper	$vr10 \tmp
392	lsx_init_upper	$vr11 \tmp
393	lsx_init_upper	$vr12 \tmp
394	lsx_init_upper	$vr13 \tmp
395	lsx_init_upper	$vr14 \tmp
396	lsx_init_upper	$vr15 \tmp
397	lsx_init_upper	$vr16 \tmp
398	lsx_init_upper	$vr17 \tmp
399	lsx_init_upper	$vr18 \tmp
400	lsx_init_upper	$vr19 \tmp
401	lsx_init_upper	$vr20 \tmp
402	lsx_init_upper	$vr21 \tmp
403	lsx_init_upper	$vr22 \tmp
404	lsx_init_upper	$vr23 \tmp
405	lsx_init_upper	$vr24 \tmp
406	lsx_init_upper	$vr25 \tmp
407	lsx_init_upper	$vr26 \tmp
408	lsx_init_upper	$vr27 \tmp
409	lsx_init_upper	$vr28 \tmp
410	lsx_init_upper	$vr29 \tmp
411	lsx_init_upper	$vr30 \tmp
412	lsx_init_upper	$vr31 \tmp
413	.endm
414
415	.macro	lasx_save_data thread tmp
416	li.w	\tmp, THREAD_FPR0
417	PTR_ADD	\tmp, \thread, \tmp
418	xvst	$xr0, \tmp, THREAD_FPR0  - THREAD_FPR0
419	xvst	$xr1, \tmp, THREAD_FPR1  - THREAD_FPR0
420	xvst	$xr2, \tmp, THREAD_FPR2  - THREAD_FPR0
421	xvst	$xr3, \tmp, THREAD_FPR3  - THREAD_FPR0
422	xvst	$xr4, \tmp, THREAD_FPR4  - THREAD_FPR0
423	xvst	$xr5, \tmp, THREAD_FPR5  - THREAD_FPR0
424	xvst	$xr6, \tmp, THREAD_FPR6  - THREAD_FPR0
425	xvst	$xr7, \tmp, THREAD_FPR7  - THREAD_FPR0
426	xvst	$xr8, \tmp, THREAD_FPR8  - THREAD_FPR0
427	xvst	$xr9, \tmp, THREAD_FPR9  - THREAD_FPR0
428	xvst	$xr10, \tmp, THREAD_FPR10 - THREAD_FPR0
429	xvst	$xr11, \tmp, THREAD_FPR11 - THREAD_FPR0
430	xvst	$xr12, \tmp, THREAD_FPR12 - THREAD_FPR0
431	xvst	$xr13, \tmp, THREAD_FPR13 - THREAD_FPR0
432	xvst	$xr14, \tmp, THREAD_FPR14 - THREAD_FPR0
433	xvst	$xr15, \tmp, THREAD_FPR15 - THREAD_FPR0
434	xvst	$xr16, \tmp, THREAD_FPR16 - THREAD_FPR0
435	xvst	$xr17, \tmp, THREAD_FPR17 - THREAD_FPR0
436	xvst	$xr18, \tmp, THREAD_FPR18 - THREAD_FPR0
437	xvst	$xr19, \tmp, THREAD_FPR19 - THREAD_FPR0
438	xvst	$xr20, \tmp, THREAD_FPR20 - THREAD_FPR0
439	xvst	$xr21, \tmp, THREAD_FPR21 - THREAD_FPR0
440	xvst	$xr22, \tmp, THREAD_FPR22 - THREAD_FPR0
441	xvst	$xr23, \tmp, THREAD_FPR23 - THREAD_FPR0
442	xvst	$xr24, \tmp, THREAD_FPR24 - THREAD_FPR0
443	xvst	$xr25, \tmp, THREAD_FPR25 - THREAD_FPR0
444	xvst	$xr26, \tmp, THREAD_FPR26 - THREAD_FPR0
445	xvst	$xr27, \tmp, THREAD_FPR27 - THREAD_FPR0
446	xvst	$xr28, \tmp, THREAD_FPR28 - THREAD_FPR0
447	xvst	$xr29, \tmp, THREAD_FPR29 - THREAD_FPR0
448	xvst	$xr30, \tmp, THREAD_FPR30 - THREAD_FPR0
449	xvst	$xr31, \tmp, THREAD_FPR31 - THREAD_FPR0
450	.endm
451
452	.macro	lasx_restore_data thread tmp
453	li.w	\tmp, THREAD_FPR0
454	PTR_ADD	\tmp, \thread, \tmp
455	xvld	$xr0, \tmp, THREAD_FPR0  - THREAD_FPR0
456	xvld	$xr1, \tmp, THREAD_FPR1  - THREAD_FPR0
457	xvld	$xr2, \tmp, THREAD_FPR2  - THREAD_FPR0
458	xvld	$xr3, \tmp, THREAD_FPR3  - THREAD_FPR0
459	xvld	$xr4, \tmp, THREAD_FPR4  - THREAD_FPR0
460	xvld	$xr5, \tmp, THREAD_FPR5  - THREAD_FPR0
461	xvld	$xr6, \tmp, THREAD_FPR6  - THREAD_FPR0
462	xvld	$xr7, \tmp, THREAD_FPR7  - THREAD_FPR0
463	xvld	$xr8, \tmp, THREAD_FPR8  - THREAD_FPR0
464	xvld	$xr9, \tmp, THREAD_FPR9  - THREAD_FPR0
465	xvld	$xr10, \tmp, THREAD_FPR10 - THREAD_FPR0
466	xvld	$xr11, \tmp, THREAD_FPR11 - THREAD_FPR0
467	xvld	$xr12, \tmp, THREAD_FPR12 - THREAD_FPR0
468	xvld	$xr13, \tmp, THREAD_FPR13 - THREAD_FPR0
469	xvld	$xr14, \tmp, THREAD_FPR14 - THREAD_FPR0
470	xvld	$xr15, \tmp, THREAD_FPR15 - THREAD_FPR0
471	xvld	$xr16, \tmp, THREAD_FPR16 - THREAD_FPR0
472	xvld	$xr17, \tmp, THREAD_FPR17 - THREAD_FPR0
473	xvld	$xr18, \tmp, THREAD_FPR18 - THREAD_FPR0
474	xvld	$xr19, \tmp, THREAD_FPR19 - THREAD_FPR0
475	xvld	$xr20, \tmp, THREAD_FPR20 - THREAD_FPR0
476	xvld	$xr21, \tmp, THREAD_FPR21 - THREAD_FPR0
477	xvld	$xr22, \tmp, THREAD_FPR22 - THREAD_FPR0
478	xvld	$xr23, \tmp, THREAD_FPR23 - THREAD_FPR0
479	xvld	$xr24, \tmp, THREAD_FPR24 - THREAD_FPR0
480	xvld	$xr25, \tmp, THREAD_FPR25 - THREAD_FPR0
481	xvld	$xr26, \tmp, THREAD_FPR26 - THREAD_FPR0
482	xvld	$xr27, \tmp, THREAD_FPR27 - THREAD_FPR0
483	xvld	$xr28, \tmp, THREAD_FPR28 - THREAD_FPR0
484	xvld	$xr29, \tmp, THREAD_FPR29 - THREAD_FPR0
485	xvld	$xr30, \tmp, THREAD_FPR30 - THREAD_FPR0
486	xvld	$xr31, \tmp, THREAD_FPR31 - THREAD_FPR0
487	.endm
488
489	.macro	lasx_save_all	thread tmp0 tmp1
490	fpu_save_cc		\thread, \tmp0, \tmp1
491	fpu_save_csr		\thread, \tmp0
492	lasx_save_data		\thread, \tmp0
493	.endm
494
495	.macro	lasx_restore_all thread tmp0 tmp1
496	lasx_restore_data	\thread, \tmp0
497	fpu_restore_cc		\thread, \tmp0, \tmp1
498	fpu_restore_csr		\thread, \tmp0, \tmp1
499	.endm
500
501	.macro	lasx_save_upper xd base tmp off
502	/* Nothing */
503	.endm
504
505	.macro	lasx_save_all_upper thread base tmp
506	/* Nothing */
507	.endm
508
509	.macro	lasx_restore_upper xd base tmp0 tmp1 off
510	vld		\tmp0, \base, (\off+16)
511	xvpermi.q 	\xd,   \tmp1, 0x2
512	.endm
513
514	.macro	lasx_restore_all_upper thread base tmp
515	li.w		\tmp, THREAD_FPR0
516	PTR_ADD		\base, \thread, \tmp
517	/* Save $vr31 ($xr31 lower bits) with xvpickve2gr */
518	xvpickve2gr.d	$r17, $xr31, 0
519	xvpickve2gr.d	$r18, $xr31, 1
520	lasx_restore_upper $xr0, \base, $vr31, $xr31, (THREAD_FPR0-THREAD_FPR0)
521	lasx_restore_upper $xr1, \base, $vr31, $xr31, (THREAD_FPR1-THREAD_FPR0)
522	lasx_restore_upper $xr2, \base, $vr31, $xr31, (THREAD_FPR2-THREAD_FPR0)
523	lasx_restore_upper $xr3, \base, $vr31, $xr31, (THREAD_FPR3-THREAD_FPR0)
524	lasx_restore_upper $xr4, \base, $vr31, $xr31, (THREAD_FPR4-THREAD_FPR0)
525	lasx_restore_upper $xr5, \base, $vr31, $xr31, (THREAD_FPR5-THREAD_FPR0)
526	lasx_restore_upper $xr6, \base, $vr31, $xr31, (THREAD_FPR6-THREAD_FPR0)
527	lasx_restore_upper $xr7, \base, $vr31, $xr31, (THREAD_FPR7-THREAD_FPR0)
528	lasx_restore_upper $xr8, \base, $vr31, $xr31, (THREAD_FPR8-THREAD_FPR0)
529	lasx_restore_upper $xr9, \base, $vr31, $xr31, (THREAD_FPR9-THREAD_FPR0)
530	lasx_restore_upper $xr10, \base, $vr31, $xr31, (THREAD_FPR10-THREAD_FPR0)
531	lasx_restore_upper $xr11, \base, $vr31, $xr31, (THREAD_FPR11-THREAD_FPR0)
532	lasx_restore_upper $xr12, \base, $vr31, $xr31, (THREAD_FPR12-THREAD_FPR0)
533	lasx_restore_upper $xr13, \base, $vr31, $xr31, (THREAD_FPR13-THREAD_FPR0)
534	lasx_restore_upper $xr14, \base, $vr31, $xr31, (THREAD_FPR14-THREAD_FPR0)
535	lasx_restore_upper $xr15, \base, $vr31, $xr31, (THREAD_FPR15-THREAD_FPR0)
536	lasx_restore_upper $xr16, \base, $vr31, $xr31, (THREAD_FPR16-THREAD_FPR0)
537	lasx_restore_upper $xr17, \base, $vr31, $xr31, (THREAD_FPR17-THREAD_FPR0)
538	lasx_restore_upper $xr18, \base, $vr31, $xr31, (THREAD_FPR18-THREAD_FPR0)
539	lasx_restore_upper $xr19, \base, $vr31, $xr31, (THREAD_FPR19-THREAD_FPR0)
540	lasx_restore_upper $xr20, \base, $vr31, $xr31, (THREAD_FPR20-THREAD_FPR0)
541	lasx_restore_upper $xr21, \base, $vr31, $xr31, (THREAD_FPR21-THREAD_FPR0)
542	lasx_restore_upper $xr22, \base, $vr31, $xr31, (THREAD_FPR22-THREAD_FPR0)
543	lasx_restore_upper $xr23, \base, $vr31, $xr31, (THREAD_FPR23-THREAD_FPR0)
544	lasx_restore_upper $xr24, \base, $vr31, $xr31, (THREAD_FPR24-THREAD_FPR0)
545	lasx_restore_upper $xr25, \base, $vr31, $xr31, (THREAD_FPR25-THREAD_FPR0)
546	lasx_restore_upper $xr26, \base, $vr31, $xr31, (THREAD_FPR26-THREAD_FPR0)
547	lasx_restore_upper $xr27, \base, $vr31, $xr31, (THREAD_FPR27-THREAD_FPR0)
548	lasx_restore_upper $xr28, \base, $vr31, $xr31, (THREAD_FPR28-THREAD_FPR0)
549	lasx_restore_upper $xr29, \base, $vr31, $xr31, (THREAD_FPR29-THREAD_FPR0)
550	lasx_restore_upper $xr30, \base, $vr31, $xr31, (THREAD_FPR30-THREAD_FPR0)
551	lasx_restore_upper $xr31, \base, $vr31, $xr31, (THREAD_FPR31-THREAD_FPR0)
552	/* Restore $vr31 ($xr31 lower bits) with xvinsgr2vr */
553	xvinsgr2vr.d	$xr31, $r17, 0
554	xvinsgr2vr.d	$xr31, $r18, 1
555	.endm
556
557	.macro	lasx_init_upper xd tmp
558	xvinsgr2vr.d	\xd, \tmp, 2
559	xvinsgr2vr.d	\xd, \tmp, 3
560	.endm
561
562	.macro	lasx_init_all_upper tmp
563	not		\tmp, zero
564	lasx_init_upper	$xr0 \tmp
565	lasx_init_upper	$xr1 \tmp
566	lasx_init_upper	$xr2 \tmp
567	lasx_init_upper	$xr3 \tmp
568	lasx_init_upper	$xr4 \tmp
569	lasx_init_upper	$xr5 \tmp
570	lasx_init_upper	$xr6 \tmp
571	lasx_init_upper	$xr7 \tmp
572	lasx_init_upper	$xr8 \tmp
573	lasx_init_upper	$xr9 \tmp
574	lasx_init_upper	$xr10 \tmp
575	lasx_init_upper	$xr11 \tmp
576	lasx_init_upper	$xr12 \tmp
577	lasx_init_upper	$xr13 \tmp
578	lasx_init_upper	$xr14 \tmp
579	lasx_init_upper	$xr15 \tmp
580	lasx_init_upper	$xr16 \tmp
581	lasx_init_upper	$xr17 \tmp
582	lasx_init_upper	$xr18 \tmp
583	lasx_init_upper	$xr19 \tmp
584	lasx_init_upper	$xr20 \tmp
585	lasx_init_upper	$xr21 \tmp
586	lasx_init_upper	$xr22 \tmp
587	lasx_init_upper	$xr23 \tmp
588	lasx_init_upper	$xr24 \tmp
589	lasx_init_upper	$xr25 \tmp
590	lasx_init_upper	$xr26 \tmp
591	lasx_init_upper	$xr27 \tmp
592	lasx_init_upper	$xr28 \tmp
593	lasx_init_upper	$xr29 \tmp
594	lasx_init_upper	$xr30 \tmp
595	lasx_init_upper	$xr31 \tmp
596	.endm
597
598.macro not dst src
599	nor	\dst, \src, zero
600.endm
601
602.macro la_abs reg, sym
603#ifndef CONFIG_RELOCATABLE
604	la.abs	\reg, \sym
605#else
606	766:
607	lu12i.w	\reg, 0
608	ori	\reg, \reg, 0
609	lu32i.d	\reg, 0
610	lu52i.d	\reg, \reg, 0
611	.pushsection ".la_abs", "aw", %progbits
612	.dword	766b
613	.dword	\sym
614	.popsection
615#endif
616.endm
617
618#endif /* _ASM_ASMMACRO_H */
619