1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3#include "rseq-bits-template.h"
4
5#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7
8static inline __always_inline
9int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
10{
11	RSEQ_INJECT_C(9)
12
13	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
14				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
15#ifdef RSEQ_COMPARE_TWICE
16				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
17				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
18#endif
19				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
20				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
21				  RSEQ_INJECT_ASM(3)
22				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
23				  RSEQ_INJECT_ASM(4)
24#ifdef RSEQ_COMPARE_TWICE
25				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
26				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
27#endif
28				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
29				  RSEQ_INJECT_ASM(5)
30				  RSEQ_ASM_DEFINE_ABORT(4, abort)
31				  : /* gcc asm goto does not allow outputs */
32				  : [cpu_id]		"r" (cpu),
33				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
34				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
35				    [v]			"m" (*v),
36				    [expect]		"r" (expect),
37				    [newv]		"r" (newv)
38				    RSEQ_INJECT_INPUT
39				  : "memory", RSEQ_ASM_TMP_REG_1
40				    RSEQ_INJECT_CLOBBER
41				  : abort, cmpfail
42#ifdef RSEQ_COMPARE_TWICE
43				    , error1, error2
44#endif
45	);
46
47	return 0;
48abort:
49	RSEQ_INJECT_FAILED
50	return -1;
51cmpfail:
52	return 1;
53#ifdef RSEQ_COMPARE_TWICE
54error1:
55	rseq_bug("cpu_id comparison failed");
56error2:
57	rseq_bug("expected value comparison failed");
58#endif
59}
60
61static inline __always_inline
62int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
63			       off_t voffp, intptr_t *load, int cpu)
64{
65	RSEQ_INJECT_C(9)
66
67	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
68				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
69#ifdef RSEQ_COMPARE_TWICE
70				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
71				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
72#endif
73				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
74				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
75				  RSEQ_INJECT_ASM(3)
76				  RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
77				  RSEQ_INJECT_ASM(4)
78#ifdef RSEQ_COMPARE_TWICE
79				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
80				  RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
81#endif
82				  RSEQ_ASM_OP_R_LOAD(v)
83				  RSEQ_ASM_OP_R_STORE(load)
84				  RSEQ_ASM_OP_R_LOAD_OFF(voffp)
85				  RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
86				  RSEQ_INJECT_ASM(5)
87				  RSEQ_ASM_DEFINE_ABORT(4, abort)
88				  : /* gcc asm goto does not allow outputs */
89				  : [cpu_id]		"r" (cpu),
90				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
91				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
92				    [v]			"m" (*v),
93				    [expectnot]		"r" (expectnot),
94				    [load]		"m" (*load),
95				    [voffp]		"r" (voffp)
96				    RSEQ_INJECT_INPUT
97				  : "memory", RSEQ_ASM_TMP_REG_1
98				    RSEQ_INJECT_CLOBBER
99				  : abort, cmpfail
100#ifdef RSEQ_COMPARE_TWICE
101				    , error1, error2
102#endif
103	);
104	return 0;
105abort:
106	RSEQ_INJECT_FAILED
107	return -1;
108cmpfail:
109	return 1;
110#ifdef RSEQ_COMPARE_TWICE
111error1:
112	rseq_bug("cpu_id comparison failed");
113error2:
114	rseq_bug("expected value comparison failed");
115#endif
116}
117
118static inline __always_inline
119int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
120{
121	RSEQ_INJECT_C(9)
122
123	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
124#ifdef RSEQ_COMPARE_TWICE
125				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
126#endif
127				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
128				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
129				  RSEQ_INJECT_ASM(3)
130#ifdef RSEQ_COMPARE_TWICE
131				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
132#endif
133				  RSEQ_ASM_OP_R_LOAD(v)
134				  RSEQ_ASM_OP_R_ADD(count)
135				  RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
136				  RSEQ_INJECT_ASM(4)
137				  RSEQ_ASM_DEFINE_ABORT(4, abort)
138				  : /* gcc asm goto does not allow outputs */
139				  : [cpu_id]		"r" (cpu),
140				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
141				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
142				    [v]			"m" (*v),
143				    [count]		"r" (count)
144				    RSEQ_INJECT_INPUT
145				  : "memory", RSEQ_ASM_TMP_REG_1
146				    RSEQ_INJECT_CLOBBER
147				  : abort
148#ifdef RSEQ_COMPARE_TWICE
149				    , error1
150#endif
151	);
152	return 0;
153abort:
154	RSEQ_INJECT_FAILED
155	return -1;
156#ifdef RSEQ_COMPARE_TWICE
157error1:
158	rseq_bug("cpu_id comparison failed");
159#endif
160}
161
162static inline __always_inline
163int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
164			      intptr_t *v2, intptr_t expect2,
165			      intptr_t newv, int cpu)
166{
167	RSEQ_INJECT_C(9)
168
169	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
170				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
171#ifdef RSEQ_COMPARE_TWICE
172				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
173				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
174				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
175#endif
176				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
177				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
178				  RSEQ_INJECT_ASM(3)
179				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
180				  RSEQ_INJECT_ASM(4)
181				  RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
182				  RSEQ_INJECT_ASM(5)
183#ifdef RSEQ_COMPARE_TWICE
184				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
185				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
186				  RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
187#endif
188				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
189				  RSEQ_INJECT_ASM(6)
190				  RSEQ_ASM_DEFINE_ABORT(4, abort)
191				  : /* gcc asm goto does not allow outputs */
192				  : [cpu_id]		"r" (cpu),
193				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
194				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
195				    [v]			"m" (*v),
196				    [expect]		"r" (expect),
197				    [v2]			"m" (*v2),
198				    [expect2]		"r" (expect2),
199				    [newv]		"r" (newv)
200				    RSEQ_INJECT_INPUT
201				  : "memory", RSEQ_ASM_TMP_REG_1
202				    RSEQ_INJECT_CLOBBER
203				  : abort, cmpfail
204#ifdef RSEQ_COMPARE_TWICE
205				    , error1, error2, error3
206#endif
207	);
208
209	return 0;
210abort:
211	RSEQ_INJECT_FAILED
212	return -1;
213cmpfail:
214	return 1;
215#ifdef RSEQ_COMPARE_TWICE
216error1:
217	rseq_bug("cpu_id comparison failed");
218error2:
219	rseq_bug("expected value comparison failed");
220error3:
221	rseq_bug("2nd expected value comparison failed");
222#endif
223}
224
225#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
226
227/*
228 *   pval = *(ptr+off)
229 *  *pval += inc;
230 */
231static inline __always_inline
232int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
233{
234	RSEQ_INJECT_C(9)
235
236	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
237#ifdef RSEQ_COMPARE_TWICE
238				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
239#endif
240				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
241				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
242				  RSEQ_INJECT_ASM(3)
243#ifdef RSEQ_COMPARE_TWICE
244				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
245#endif
246				  RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
247				  RSEQ_INJECT_ASM(4)
248				  RSEQ_ASM_DEFINE_ABORT(4, abort)
249				  : /* gcc asm goto does not allow outputs */
250				  : [cpu_id]			"r" (cpu),
251				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
252				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
253				    [ptr]			"r" (ptr),
254				    [off]			"er" (off),
255				    [inc]			"er" (inc)
256				    RSEQ_INJECT_INPUT
257				  : "memory", RSEQ_ASM_TMP_REG_1
258				    RSEQ_INJECT_CLOBBER
259				  : abort
260#ifdef RSEQ_COMPARE_TWICE
261				    , error1
262#endif
263	);
264	return 0;
265abort:
266	RSEQ_INJECT_FAILED
267	return -1;
268#ifdef RSEQ_COMPARE_TWICE
269error1:
270	rseq_bug("cpu_id comparison failed");
271#endif
272}
273
274#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
275	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
276
277#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
278	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
279
280static inline __always_inline
281int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
282				 intptr_t *v2, intptr_t newv2,
283				 intptr_t newv, int cpu)
284{
285	RSEQ_INJECT_C(9)
286
287	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
288				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
289#ifdef RSEQ_COMPARE_TWICE
290				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
291				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
292#endif
293				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
294				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
295				  RSEQ_INJECT_ASM(3)
296				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
297				  RSEQ_INJECT_ASM(4)
298#ifdef RSEQ_COMPARE_TWICE
299				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
300				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
301#endif
302				  RSEQ_ASM_OP_STORE(newv2, v2)
303				  RSEQ_INJECT_ASM(5)
304#ifdef RSEQ_TEMPLATE_MO_RELEASE
305				  RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
306#else
307				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
308#endif
309				  RSEQ_INJECT_ASM(6)
310				  RSEQ_ASM_DEFINE_ABORT(4, abort)
311				  : /* gcc asm goto does not allow outputs */
312				  : [cpu_id]			"r" (cpu),
313				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
314				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
315				    [expect]			"r" (expect),
316				    [v]				"m" (*v),
317				    [newv]			"r" (newv),
318				    [v2]			"m" (*v2),
319				    [newv2]			"r" (newv2)
320				    RSEQ_INJECT_INPUT
321				  : "memory", RSEQ_ASM_TMP_REG_1
322				    RSEQ_INJECT_CLOBBER
323				  : abort, cmpfail
324#ifdef RSEQ_COMPARE_TWICE
325				    , error1, error2
326#endif
327	);
328
329	return 0;
330abort:
331	RSEQ_INJECT_FAILED
332	return -1;
333cmpfail:
334	return 1;
335#ifdef RSEQ_COMPARE_TWICE
336error1:
337	rseq_bug("cpu_id comparison failed");
338error2:
339	rseq_bug("expected value comparison failed");
340#endif
341}
342
343static inline __always_inline
344int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
345				 void *dst, void *src, size_t len,
346				 intptr_t newv, int cpu)
347{
348	RSEQ_INJECT_C(9)
349	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
350				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
351#ifdef RSEQ_COMPARE_TWICE
352				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
353				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
354#endif
355				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
356				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
357				  RSEQ_INJECT_ASM(3)
358				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
359				  RSEQ_INJECT_ASM(4)
360#ifdef RSEQ_COMPARE_TWICE
361				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
362				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
363#endif
364				  RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
365				  RSEQ_INJECT_ASM(5)
366#ifdef RSEQ_TEMPLATE_MO_RELEASE
367				  RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
368#else
369				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
370#endif
371				  RSEQ_INJECT_ASM(6)
372				  RSEQ_ASM_DEFINE_ABORT(4, abort)
373				  : /* gcc asm goto does not allow outputs */
374				  : [cpu_id]			"r" (cpu),
375				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
376				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
377				    [expect]			"r" (expect),
378				    [v]				"m" (*v),
379				    [newv]			"r" (newv),
380				    [dst]			"r" (dst),
381				    [src]			"r" (src),
382				    [len]			"r" (len)
383				    RSEQ_INJECT_INPUT
384				  : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
385				    RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
386				    RSEQ_INJECT_CLOBBER
387				  : abort, cmpfail
388#ifdef RSEQ_COMPARE_TWICE
389				    , error1, error2
390#endif
391	);
392
393	return 0;
394abort:
395	RSEQ_INJECT_FAILED
396	return -1;
397cmpfail:
398	return 1;
399#ifdef RSEQ_COMPARE_TWICE
400error1:
401	rseq_bug("cpu_id comparison failed");
402error2:
403	rseq_bug("expected value comparison failed");
404#endif
405}
406
407#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
408	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
409
410#include "rseq-bits-reset.h"
411