1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef __ASM_PERCPU_H
6#define __ASM_PERCPU_H
7
8#include <asm/cmpxchg.h>
9#include <asm/loongarch.h>
10
11/*
12 * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
13 * the loading address of main kernel image, but far from where the modules are
14 * loaded. Tell the compiler this fact when using explicit relocs.
15 */
16#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
17# if __has_attribute(model)
18#  define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
19# else
20#  error compiler support for the model attribute is necessary when a recent assembler is used
21# endif
22#endif
23
24/* Use r21 for fast access */
25register unsigned long __my_cpu_offset __asm__("$r21");
26
27static inline void set_my_cpu_offset(unsigned long off)
28{
29	__my_cpu_offset = off;
30	csr_write64(off, PERCPU_BASE_KS);
31}
32
33#define __my_cpu_offset					\
34({							\
35	__asm__ __volatile__("":"+r"(__my_cpu_offset));	\
36	__my_cpu_offset;				\
37})
38
39#define PERCPU_OP(op, asm_op, c_op)					\
40static __always_inline unsigned long __percpu_##op(void *ptr,		\
41			unsigned long val, int size)			\
42{									\
43	unsigned long ret;						\
44									\
45	switch (size) {							\
46	case 4:								\
47		__asm__ __volatile__(					\
48		"am"#asm_op".w"	" %[ret], %[val], %[ptr]	\n"	\
49		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr)		\
50		: [val] "r" (val));					\
51		break;							\
52	case 8:								\
53		__asm__ __volatile__(					\
54		"am"#asm_op".d" " %[ret], %[val], %[ptr]	\n"	\
55		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr)		\
56		: [val] "r" (val));					\
57		break;							\
58	default:							\
59		ret = 0;						\
60		BUILD_BUG();						\
61	}								\
62									\
63	return ret c_op val;						\
64}
65
66PERCPU_OP(add, add, +)
67PERCPU_OP(and, and, &)
68PERCPU_OP(or, or, |)
69#undef PERCPU_OP
70
71static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
72{
73	unsigned long ret;
74
75	switch (size) {
76	case 1:
77		__asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr]	\n"
78		: [ret] "=&r"(ret)
79		: [ptr] "r"(ptr)
80		: "memory");
81		break;
82	case 2:
83		__asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr]	\n"
84		: [ret] "=&r"(ret)
85		: [ptr] "r"(ptr)
86		: "memory");
87		break;
88	case 4:
89		__asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr]	\n"
90		: [ret] "=&r"(ret)
91		: [ptr] "r"(ptr)
92		: "memory");
93		break;
94	case 8:
95		__asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr]	\n"
96		: [ret] "=&r"(ret)
97		: [ptr] "r"(ptr)
98		: "memory");
99		break;
100	default:
101		ret = 0;
102		BUILD_BUG();
103	}
104
105	return ret;
106}
107
108static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
109{
110	switch (size) {
111	case 1:
112		__asm__ __volatile__("stx.b %[val], $r21, %[ptr]	\n"
113		:
114		: [val] "r" (val), [ptr] "r" (ptr)
115		: "memory");
116		break;
117	case 2:
118		__asm__ __volatile__("stx.h %[val], $r21, %[ptr]	\n"
119		:
120		: [val] "r" (val), [ptr] "r" (ptr)
121		: "memory");
122		break;
123	case 4:
124		__asm__ __volatile__("stx.w %[val], $r21, %[ptr]	\n"
125		:
126		: [val] "r" (val), [ptr] "r" (ptr)
127		: "memory");
128		break;
129	case 8:
130		__asm__ __volatile__("stx.d %[val], $r21, %[ptr]	\n"
131		:
132		: [val] "r" (val), [ptr] "r" (ptr)
133		: "memory");
134		break;
135	default:
136		BUILD_BUG();
137	}
138}
139
140static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
141{
142	switch (size) {
143	case 1:
144	case 2:
145		return __xchg_small((volatile void *)ptr, val, size);
146
147	case 4:
148		return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
149
150	case 8:
151		return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
152
153	default:
154		BUILD_BUG();
155	}
156
157	return 0;
158}
159
160/* this_cpu_cmpxchg */
161#define _protect_cmpxchg_local(pcp, o, n)			\
162({								\
163	typeof(*raw_cpu_ptr(&(pcp))) __ret;			\
164	preempt_disable_notrace();				\
165	__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);	\
166	preempt_enable_notrace();				\
167	__ret;							\
168})
169
170#define _percpu_read(pcp)						\
171({									\
172	typeof(pcp) __retval;						\
173	__retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));	\
174	__retval;							\
175})
176
177#define _percpu_write(pcp, val)						\
178do {									\
179	__percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp));	\
180} while (0)								\
181
182#define _pcp_protect(operation, pcp, val)			\
183({								\
184	typeof(pcp) __retval;					\
185	preempt_disable_notrace();				\
186	__retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),	\
187					  (val), sizeof(pcp));	\
188	preempt_enable_notrace();				\
189	__retval;						\
190})
191
192#define _percpu_add(pcp, val) \
193	_pcp_protect(__percpu_add, pcp, val)
194
195#define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
196
197#define _percpu_and(pcp, val) \
198	_pcp_protect(__percpu_and, pcp, val)
199
200#define _percpu_or(pcp, val) \
201	_pcp_protect(__percpu_or, pcp, val)
202
203#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
204	_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
205
206#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
207#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
208
209#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
210#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
211
212#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
213#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
214
215#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
216#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
217
218#define this_cpu_read_1(pcp) _percpu_read(pcp)
219#define this_cpu_read_2(pcp) _percpu_read(pcp)
220#define this_cpu_read_4(pcp) _percpu_read(pcp)
221#define this_cpu_read_8(pcp) _percpu_read(pcp)
222
223#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
224#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
225#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
226#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
227
228#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
229#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
230#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
231#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
232
233#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
234#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
235#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
236#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
237
238#include <asm-generic/percpu.h>
239
240#endif /* __ASM_PERCPU_H */
241