include/asm/percpu.h

21 #define PER_CPU_VAR(var)	__percpu(var)__percpu_rel
24 #define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
26 #define INIT_PER_CPU_VAR(var)  var
93 #define PER_CPU_VAR(var)	%__percpu_seg:(var)__percpu_rel
100 #define PER_CPU_VAR(var)	(var)__percpu_rel
104 #define __my_cpu_type(var)	typeof(var) __percpu_seg_override
106 #define __my_cpu_var(var)	(*__my_cpu_ptr(&var))
117 #define DECLARE_INIT_PER_CPU(var) \
118        extern typeof(var) init_per_cpu_var(var)
121 #define init_per_cpu_var(var)  init_per_cpu__##var
123 #define init_per_cpu_var(var)  var
167 	asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var]))	\
168 	    : [var] "+m" (__my_cpu_var(_var))				\
174 	asm qual (__pcpu_op1_##size(op, __percpu_arg([var]))		\
175 	    : [var] "+m" (__my_cpu_var(_var)));				\
182 #define percpu_add_op(size, qual, var, val)				\
188 		typeof(var) pao_tmp__;					\
193 		percpu_unary_op(size, qual, "inc", var);		\
195 		percpu_unary_op(size, qual, "dec", var);		\
197 		percpu_to_op(size, qual, "add", var, val);		\
203 	asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]")	\
205 	    : [var] "m" (__my_cpu_var(_var)));				\
212 	asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]")	\
214 	    : [var] "i" (&(_var)));					\
225 				     __percpu_arg([var]))		\
227 		    [var] "+m" (__my_cpu_var(_var))			\
241 	asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]),		\
245 				    __percpu_arg([var]))		\
248 		    [var] "+m" (__my_cpu_var(_var))			\
263 				    __percpu_arg([var]))		\
265 		    [var] "+m" (__my_cpu_var(_var))			\
278 				    __percpu_arg([var]))		\
282 		    [var] "+m" (__my_cpu_var(_var))			\
294 		u64 var;						\
300 	old__.var = _oval;						\
301 	new__.var = _nval;						\
304 			      "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
305 		  : [var] "+m" (__my_cpu_var(_var)),			\
313 	old__.var;							\
324 		u64 var;						\
330 	old__.var = *_oval;						\
331 	new__.var = _nval;						\
334 			      "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
337 		    [var] "+m" (__my_cpu_var(_var)),			\
345 		*_oval = old__.var;					\
363 		u128 var;						\
369 	old__.var = _oval;						\
370 	new__.var = _nval;						\
373 			      "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
374 		  : [var] "+m" (__my_cpu_var(_var)),			\
382 	old__.var;							\
393 		u128 var;						\
399 	old__.var = *_oval;						\
400 	new__.var = _nval;						\
403 			      "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
406 		    [var] "+m" (__my_cpu_var(_var)),			\
414 		*_oval = old__.var;					\
517 #define raw_percpu_xchg_op(var, nval)					\
519 	typeof(var) pxo_ret__ = raw_cpu_read(var);			\
520 	raw_cpu_write(var, (nval));					\