1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_CMPXCHG_32_H
3#define _ASM_X86_CMPXCHG_32_H
4
5/*
6 * Note: if you use set64_bit(), __cmpxchg64(), or their variants,
7 *       you need to test for the feature in boot_cpu_data.
8 */
9
10#ifdef CONFIG_X86_CMPXCHG64
11#define arch_cmpxchg64(ptr, o, n)					\
12	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
13					 (unsigned long long)(n)))
14#define arch_cmpxchg64_local(ptr, o, n)					\
15	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
16					       (unsigned long long)(n)))
17#define arch_try_cmpxchg64(ptr, po, n)					\
18	__try_cmpxchg64((ptr), (unsigned long long *)(po), \
19			(unsigned long long)(n))
20#endif
21
22static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
23{
24	u64 prev;
25	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
26		     : "=A" (prev),
27		       "+m" (*ptr)
28		     : "b" ((u32)new),
29		       "c" ((u32)(new >> 32)),
30		       "0" (old)
31		     : "memory");
32	return prev;
33}
34
35static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
36{
37	u64 prev;
38	asm volatile("cmpxchg8b %1"
39		     : "=A" (prev),
40		       "+m" (*ptr)
41		     : "b" ((u32)new),
42		       "c" ((u32)(new >> 32)),
43		       "0" (old)
44		     : "memory");
45	return prev;
46}
47
48static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new)
49{
50	bool success;
51	u64 old = *pold;
52	asm volatile(LOCK_PREFIX "cmpxchg8b %[ptr]"
53		     CC_SET(z)
54		     : CC_OUT(z) (success),
55		       [ptr] "+m" (*ptr),
56		       "+A" (old)
57		     : "b" ((u32)new),
58		       "c" ((u32)(new >> 32))
59		     : "memory");
60
61	if (unlikely(!success))
62		*pold = old;
63	return success;
64}
65
66#ifndef CONFIG_X86_CMPXCHG64
67/*
68 * Building a kernel capable running on 80386 and 80486. It may be necessary
69 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
70 */
71
72#define arch_cmpxchg64(ptr, o, n)				\
73({								\
74	__typeof__(*(ptr)) __ret;				\
75	__typeof__(*(ptr)) __old = (o);				\
76	__typeof__(*(ptr)) __new = (n);				\
77	alternative_io(LOCK_PREFIX_HERE				\
78			"call cmpxchg8b_emu",			\
79			"lock; cmpxchg8b (%%esi)" ,		\
80		       X86_FEATURE_CX8,				\
81		       "=A" (__ret),				\
82		       "S" ((ptr)), "0" (__old),		\
83		       "b" ((unsigned int)__new),		\
84		       "c" ((unsigned int)(__new>>32))		\
85		       : "memory");				\
86	__ret; })
87
88
89#define arch_cmpxchg64_local(ptr, o, n)				\
90({								\
91	__typeof__(*(ptr)) __ret;				\
92	__typeof__(*(ptr)) __old = (o);				\
93	__typeof__(*(ptr)) __new = (n);				\
94	alternative_io("call cmpxchg8b_emu",			\
95		       "cmpxchg8b (%%esi)" ,			\
96		       X86_FEATURE_CX8,				\
97		       "=A" (__ret),				\
98		       "S" ((ptr)), "0" (__old),		\
99		       "b" ((unsigned int)__new),		\
100		       "c" ((unsigned int)(__new>>32))		\
101		       : "memory");				\
102	__ret; })
103
104#endif
105
106#define system_has_cmpxchg64()		boot_cpu_has(X86_FEATURE_CX8)
107
108#endif /* _ASM_X86_CMPXCHG_32_H */
109