1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_CMPXCHG_64_H
3#define _ASM_X86_CMPXCHG_64_H
4
5#define arch_cmpxchg64(ptr, o, n)					\
6({									\
7	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
8	arch_cmpxchg((ptr), (o), (n));					\
9})
10
11#define arch_cmpxchg64_local(ptr, o, n)					\
12({									\
13	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
14	arch_cmpxchg_local((ptr), (o), (n));				\
15})
16
17#define arch_try_cmpxchg64(ptr, po, n)					\
18({									\
19	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
20	arch_try_cmpxchg((ptr), (po), (n));				\
21})
22
23union __u128_halves {
24	u128 full;
25	struct {
26		u64 low, high;
27	};
28};
29
30#define __arch_cmpxchg128(_ptr, _old, _new, _lock)			\
31({									\
32	union __u128_halves o = { .full = (_old), },			\
33			    n = { .full = (_new), };			\
34									\
35	asm volatile(_lock "cmpxchg16b %[ptr]"				\
36		     : [ptr] "+m" (*(_ptr)),				\
37		       "+a" (o.low), "+d" (o.high)			\
38		     : "b" (n.low), "c" (n.high)			\
39		     : "memory");					\
40									\
41	o.full;								\
42})
43
44static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
45{
46	return __arch_cmpxchg128(ptr, old, new, LOCK_PREFIX);
47}
48#define arch_cmpxchg128 arch_cmpxchg128
49
50static __always_inline u128 arch_cmpxchg128_local(volatile u128 *ptr, u128 old, u128 new)
51{
52	return __arch_cmpxchg128(ptr, old, new,);
53}
54#define arch_cmpxchg128_local arch_cmpxchg128_local
55
56#define __arch_try_cmpxchg128(_ptr, _oldp, _new, _lock)			\
57({									\
58	union __u128_halves o = { .full = *(_oldp), },			\
59			    n = { .full = (_new), };			\
60	bool ret;							\
61									\
62	asm volatile(_lock "cmpxchg16b %[ptr]"				\
63		     CC_SET(e)						\
64		     : CC_OUT(e) (ret),					\
65		       [ptr] "+m" (*ptr),				\
66		       "+a" (o.low), "+d" (o.high)			\
67		     : "b" (n.low), "c" (n.high)			\
68		     : "memory");					\
69									\
70	if (unlikely(!ret))						\
71		*(_oldp) = o.full;					\
72									\
73	likely(ret);							\
74})
75
76static __always_inline bool arch_try_cmpxchg128(volatile u128 *ptr, u128 *oldp, u128 new)
77{
78	return __arch_try_cmpxchg128(ptr, oldp, new, LOCK_PREFIX);
79}
80#define arch_try_cmpxchg128 arch_try_cmpxchg128
81
82static __always_inline bool arch_try_cmpxchg128_local(volatile u128 *ptr, u128 *oldp, u128 new)
83{
84	return __arch_try_cmpxchg128(ptr, oldp, new,);
85}
86#define arch_try_cmpxchg128_local arch_try_cmpxchg128_local
87
88#define system_has_cmpxchg128()		boot_cpu_has(X86_FEATURE_CX16)
89
90#endif /* _ASM_X86_CMPXCHG_64_H */
91