1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_POWERPC_CMPXCHG_H_
3#define _ASM_POWERPC_CMPXCHG_H_
4
5#ifdef __KERNEL__
6#include <linux/compiler.h>
7#include <asm/synch.h>
8#include <linux/bug.h>
9
10#ifdef __BIG_ENDIAN
11#define BITOFF_CAL(size, off)	((sizeof(u32) - size - off) * BITS_PER_BYTE)
12#else
13#define BITOFF_CAL(size, off)	(off * BITS_PER_BYTE)
14#endif
15
16#define XCHG_GEN(type, sfx, cl)				\
17static inline u32 __xchg_##type##sfx(volatile void *p, u32 val)	\
18{								\
19	unsigned int prev, prev_mask, tmp, bitoff, off;		\
20								\
21	off = (unsigned long)p % sizeof(u32);			\
22	bitoff = BITOFF_CAL(sizeof(type), off);			\
23	p -= off;						\
24	val <<= bitoff;						\
25	prev_mask = (u32)(type)-1 << bitoff;			\
26								\
27	__asm__ __volatile__(					\
28"1:	lwarx   %0,0,%3\n"					\
29"	andc	%1,%0,%5\n"					\
30"	or	%1,%1,%4\n"					\
31"	stwcx.	%1,0,%3\n"					\
32"	bne-	1b\n"						\
33	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
34	: "r" (p), "r" (val), "r" (prev_mask)			\
35	: "cc", cl);						\
36								\
37	return prev >> bitoff;					\
38}
39
40#define CMPXCHG_GEN(type, sfx, br, br2, cl)			\
41static inline							\
42u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new)	\
43{								\
44	unsigned int prev, prev_mask, tmp, bitoff, off;		\
45								\
46	off = (unsigned long)p % sizeof(u32);			\
47	bitoff = BITOFF_CAL(sizeof(type), off);			\
48	p -= off;						\
49	old <<= bitoff;						\
50	new <<= bitoff;						\
51	prev_mask = (u32)(type)-1 << bitoff;			\
52								\
53	__asm__ __volatile__(					\
54	br							\
55"1:	lwarx   %0,0,%3\n"					\
56"	and	%1,%0,%6\n"					\
57"	cmpw	0,%1,%4\n"					\
58"	bne-	2f\n"						\
59"	andc	%1,%0,%6\n"					\
60"	or	%1,%1,%5\n"					\
61"	stwcx.  %1,0,%3\n"					\
62"	bne-    1b\n"						\
63	br2							\
64	"\n"							\
65"2:"								\
66	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
67	: "r" (p), "r" (old), "r" (new), "r" (prev_mask)	\
68	: "cc", cl);						\
69								\
70	return prev >> bitoff;					\
71}
72
73/*
74 * Atomic exchange
75 *
76 * Changes the memory location '*p' to be val and returns
77 * the previous value stored there.
78 */
79
80#ifndef CONFIG_PPC_HAS_LBARX_LHARX
81XCHG_GEN(u8, _local, "memory");
82XCHG_GEN(u8, _relaxed, "cc");
83XCHG_GEN(u16, _local, "memory");
84XCHG_GEN(u16, _relaxed, "cc");
85#else
86static __always_inline unsigned long
87__xchg_u8_local(volatile void *p, unsigned long val)
88{
89	unsigned long prev;
90
91	__asm__ __volatile__(
92"1:	lbarx	%0,0,%2		# __xchg_u8_local\n"
93"	stbcx.	%3,0,%2 \n"
94"	bne-	1b"
95	: "=&r" (prev), "+m" (*(volatile unsigned char *)p)
96	: "r" (p), "r" (val)
97	: "cc", "memory");
98
99	return prev;
100}
101
102static __always_inline unsigned long
103__xchg_u8_relaxed(u8 *p, unsigned long val)
104{
105	unsigned long prev;
106
107	__asm__ __volatile__(
108"1:	lbarx	%0,0,%2		# __xchg_u8_relaxed\n"
109"	stbcx.	%3,0,%2\n"
110"	bne-	1b"
111	: "=&r" (prev), "+m" (*p)
112	: "r" (p), "r" (val)
113	: "cc");
114
115	return prev;
116}
117
118static __always_inline unsigned long
119__xchg_u16_local(volatile void *p, unsigned long val)
120{
121	unsigned long prev;
122
123	__asm__ __volatile__(
124"1:	lharx	%0,0,%2		# __xchg_u16_local\n"
125"	sthcx.	%3,0,%2\n"
126"	bne-	1b"
127	: "=&r" (prev), "+m" (*(volatile unsigned short *)p)
128	: "r" (p), "r" (val)
129	: "cc", "memory");
130
131	return prev;
132}
133
134static __always_inline unsigned long
135__xchg_u16_relaxed(u16 *p, unsigned long val)
136{
137	unsigned long prev;
138
139	__asm__ __volatile__(
140"1:	lharx	%0,0,%2		# __xchg_u16_relaxed\n"
141"	sthcx.	%3,0,%2\n"
142"	bne-	1b"
143	: "=&r" (prev), "+m" (*p)
144	: "r" (p), "r" (val)
145	: "cc");
146
147	return prev;
148}
149#endif
150
151static __always_inline unsigned long
152__xchg_u32_local(volatile void *p, unsigned long val)
153{
154	unsigned long prev;
155
156	__asm__ __volatile__(
157"1:	lwarx	%0,0,%2 \n"
158"	stwcx.	%3,0,%2 \n\
159	bne-	1b"
160	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
161	: "r" (p), "r" (val)
162	: "cc", "memory");
163
164	return prev;
165}
166
167static __always_inline unsigned long
168__xchg_u32_relaxed(u32 *p, unsigned long val)
169{
170	unsigned long prev;
171
172	__asm__ __volatile__(
173"1:	lwarx	%0,0,%2\n"
174"	stwcx.	%3,0,%2\n"
175"	bne-	1b"
176	: "=&r" (prev), "+m" (*p)
177	: "r" (p), "r" (val)
178	: "cc");
179
180	return prev;
181}
182
183#ifdef CONFIG_PPC64
184static __always_inline unsigned long
185__xchg_u64_local(volatile void *p, unsigned long val)
186{
187	unsigned long prev;
188
189	__asm__ __volatile__(
190"1:	ldarx	%0,0,%2 \n"
191"	stdcx.	%3,0,%2 \n\
192	bne-	1b"
193	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
194	: "r" (p), "r" (val)
195	: "cc", "memory");
196
197	return prev;
198}
199
200static __always_inline unsigned long
201__xchg_u64_relaxed(u64 *p, unsigned long val)
202{
203	unsigned long prev;
204
205	__asm__ __volatile__(
206"1:	ldarx	%0,0,%2\n"
207"	stdcx.	%3,0,%2\n"
208"	bne-	1b"
209	: "=&r" (prev), "+m" (*p)
210	: "r" (p), "r" (val)
211	: "cc");
212
213	return prev;
214}
215#endif
216
217static __always_inline unsigned long
218__xchg_local(void *ptr, unsigned long x, unsigned int size)
219{
220	switch (size) {
221	case 1:
222		return __xchg_u8_local(ptr, x);
223	case 2:
224		return __xchg_u16_local(ptr, x);
225	case 4:
226		return __xchg_u32_local(ptr, x);
227#ifdef CONFIG_PPC64
228	case 8:
229		return __xchg_u64_local(ptr, x);
230#endif
231	}
232	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
233	return x;
234}
235
236static __always_inline unsigned long
237__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
238{
239	switch (size) {
240	case 1:
241		return __xchg_u8_relaxed(ptr, x);
242	case 2:
243		return __xchg_u16_relaxed(ptr, x);
244	case 4:
245		return __xchg_u32_relaxed(ptr, x);
246#ifdef CONFIG_PPC64
247	case 8:
248		return __xchg_u64_relaxed(ptr, x);
249#endif
250	}
251	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed");
252	return x;
253}
254#define arch_xchg_local(ptr,x)						     \
255  ({									     \
256     __typeof__(*(ptr)) _x_ = (x);					     \
257     (__typeof__(*(ptr))) __xchg_local((ptr),				     \
258     		(unsigned long)_x_, sizeof(*(ptr))); 			     \
259  })
260
261#define arch_xchg_relaxed(ptr, x)					\
262({									\
263	__typeof__(*(ptr)) _x_ = (x);					\
264	(__typeof__(*(ptr))) __xchg_relaxed((ptr),			\
265			(unsigned long)_x_, sizeof(*(ptr)));		\
266})
267
268/*
269 * Compare and exchange - if *p == old, set it to new,
270 * and return the old value of *p.
271 */
272#ifndef CONFIG_PPC_HAS_LBARX_LHARX
273CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
274CMPXCHG_GEN(u8, _local, , , "memory");
275CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
276CMPXCHG_GEN(u8, _relaxed, , , "cc");
277CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
278CMPXCHG_GEN(u16, _local, , , "memory");
279CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
280CMPXCHG_GEN(u16, _relaxed, , , "cc");
281#else
282static __always_inline unsigned long
283__cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)
284{
285	unsigned int prev;
286
287	__asm__ __volatile__ (
288	PPC_ATOMIC_ENTRY_BARRIER
289"1:	lbarx	%0,0,%2		# __cmpxchg_u8\n"
290"	cmpw	0,%0,%3\n"
291"	bne-	2f\n"
292"	stbcx.	%4,0,%2\n"
293"	bne-	1b"
294	PPC_ATOMIC_EXIT_BARRIER
295	"\n\
2962:"
297	: "=&r" (prev), "+m" (*p)
298	: "r" (p), "r" (old), "r" (new)
299	: "cc", "memory");
300
301	return prev;
302}
303
304static __always_inline unsigned long
305__cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,
306			unsigned long new)
307{
308	unsigned int prev;
309
310	__asm__ __volatile__ (
311"1:	lbarx	%0,0,%2		# __cmpxchg_u8_local\n"
312"	cmpw	0,%0,%3\n"
313"	bne-	2f\n"
314"	stbcx.	%4,0,%2\n"
315"	bne-	1b\n"
316"2:"
317	: "=&r" (prev), "+m" (*p)
318	: "r" (p), "r" (old), "r" (new)
319	: "cc", "memory");
320
321	return prev;
322}
323
324static __always_inline unsigned long
325__cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)
326{
327	unsigned long prev;
328
329	__asm__ __volatile__ (
330"1:	lbarx	%0,0,%2		# __cmpxchg_u8_relaxed\n"
331"	cmpw	0,%0,%3\n"
332"	bne-	2f\n"
333"	stbcx.	%4,0,%2\n"
334"	bne-	1b\n"
335"2:"
336	: "=&r" (prev), "+m" (*p)
337	: "r" (p), "r" (old), "r" (new)
338	: "cc");
339
340	return prev;
341}
342
343static __always_inline unsigned long
344__cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)
345{
346	unsigned long prev;
347
348	__asm__ __volatile__ (
349"1:	lbarx	%0,0,%2		# __cmpxchg_u8_acquire\n"
350"	cmpw	0,%0,%3\n"
351"	bne-	2f\n"
352"	stbcx.	%4,0,%2\n"
353"	bne-	1b\n"
354	PPC_ACQUIRE_BARRIER
355"2:"
356	: "=&r" (prev), "+m" (*p)
357	: "r" (p), "r" (old), "r" (new)
358	: "cc", "memory");
359
360	return prev;
361}
362
363static __always_inline unsigned long
364__cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)
365{
366	unsigned int prev;
367
368	__asm__ __volatile__ (
369	PPC_ATOMIC_ENTRY_BARRIER
370"1:	lharx	%0,0,%2		# __cmpxchg_u16\n"
371"	cmpw	0,%0,%3\n"
372"	bne-	2f\n"
373"	sthcx.	%4,0,%2\n"
374"	bne-	1b\n"
375	PPC_ATOMIC_EXIT_BARRIER
376"2:"
377	: "=&r" (prev), "+m" (*p)
378	: "r" (p), "r" (old), "r" (new)
379	: "cc", "memory");
380
381	return prev;
382}
383
384static __always_inline unsigned long
385__cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,
386			unsigned long new)
387{
388	unsigned int prev;
389
390	__asm__ __volatile__ (
391"1:	lharx	%0,0,%2		# __cmpxchg_u16_local\n"
392"	cmpw	0,%0,%3\n"
393"	bne-	2f\n"
394"	sthcx.	%4,0,%2\n"
395"	bne-	1b"
396"2:"
397	: "=&r" (prev), "+m" (*p)
398	: "r" (p), "r" (old), "r" (new)
399	: "cc", "memory");
400
401	return prev;
402}
403
404static __always_inline unsigned long
405__cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)
406{
407	unsigned long prev;
408
409	__asm__ __volatile__ (
410"1:	lharx	%0,0,%2		# __cmpxchg_u16_relaxed\n"
411"	cmpw	0,%0,%3\n"
412"	bne-	2f\n"
413"	sthcx.	%4,0,%2\n"
414"	bne-	1b\n"
415"2:"
416	: "=&r" (prev), "+m" (*p)
417	: "r" (p), "r" (old), "r" (new)
418	: "cc");
419
420	return prev;
421}
422
423static __always_inline unsigned long
424__cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)
425{
426	unsigned long prev;
427
428	__asm__ __volatile__ (
429"1:	lharx	%0,0,%2		# __cmpxchg_u16_acquire\n"
430"	cmpw	0,%0,%3\n"
431"	bne-	2f\n"
432"	sthcx.	%4,0,%2\n"
433"	bne-	1b\n"
434	PPC_ACQUIRE_BARRIER
435"2:"
436	: "=&r" (prev), "+m" (*p)
437	: "r" (p), "r" (old), "r" (new)
438	: "cc", "memory");
439
440	return prev;
441}
442#endif
443
444static __always_inline unsigned long
445__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
446{
447	unsigned int prev;
448
449	__asm__ __volatile__ (
450	PPC_ATOMIC_ENTRY_BARRIER
451"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
452	cmpw	0,%0,%3\n\
453	bne-	2f\n"
454"	stwcx.	%4,0,%2\n\
455	bne-	1b"
456	PPC_ATOMIC_EXIT_BARRIER
457	"\n\
4582:"
459	: "=&r" (prev), "+m" (*p)
460	: "r" (p), "r" (old), "r" (new)
461	: "cc", "memory");
462
463	return prev;
464}
465
466static __always_inline unsigned long
467__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
468			unsigned long new)
469{
470	unsigned int prev;
471
472	__asm__ __volatile__ (
473"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
474	cmpw	0,%0,%3\n\
475	bne-	2f\n"
476"	stwcx.	%4,0,%2\n\
477	bne-	1b"
478	"\n\
4792:"
480	: "=&r" (prev), "+m" (*p)
481	: "r" (p), "r" (old), "r" (new)
482	: "cc", "memory");
483
484	return prev;
485}
486
487static __always_inline unsigned long
488__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
489{
490	unsigned long prev;
491
492	__asm__ __volatile__ (
493"1:	lwarx	%0,0,%2		# __cmpxchg_u32_relaxed\n"
494"	cmpw	0,%0,%3\n"
495"	bne-	2f\n"
496"	stwcx.	%4,0,%2\n"
497"	bne-	1b\n"
498"2:"
499	: "=&r" (prev), "+m" (*p)
500	: "r" (p), "r" (old), "r" (new)
501	: "cc");
502
503	return prev;
504}
505
506/*
507 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
508 * can avoid superfluous barriers if we use assembly code to implement
509 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
510 * cmpxchg_release() because that will result in putting a barrier in the
511 * middle of a ll/sc loop, which is probably a bad idea. For example, this
512 * might cause the conditional store more likely to fail.
513 */
514static __always_inline unsigned long
515__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
516{
517	unsigned long prev;
518
519	__asm__ __volatile__ (
520"1:	lwarx	%0,0,%2		# __cmpxchg_u32_acquire\n"
521"	cmpw	0,%0,%3\n"
522"	bne-	2f\n"
523"	stwcx.	%4,0,%2\n"
524"	bne-	1b\n"
525	PPC_ACQUIRE_BARRIER
526	"\n"
527"2:"
528	: "=&r" (prev), "+m" (*p)
529	: "r" (p), "r" (old), "r" (new)
530	: "cc", "memory");
531
532	return prev;
533}
534
535#ifdef CONFIG_PPC64
536static __always_inline unsigned long
537__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
538{
539	unsigned long prev;
540
541	__asm__ __volatile__ (
542	PPC_ATOMIC_ENTRY_BARRIER
543"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
544	cmpd	0,%0,%3\n\
545	bne-	2f\n\
546	stdcx.	%4,0,%2\n\
547	bne-	1b"
548	PPC_ATOMIC_EXIT_BARRIER
549	"\n\
5502:"
551	: "=&r" (prev), "+m" (*p)
552	: "r" (p), "r" (old), "r" (new)
553	: "cc", "memory");
554
555	return prev;
556}
557
558static __always_inline unsigned long
559__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
560			unsigned long new)
561{
562	unsigned long prev;
563
564	__asm__ __volatile__ (
565"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
566	cmpd	0,%0,%3\n\
567	bne-	2f\n\
568	stdcx.	%4,0,%2\n\
569	bne-	1b"
570	"\n\
5712:"
572	: "=&r" (prev), "+m" (*p)
573	: "r" (p), "r" (old), "r" (new)
574	: "cc", "memory");
575
576	return prev;
577}
578
579static __always_inline unsigned long
580__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
581{
582	unsigned long prev;
583
584	__asm__ __volatile__ (
585"1:	ldarx	%0,0,%2		# __cmpxchg_u64_relaxed\n"
586"	cmpd	0,%0,%3\n"
587"	bne-	2f\n"
588"	stdcx.	%4,0,%2\n"
589"	bne-	1b\n"
590"2:"
591	: "=&r" (prev), "+m" (*p)
592	: "r" (p), "r" (old), "r" (new)
593	: "cc");
594
595	return prev;
596}
597
598static __always_inline unsigned long
599__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
600{
601	unsigned long prev;
602
603	__asm__ __volatile__ (
604"1:	ldarx	%0,0,%2		# __cmpxchg_u64_acquire\n"
605"	cmpd	0,%0,%3\n"
606"	bne-	2f\n"
607"	stdcx.	%4,0,%2\n"
608"	bne-	1b\n"
609	PPC_ACQUIRE_BARRIER
610	"\n"
611"2:"
612	: "=&r" (prev), "+m" (*p)
613	: "r" (p), "r" (old), "r" (new)
614	: "cc", "memory");
615
616	return prev;
617}
618#endif
619
620static __always_inline unsigned long
621__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
622	  unsigned int size)
623{
624	switch (size) {
625	case 1:
626		return __cmpxchg_u8(ptr, old, new);
627	case 2:
628		return __cmpxchg_u16(ptr, old, new);
629	case 4:
630		return __cmpxchg_u32(ptr, old, new);
631#ifdef CONFIG_PPC64
632	case 8:
633		return __cmpxchg_u64(ptr, old, new);
634#endif
635	}
636	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
637	return old;
638}
639
640static __always_inline unsigned long
641__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
642	  unsigned int size)
643{
644	switch (size) {
645	case 1:
646		return __cmpxchg_u8_local(ptr, old, new);
647	case 2:
648		return __cmpxchg_u16_local(ptr, old, new);
649	case 4:
650		return __cmpxchg_u32_local(ptr, old, new);
651#ifdef CONFIG_PPC64
652	case 8:
653		return __cmpxchg_u64_local(ptr, old, new);
654#endif
655	}
656	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
657	return old;
658}
659
660static __always_inline unsigned long
661__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
662		  unsigned int size)
663{
664	switch (size) {
665	case 1:
666		return __cmpxchg_u8_relaxed(ptr, old, new);
667	case 2:
668		return __cmpxchg_u16_relaxed(ptr, old, new);
669	case 4:
670		return __cmpxchg_u32_relaxed(ptr, old, new);
671#ifdef CONFIG_PPC64
672	case 8:
673		return __cmpxchg_u64_relaxed(ptr, old, new);
674#endif
675	}
676	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
677	return old;
678}
679
680static __always_inline unsigned long
681__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
682		  unsigned int size)
683{
684	switch (size) {
685	case 1:
686		return __cmpxchg_u8_acquire(ptr, old, new);
687	case 2:
688		return __cmpxchg_u16_acquire(ptr, old, new);
689	case 4:
690		return __cmpxchg_u32_acquire(ptr, old, new);
691#ifdef CONFIG_PPC64
692	case 8:
693		return __cmpxchg_u64_acquire(ptr, old, new);
694#endif
695	}
696	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
697	return old;
698}
699#define arch_cmpxchg(ptr, o, n)						 \
700  ({									 \
701     __typeof__(*(ptr)) _o_ = (o);					 \
702     __typeof__(*(ptr)) _n_ = (n);					 \
703     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,		 \
704				    (unsigned long)_n_, sizeof(*(ptr))); \
705  })
706
707
708#define arch_cmpxchg_local(ptr, o, n)					 \
709  ({									 \
710     __typeof__(*(ptr)) _o_ = (o);					 \
711     __typeof__(*(ptr)) _n_ = (n);					 \
712     (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,	 \
713				    (unsigned long)_n_, sizeof(*(ptr))); \
714  })
715
716#define arch_cmpxchg_relaxed(ptr, o, n)					\
717({									\
718	__typeof__(*(ptr)) _o_ = (o);					\
719	__typeof__(*(ptr)) _n_ = (n);					\
720	(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),			\
721			(unsigned long)_o_, (unsigned long)_n_,		\
722			sizeof(*(ptr)));				\
723})
724
725#define arch_cmpxchg_acquire(ptr, o, n)					\
726({									\
727	__typeof__(*(ptr)) _o_ = (o);					\
728	__typeof__(*(ptr)) _n_ = (n);					\
729	(__typeof__(*(ptr))) __cmpxchg_acquire((ptr),			\
730			(unsigned long)_o_, (unsigned long)_n_,		\
731			sizeof(*(ptr)));				\
732})
733#ifdef CONFIG_PPC64
734#define arch_cmpxchg64(ptr, o, n)					\
735  ({									\
736	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
737	arch_cmpxchg((ptr), (o), (n));					\
738  })
739#define arch_cmpxchg64_local(ptr, o, n)					\
740  ({									\
741	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
742	arch_cmpxchg_local((ptr), (o), (n));				\
743  })
744#define arch_cmpxchg64_relaxed(ptr, o, n)				\
745({									\
746	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
747	arch_cmpxchg_relaxed((ptr), (o), (n));				\
748})
749#define arch_cmpxchg64_acquire(ptr, o, n)				\
750({									\
751	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
752	arch_cmpxchg_acquire((ptr), (o), (n));				\
753})
754#else
755#include <asm-generic/cmpxchg-local.h>
756#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
757#endif
758
759#endif /* __KERNEL__ */
760#endif /* _ASM_POWERPC_CMPXCHG_H_ */
761