167117Sdfr/*
267117Sdfr * This file is subject to the terms and conditions of the GNU General Public
367117Sdfr * License.  See the file "COPYING" in the main directory of this archive
467117Sdfr * for more details.
567117Sdfr *
667117Sdfr * Copyright (c) 2006  Ralf Baechle (ralf@linux-mips.org)
767117Sdfr */
867117Sdfr#ifndef _ASM_FUTEX_H
967117Sdfr#define _ASM_FUTEX_H
1067117Sdfr
1167117Sdfr#ifdef __KERNEL__
1267117Sdfr
1367117Sdfr#include <linux/futex.h>
1467117Sdfr#include <linux/uaccess.h>
1567117Sdfr#include <asm/asm-eva.h>
1667117Sdfr#include <asm/barrier.h>
1767117Sdfr#include <asm/compiler.h>
1867117Sdfr#include <asm/errno.h>
1967117Sdfr#include <asm/sync.h>
2067117Sdfr
2167117Sdfr#define arch_futex_atomic_op_inuser arch_futex_atomic_op_inuser
2267117Sdfr#define futex_atomic_cmpxchg_inatomic futex_atomic_cmpxchg_inatomic
2367117Sdfr#include <asm-generic/futex.h>
2467117Sdfr
2567117Sdfr#define __futex_atomic_op(op, insn, ret, oldval, uaddr, oparg)		\
2667117Sdfr{									\
2767117Sdfr	if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) {	\
28111777Sobrien		__asm__ __volatile__(					\
2967117Sdfr		"	.set	push				\n"	\
3067117Sdfr		"	.set	noat				\n"	\
31134743Smarcel		"	.set	push				\n"	\
32134743Smarcel		"	.set	arch=r4000			\n"	\
33134743Smarcel		"1:	ll	%1, %4	# __futex_atomic_op	\n"	\
34134743Smarcel		"	.set	pop				\n"	\
35134743Smarcel		"	" insn	"				\n"	\
36134743Smarcel		"	.set	arch=r4000			\n"	\
37		"2:	sc	$1, %2				\n"	\
38		"	beqzl	$1, 1b				\n"	\
39		__stringify(__WEAK_LLSC_MB) "			\n"	\
40		"3:						\n"	\
41		"	.insn					\n"	\
42		"	.set	pop				\n"	\
43		"	.section .fixup,\"ax\"			\n"	\
44		"4:	li	%0, %6				\n"	\
45		"	j	3b				\n"	\
46		"	.previous				\n"	\
47		"	.section __ex_table,\"a\"		\n"	\
48		"	"__UA_ADDR "\t1b, 4b			\n"	\
49		"	"__UA_ADDR "\t2b, 4b			\n"	\
50		"	.previous				\n"	\
51		: "=r" (ret), "=&r" (oldval),				\
52		  "=" GCC_OFF_SMALL_ASM() (*uaddr)				\
53		: "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg),	\
54		  "i" (-EFAULT)						\
55		: "memory");						\
56	} else if (cpu_has_llsc) {					\
57		__asm__ __volatile__(					\
58		"	.set	push				\n"	\
59		"	.set	noat				\n"	\
60		"	.set	push				\n"	\
61		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"	\
62		"	" __SYNC(full, loongson3_war) "		\n"	\
63		"1:	"user_ll("%1", "%4")" # __futex_atomic_op\n"	\
64		"	.set	pop				\n"	\
65		"	" insn	"				\n"	\
66		"	.set	"MIPS_ISA_ARCH_LEVEL"		\n"	\
67		"2:	"user_sc("$1", "%2")"			\n"	\
68		"	beqz	$1, 1b				\n"	\
69		__stringify(__WEAK_LLSC_MB) "			\n"	\
70		"3:						\n"	\
71		"	.insn					\n"	\
72		"	.set	pop				\n"	\
73		"	.section .fixup,\"ax\"			\n"	\
74		"4:	li	%0, %6				\n"	\
75		"	j	3b				\n"	\
76		"	.previous				\n"	\
77		"	.section __ex_table,\"a\"		\n"	\
78		"	"__UA_ADDR "\t1b, 4b			\n"	\
79		"	"__UA_ADDR "\t2b, 4b			\n"	\
80		"	.previous				\n"	\
81		: "=r" (ret), "=&r" (oldval),				\
82		  "=" GCC_OFF_SMALL_ASM() (*uaddr)				\
83		: "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg),	\
84		  "i" (-EFAULT)						\
85		: "memory");						\
86	} else {							\
87		/* fallback for non-SMP */				\
88		ret = futex_atomic_op_inuser_local(op, oparg, oval, uaddr);	\
89	}								\
90}
91
92static inline int
93arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
94{
95	int oldval = 0, ret;
96
97	if (!access_ok(uaddr, sizeof(u32)))
98		return -EFAULT;
99
100	switch (op) {
101	case FUTEX_OP_SET:
102		__futex_atomic_op(op, "move $1, %z5", ret, oldval, uaddr, oparg);
103		break;
104
105	case FUTEX_OP_ADD:
106		__futex_atomic_op(op, "addu $1, %1, %z5",
107				  ret, oldval, uaddr, oparg);
108		break;
109	case FUTEX_OP_OR:
110		__futex_atomic_op(op, "or	$1, %1, %z5",
111				  ret, oldval, uaddr, oparg);
112		break;
113	case FUTEX_OP_ANDN:
114		__futex_atomic_op(op, "and	$1, %1, %z5",
115				  ret, oldval, uaddr, ~oparg);
116		break;
117	case FUTEX_OP_XOR:
118		__futex_atomic_op(op, "xor	$1, %1, %z5",
119				  ret, oldval, uaddr, oparg);
120		break;
121	default:
122		ret = -ENOSYS;
123	}
124
125	if (!ret)
126		*oval = oldval;
127
128	return ret;
129}
130
131static inline int
132futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
133			      u32 oldval, u32 newval)
134{
135	int ret = 0;
136	u32 val;
137
138	if (!access_ok(uaddr, sizeof(u32)))
139		return -EFAULT;
140
141	if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) {
142		__asm__ __volatile__(
143		"# futex_atomic_cmpxchg_inatomic			\n"
144		"	.set	push					\n"
145		"	.set	noat					\n"
146		"	.set	push					\n"
147		"	.set	arch=r4000				\n"
148		"1:	ll	%1, %3					\n"
149		"	bne	%1, %z4, 3f				\n"
150		"	.set	pop					\n"
151		"	move	$1, %z5					\n"
152		"	.set	arch=r4000				\n"
153		"2:	sc	$1, %2					\n"
154		"	beqzl	$1, 1b					\n"
155		__stringify(__WEAK_LLSC_MB) "				\n"
156		"3:							\n"
157		"	.insn						\n"
158		"	.set	pop					\n"
159		"	.section .fixup,\"ax\"				\n"
160		"4:	li	%0, %6					\n"
161		"	j	3b					\n"
162		"	.previous					\n"
163		"	.section __ex_table,\"a\"			\n"
164		"	"__UA_ADDR "\t1b, 4b				\n"
165		"	"__UA_ADDR "\t2b, 4b				\n"
166		"	.previous					\n"
167		: "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr)
168		: GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval),
169		  "i" (-EFAULT)
170		: "memory");
171	} else if (cpu_has_llsc) {
172		__asm__ __volatile__(
173		"# futex_atomic_cmpxchg_inatomic			\n"
174		"	.set	push					\n"
175		"	.set	noat					\n"
176		"	.set	push					\n"
177		"	.set	"MIPS_ISA_ARCH_LEVEL"			\n"
178		"	" __SYNC(full, loongson3_war) "			\n"
179		"1:	"user_ll("%1", "%3")"				\n"
180		"	bne	%1, %z4, 3f				\n"
181		"	.set	pop					\n"
182		"	move	$1, %z5					\n"
183		"	.set	"MIPS_ISA_ARCH_LEVEL"			\n"
184		"2:	"user_sc("$1", "%2")"				\n"
185		"	beqz	$1, 1b					\n"
186		"3:	" __SYNC_ELSE(full, loongson3_war, __WEAK_LLSC_MB) "\n"
187		"	.insn						\n"
188		"	.set	pop					\n"
189		"	.section .fixup,\"ax\"				\n"
190		"4:	li	%0, %6					\n"
191		"	j	3b					\n"
192		"	.previous					\n"
193		"	.section __ex_table,\"a\"			\n"
194		"	"__UA_ADDR "\t1b, 4b				\n"
195		"	"__UA_ADDR "\t2b, 4b				\n"
196		"	.previous					\n"
197		: "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr)
198		: GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval),
199		  "i" (-EFAULT)
200		: "memory");
201	} else {
202		return futex_atomic_cmpxchg_inatomic_local(uval, uaddr, oldval, newval);
203	}
204
205	*uval = val;
206	return ret;
207}
208
209#endif
210#endif /* _ASM_FUTEX_H */
211