1// SPDX-License-Identifier: GPL-2.0
2#include <linux/export.h>
3#include <linux/lockref.h>
4
5#if USE_CMPXCHG_LOCKREF
6
7/*
8 * Note that the "cmpxchg()" reloads the "old" value for the
9 * failure case.
10 */
11#define CMPXCHG_LOOP(CODE, SUCCESS) do {					\
12	int retry = 100;							\
13	struct lockref old;							\
14	BUILD_BUG_ON(sizeof(old) != 8);						\
15	old.lock_count = READ_ONCE(lockref->lock_count);			\
16	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {  	\
17		struct lockref new = old;					\
18		CODE								\
19		if (likely(try_cmpxchg64_relaxed(&lockref->lock_count,		\
20						 &old.lock_count,		\
21						 new.lock_count))) {		\
22			SUCCESS;						\
23		}								\
24		if (!--retry)							\
25			break;							\
26		cpu_relax();							\
27	}									\
28} while (0)
29
30#else
31
32#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
33
34#endif
35
36/**
37 * lockref_get - Increments reference count unconditionally
38 * @lockref: pointer to lockref structure
39 *
40 * This operation is only valid if you already hold a reference
41 * to the object, so you know the count cannot be zero.
42 */
43void lockref_get(struct lockref *lockref)
44{
45	CMPXCHG_LOOP(
46		new.count++;
47	,
48		return;
49	);
50
51	spin_lock(&lockref->lock);
52	lockref->count++;
53	spin_unlock(&lockref->lock);
54}
55EXPORT_SYMBOL(lockref_get);
56
57/**
58 * lockref_get_not_zero - Increments count unless the count is 0 or dead
59 * @lockref: pointer to lockref structure
60 * Return: 1 if count updated successfully or 0 if count was zero
61 */
62int lockref_get_not_zero(struct lockref *lockref)
63{
64	int retval;
65
66	CMPXCHG_LOOP(
67		new.count++;
68		if (old.count <= 0)
69			return 0;
70	,
71		return 1;
72	);
73
74	spin_lock(&lockref->lock);
75	retval = 0;
76	if (lockref->count > 0) {
77		lockref->count++;
78		retval = 1;
79	}
80	spin_unlock(&lockref->lock);
81	return retval;
82}
83EXPORT_SYMBOL(lockref_get_not_zero);
84
85/**
86 * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
87 * @lockref: pointer to lockref structure
88 * Return: 1 if count updated successfully or 0 if count would become zero
89 */
90int lockref_put_not_zero(struct lockref *lockref)
91{
92	int retval;
93
94	CMPXCHG_LOOP(
95		new.count--;
96		if (old.count <= 1)
97			return 0;
98	,
99		return 1;
100	);
101
102	spin_lock(&lockref->lock);
103	retval = 0;
104	if (lockref->count > 1) {
105		lockref->count--;
106		retval = 1;
107	}
108	spin_unlock(&lockref->lock);
109	return retval;
110}
111EXPORT_SYMBOL(lockref_put_not_zero);
112
113/**
114 * lockref_get_or_lock - Increments count unless the count is 0 or dead
115 * @lockref: pointer to lockref structure
116 * Return: 1 if count updated successfully or 0 if count was zero
117 * and we got the lock instead.
118 */
119int lockref_get_or_lock(struct lockref *lockref)
120{
121	CMPXCHG_LOOP(
122		new.count++;
123		if (old.count <= 0)
124			break;
125	,
126		return 1;
127	);
128
129	spin_lock(&lockref->lock);
130	if (lockref->count <= 0)
131		return 0;
132	lockref->count++;
133	spin_unlock(&lockref->lock);
134	return 1;
135}
136EXPORT_SYMBOL(lockref_get_or_lock);
137
138/**
139 * lockref_put_return - Decrement reference count if possible
140 * @lockref: pointer to lockref structure
141 *
142 * Decrement the reference count and return the new value.
143 * If the lockref was dead or locked, return an error.
144 */
145int lockref_put_return(struct lockref *lockref)
146{
147	CMPXCHG_LOOP(
148		new.count--;
149		if (old.count <= 0)
150			return -1;
151	,
152		return new.count;
153	);
154	return -1;
155}
156EXPORT_SYMBOL(lockref_put_return);
157
158/**
159 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
160 * @lockref: pointer to lockref structure
161 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
162 */
163int lockref_put_or_lock(struct lockref *lockref)
164{
165	CMPXCHG_LOOP(
166		new.count--;
167		if (old.count <= 1)
168			break;
169	,
170		return 1;
171	);
172
173	spin_lock(&lockref->lock);
174	if (lockref->count <= 1)
175		return 0;
176	lockref->count--;
177	spin_unlock(&lockref->lock);
178	return 1;
179}
180EXPORT_SYMBOL(lockref_put_or_lock);
181
182/**
183 * lockref_mark_dead - mark lockref dead
184 * @lockref: pointer to lockref structure
185 */
186void lockref_mark_dead(struct lockref *lockref)
187{
188	assert_spin_locked(&lockref->lock);
189	lockref->count = -128;
190}
191EXPORT_SYMBOL(lockref_mark_dead);
192
193/**
194 * lockref_get_not_dead - Increments count unless the ref is dead
195 * @lockref: pointer to lockref structure
196 * Return: 1 if count updated successfully or 0 if lockref was dead
197 */
198int lockref_get_not_dead(struct lockref *lockref)
199{
200	int retval;
201
202	CMPXCHG_LOOP(
203		new.count++;
204		if (old.count < 0)
205			return 0;
206	,
207		return 1;
208	);
209
210	spin_lock(&lockref->lock);
211	retval = 0;
212	if (lockref->count >= 0) {
213		lockref->count++;
214		retval = 1;
215	}
216	spin_unlock(&lockref->lock);
217	return retval;
218}
219EXPORT_SYMBOL(lockref_get_not_dead);
220