1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1998 Alex Nash
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <errno.h>
30#include <limits.h>
31#include <stdlib.h>
32#include <string.h>
33
34#include "namespace.h"
35#include <pthread.h>
36#include "un-namespace.h"
37#include "thr_private.h"
38
39_Static_assert(sizeof(struct pthread_rwlock) <= THR_PAGE_SIZE_MIN,
40    "pthread_rwlock is too large for off-page");
41
42__weak_reference(_thr_rwlock_destroy, pthread_rwlock_destroy);
43__weak_reference(_thr_rwlock_destroy, _pthread_rwlock_destroy);
44__weak_reference(_thr_rwlock_init, pthread_rwlock_init);
45__weak_reference(_thr_rwlock_init, _pthread_rwlock_init);
46__weak_reference(_Tthr_rwlock_rdlock, pthread_rwlock_rdlock);
47__weak_reference(_Tthr_rwlock_rdlock, _pthread_rwlock_rdlock);
48__weak_reference(_pthread_rwlock_timedrdlock, pthread_rwlock_timedrdlock);
49__weak_reference(_Tthr_rwlock_tryrdlock, pthread_rwlock_tryrdlock);
50__weak_reference(_Tthr_rwlock_tryrdlock, _pthread_rwlock_tryrdlock);
51__weak_reference(_Tthr_rwlock_trywrlock, pthread_rwlock_trywrlock);
52__weak_reference(_Tthr_rwlock_trywrlock, _pthread_rwlock_trywrlock);
53__weak_reference(_Tthr_rwlock_unlock, pthread_rwlock_unlock);
54__weak_reference(_Tthr_rwlock_unlock, _pthread_rwlock_unlock);
55__weak_reference(_Tthr_rwlock_wrlock, pthread_rwlock_wrlock);
56__weak_reference(_Tthr_rwlock_wrlock, _pthread_rwlock_wrlock);
57__weak_reference(_pthread_rwlock_timedwrlock, pthread_rwlock_timedwrlock);
58
59static int init_static(struct pthread *thread, pthread_rwlock_t *rwlock);
60static int init_rwlock(pthread_rwlock_t *rwlock, pthread_rwlock_t *rwlock_out);
61
62static int __always_inline
63check_and_init_rwlock(pthread_rwlock_t *rwlock, pthread_rwlock_t *rwlock_out)
64{
65	if (__predict_false(*rwlock == THR_PSHARED_PTR ||
66	    *rwlock <= THR_RWLOCK_DESTROYED))
67		return (init_rwlock(rwlock, rwlock_out));
68	*rwlock_out = *rwlock;
69	return (0);
70}
71
72static int __noinline
73init_rwlock(pthread_rwlock_t *rwlock, pthread_rwlock_t *rwlock_out)
74{
75	pthread_rwlock_t prwlock;
76	int ret;
77
78	if (*rwlock == THR_PSHARED_PTR) {
79		prwlock = __thr_pshared_offpage(rwlock, 0);
80		if (prwlock == NULL)
81			return (EINVAL);
82	} else if ((prwlock = *rwlock) <= THR_RWLOCK_DESTROYED) {
83		if (prwlock == THR_RWLOCK_INITIALIZER) {
84			ret = init_static(_get_curthread(), rwlock);
85			if (ret != 0)
86				return (ret);
87		} else if (prwlock == THR_RWLOCK_DESTROYED) {
88			return (EINVAL);
89		}
90		prwlock = *rwlock;
91	}
92	*rwlock_out = prwlock;
93	return (0);
94}
95
96static int
97rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
98{
99	pthread_rwlock_t prwlock;
100
101	if (attr == NULL || *attr == NULL ||
102	    (*attr)->pshared == PTHREAD_PROCESS_PRIVATE) {
103		prwlock = aligned_alloc(CACHE_LINE_SIZE,
104		    roundup(sizeof(struct pthread_rwlock), CACHE_LINE_SIZE));
105		if (prwlock == NULL)
106			return (ENOMEM);
107		memset(prwlock, 0, sizeof(struct pthread_rwlock));
108		*rwlock = prwlock;
109	} else {
110		prwlock = __thr_pshared_offpage(rwlock, 1);
111		if (prwlock == NULL)
112			return (EFAULT);
113		prwlock->lock.rw_flags |= USYNC_PROCESS_SHARED;
114		*rwlock = THR_PSHARED_PTR;
115	}
116	return (0);
117}
118
119int
120_thr_rwlock_destroy(pthread_rwlock_t *rwlock)
121{
122	pthread_rwlock_t prwlock;
123	int ret;
124
125	prwlock = *rwlock;
126	if (prwlock == THR_RWLOCK_INITIALIZER)
127		ret = 0;
128	else if (prwlock == THR_RWLOCK_DESTROYED)
129		ret = EINVAL;
130	else if (prwlock == THR_PSHARED_PTR) {
131		*rwlock = THR_RWLOCK_DESTROYED;
132		__thr_pshared_destroy(rwlock);
133		ret = 0;
134	} else {
135		*rwlock = THR_RWLOCK_DESTROYED;
136		free(prwlock);
137		ret = 0;
138	}
139	return (ret);
140}
141
142static int
143init_static(struct pthread *thread, pthread_rwlock_t *rwlock)
144{
145	int ret;
146
147	THR_LOCK_ACQUIRE(thread, &_rwlock_static_lock);
148
149	if (*rwlock == THR_RWLOCK_INITIALIZER)
150		ret = rwlock_init(rwlock, NULL);
151	else
152		ret = 0;
153
154	THR_LOCK_RELEASE(thread, &_rwlock_static_lock);
155
156	return (ret);
157}
158
159int
160_thr_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
161{
162
163	_thr_check_init();
164	*rwlock = NULL;
165	return (rwlock_init(rwlock, attr));
166}
167
168static int
169rwlock_rdlock_common(pthread_rwlock_t *rwlock, const struct timespec *abstime)
170{
171	struct pthread *curthread = _get_curthread();
172	pthread_rwlock_t prwlock;
173	int flags;
174	int ret;
175
176	ret = check_and_init_rwlock(rwlock, &prwlock);
177	if (ret != 0)
178		return (ret);
179
180	if (curthread->rdlock_count) {
181		/*
182		 * To avoid having to track all the rdlocks held by
183		 * a thread or all of the threads that hold a rdlock,
184		 * we keep a simple count of all the rdlocks held by
185		 * a thread.  If a thread holds any rdlocks it is
186		 * possible that it is attempting to take a recursive
187		 * rdlock.  If there are blocked writers and precedence
188		 * is given to them, then that would result in the thread
189		 * deadlocking.  So allowing a thread to take the rdlock
190		 * when it already has one or more rdlocks avoids the
191		 * deadlock.  I hope the reader can follow that logic ;-)
192		 */
193		flags = URWLOCK_PREFER_READER;
194	} else {
195		flags = 0;
196	}
197
198	/*
199	 * POSIX said the validity of the abstimeout parameter need
200	 * not be checked if the lock can be immediately acquired.
201	 */
202	ret = _thr_rwlock_tryrdlock(&prwlock->lock, flags);
203	if (ret == 0) {
204		curthread->rdlock_count++;
205		return (ret);
206	}
207
208	if (__predict_false(abstime &&
209		(abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0)))
210		return (EINVAL);
211
212	for (;;) {
213		/* goto kernel and lock it */
214		ret = __thr_rwlock_rdlock(&prwlock->lock, flags, abstime);
215		if (ret != EINTR)
216			break;
217
218		/* if interrupted, try to lock it in userland again. */
219		if (_thr_rwlock_tryrdlock(&prwlock->lock, flags) == 0) {
220			ret = 0;
221			break;
222		}
223	}
224	if (ret == 0)
225		curthread->rdlock_count++;
226	return (ret);
227}
228
229int
230_Tthr_rwlock_rdlock(pthread_rwlock_t *rwlock)
231{
232	_thr_check_init();
233	return (rwlock_rdlock_common(rwlock, NULL));
234}
235
236int
237_pthread_rwlock_timedrdlock(pthread_rwlock_t * __restrict rwlock,
238    const struct timespec * __restrict abstime)
239{
240	_thr_check_init();
241	return (rwlock_rdlock_common(rwlock, abstime));
242}
243
244int
245_Tthr_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
246{
247	struct pthread *curthread;
248	pthread_rwlock_t prwlock;
249	int flags;
250	int ret;
251
252	_thr_check_init();
253	ret = check_and_init_rwlock(rwlock, &prwlock);
254	if (ret != 0)
255		return (ret);
256
257	curthread = _get_curthread();
258	if (curthread->rdlock_count) {
259		/*
260		 * To avoid having to track all the rdlocks held by
261		 * a thread or all of the threads that hold a rdlock,
262		 * we keep a simple count of all the rdlocks held by
263		 * a thread.  If a thread holds any rdlocks it is
264		 * possible that it is attempting to take a recursive
265		 * rdlock.  If there are blocked writers and precedence
266		 * is given to them, then that would result in the thread
267		 * deadlocking.  So allowing a thread to take the rdlock
268		 * when it already has one or more rdlocks avoids the
269		 * deadlock.  I hope the reader can follow that logic ;-)
270		 */
271		flags = URWLOCK_PREFER_READER;
272	} else {
273		flags = 0;
274	}
275
276	ret = _thr_rwlock_tryrdlock(&prwlock->lock, flags);
277	if (ret == 0)
278		curthread->rdlock_count++;
279	return (ret);
280}
281
282int
283_Tthr_rwlock_trywrlock(pthread_rwlock_t *rwlock)
284{
285	struct pthread *curthread;
286	pthread_rwlock_t prwlock;
287	int ret;
288
289	_thr_check_init();
290	ret = check_and_init_rwlock(rwlock, &prwlock);
291	if (ret != 0)
292		return (ret);
293
294	curthread = _get_curthread();
295	ret = _thr_rwlock_trywrlock(&prwlock->lock);
296	if (ret == 0)
297		prwlock->owner = TID(curthread);
298	return (ret);
299}
300
301static int
302rwlock_wrlock_common(pthread_rwlock_t *rwlock, const struct timespec *abstime)
303{
304	struct pthread *curthread = _get_curthread();
305	pthread_rwlock_t prwlock;
306	int ret;
307
308	ret = check_and_init_rwlock(rwlock, &prwlock);
309	if (ret != 0)
310		return (ret);
311
312	/*
313	 * POSIX said the validity of the abstimeout parameter need
314	 * not be checked if the lock can be immediately acquired.
315	 */
316	ret = _thr_rwlock_trywrlock(&prwlock->lock);
317	if (ret == 0) {
318		prwlock->owner = TID(curthread);
319		return (ret);
320	}
321
322	if (__predict_false(abstime &&
323	    (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0)))
324		return (EINVAL);
325
326	for (;;) {
327		/* goto kernel and lock it */
328		ret = __thr_rwlock_wrlock(&prwlock->lock, abstime);
329		if (ret == 0) {
330			prwlock->owner = TID(curthread);
331			break;
332		}
333
334		if (ret != EINTR)
335			break;
336
337		/* if interrupted, try to lock it in userland again. */
338		if (_thr_rwlock_trywrlock(&prwlock->lock) == 0) {
339			ret = 0;
340			prwlock->owner = TID(curthread);
341			break;
342		}
343	}
344	return (ret);
345}
346
347int
348_Tthr_rwlock_wrlock(pthread_rwlock_t *rwlock)
349{
350	_thr_check_init();
351	return (rwlock_wrlock_common(rwlock, NULL));
352}
353
354int
355_pthread_rwlock_timedwrlock(pthread_rwlock_t * __restrict rwlock,
356    const struct timespec * __restrict abstime)
357{
358	_thr_check_init();
359	return (rwlock_wrlock_common(rwlock, abstime));
360}
361
362int
363_Tthr_rwlock_unlock(pthread_rwlock_t *rwlock)
364{
365	struct pthread *curthread = _get_curthread();
366	pthread_rwlock_t prwlock;
367	int ret;
368	int32_t state;
369
370	if (*rwlock == THR_PSHARED_PTR) {
371		prwlock = __thr_pshared_offpage(rwlock, 0);
372		if (prwlock == NULL)
373			return (EINVAL);
374	} else {
375		prwlock = *rwlock;
376	}
377
378	if (__predict_false(prwlock <= THR_RWLOCK_DESTROYED))
379		return (EINVAL);
380
381	state = prwlock->lock.rw_state;
382	if (state & URWLOCK_WRITE_OWNER) {
383		if (__predict_false(prwlock->owner != TID(curthread)))
384			return (EPERM);
385		prwlock->owner = 0;
386	}
387
388	ret = _thr_rwlock_unlock(&prwlock->lock);
389	if (ret == 0 && (state & URWLOCK_WRITE_OWNER) == 0)
390		curthread->rdlock_count--;
391
392	return (ret);
393}
394