thr_umtx.h revision 233912 
1/*-
2 * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: head/lib/libthr/thread/thr_umtx.h 233912 2012-04-05 02:24:08Z davidxu $
27 */
28
29#ifndef _THR_FBSD_UMTX_H_
30#define _THR_FBSD_UMTX_H_
31
32#include <strings.h>
33#include <sys/umtx.h>
34
35#define DEFAULT_UMUTEX	{0,0,{0,0},{0,0,0,0}}
36#define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}}
37
38int _umtx_op_err(void *, int op, u_long, void *, void *) __hidden;
39int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
40int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden;
41int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
42	const struct timespec *timeout) __hidden;
43int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) __hidden;
44int __thr_umutex_trylock(struct umutex *mtx) __hidden;
45int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
46	uint32_t *oldceiling) __hidden;
47
48void _thr_umutex_init(struct umutex *mtx) __hidden;
49void _thr_urwlock_init(struct urwlock *rwl) __hidden;
50
51int _thr_umtx_wait(volatile long *mtx, long exp,
52	const struct timespec *timeout) __hidden;
53int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp,
54	const struct timespec *timeout, int shared) __hidden;
55int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid,
56	const struct timespec *timeout, int shared) __hidden;
57int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden;
58int _thr_ucond_wait(struct ucond *cv, struct umutex *m,
59        const struct timespec *timeout, int check_unpaking) __hidden;
60void _thr_ucond_init(struct ucond *cv) __hidden;
61int _thr_ucond_signal(struct ucond *cv) __hidden;
62int _thr_ucond_broadcast(struct ucond *cv) __hidden;
63
64int __thr_rwlock_rdlock(struct urwlock *rwlock, int flags,
65	const struct timespec *tsp) __hidden;
66int __thr_rwlock_wrlock(struct urwlock *rwlock,
67	const struct timespec *tsp) __hidden;
68int __thr_rwlock_unlock(struct urwlock *rwlock) __hidden;
69
70/* Internal used only */
71void _thr_rwl_rdlock(struct urwlock *rwlock) __hidden;
72void _thr_rwl_wrlock(struct urwlock *rwlock) __hidden;
73void _thr_rwl_unlock(struct urwlock *rwlock) __hidden;
74
75static inline int
76_thr_umutex_trylock(struct umutex *mtx, uint32_t id)
77{
78    if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
79	return (0);
80    if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0)
81    	return (EBUSY);
82    return (__thr_umutex_trylock(mtx));
83}
84
85static inline int
86_thr_umutex_trylock2(struct umutex *mtx, uint32_t id)
87{
88    if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0)
89	return (0);
90    if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED &&
91        __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0))
92    	if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED))
93		return (0);
94    return (EBUSY);
95}
96
97static inline int
98_thr_umutex_lock(struct umutex *mtx, uint32_t id)
99{
100    if (_thr_umutex_trylock2(mtx, id) == 0)
101	return (0);
102    return (__thr_umutex_lock(mtx, id));
103}
104
105static inline int
106_thr_umutex_lock_spin(struct umutex *mtx, uint32_t id)
107{
108    if (_thr_umutex_trylock2(mtx, id) == 0)
109	return (0);
110    return (__thr_umutex_lock_spin(mtx, id));
111}
112
113static inline int
114_thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
115	const struct timespec *timeout)
116{
117    if (_thr_umutex_trylock2(mtx, id) == 0)
118	return (0);
119    return (__thr_umutex_timedlock(mtx, id, timeout));
120}
121
122static inline int
123_thr_umutex_unlock(struct umutex *mtx, uint32_t id)
124{
125	uint32_t flags = mtx->m_flags;
126
127	if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) {
128		uint32_t owner;
129		do {
130			owner = mtx->m_owner;
131			if (__predict_false((owner & ~UMUTEX_CONTESTED) != id))
132				return (EPERM);
133		} while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner,
134					 owner, UMUTEX_UNOWNED)));
135		if ((owner & UMUTEX_CONTESTED))
136			(void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2, flags, 0, 0);
137		return (0);
138	}
139    	if (atomic_cmpset_rel_32(&mtx->m_owner, id, UMUTEX_UNOWNED))
140		return (0);
141	return (__thr_umutex_unlock(mtx, id));
142}
143
144static inline int
145_thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags)
146{
147	int32_t state;
148	int32_t wrflags;
149
150	if (flags & URWLOCK_PREFER_READER || rwlock->rw_flags & URWLOCK_PREFER_READER)
151		wrflags = URWLOCK_WRITE_OWNER;
152	else
153		wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS;
154	state = rwlock->rw_state;
155	while (!(state & wrflags)) {
156		if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS))
157			return (EAGAIN);
158		if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1))
159			return (0);
160		state = rwlock->rw_state;
161	}
162
163	return (EBUSY);
164}
165
166static inline int
167_thr_rwlock_trywrlock(struct urwlock *rwlock)
168{
169	int32_t state;
170
171	state = rwlock->rw_state;
172	while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
173		if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER))
174			return (0);
175		state = rwlock->rw_state;
176	}
177
178	return (EBUSY);
179}
180
181static inline int
182_thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp)
183{
184	if (_thr_rwlock_tryrdlock(rwlock, flags) == 0)
185		return (0);
186	return (__thr_rwlock_rdlock(rwlock, flags, tsp));
187}
188
189static inline int
190_thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp)
191{
192	if (_thr_rwlock_trywrlock(rwlock) == 0)
193		return (0);
194	return (__thr_rwlock_wrlock(rwlock, tsp));
195}
196
197static inline int
198_thr_rwlock_unlock(struct urwlock *rwlock)
199{
200	int32_t state;
201
202	state = rwlock->rw_state;
203	if (state & URWLOCK_WRITE_OWNER) {
204		if (atomic_cmpset_rel_32(&rwlock->rw_state, URWLOCK_WRITE_OWNER, 0))
205			return (0);
206	} else {
207		for (;;) {
208			if (__predict_false(URWLOCK_READER_COUNT(state) == 0))
209				return (EPERM);
210			if (!((state & (URWLOCK_WRITE_WAITERS |
211			    URWLOCK_READ_WAITERS)) &&
212			    URWLOCK_READER_COUNT(state) == 1)) {
213				if (atomic_cmpset_rel_32(&rwlock->rw_state,
214				    state, state-1))
215					return (0);
216				state = rwlock->rw_state;
217			} else {
218				break;
219			}
220		}
221    	}
222    	return (__thr_rwlock_unlock(rwlock));
223}
224#endif
225