thr_umtx.c revision 212077
1144518Sdavidxu/* 2144518Sdavidxu * Copyright (c) 2005 David Xu <davidxu@freebsd.org> 3144518Sdavidxu * All rights reserved. 4144518Sdavidxu * 5144518Sdavidxu * Redistribution and use in source and binary forms, with or without 6144518Sdavidxu * modification, are permitted provided that the following conditions 7144518Sdavidxu * are met: 8144518Sdavidxu * 1. Redistributions of source code must retain the above copyright 9144518Sdavidxu * notice unmodified, this list of conditions, and the following 10144518Sdavidxu * disclaimer. 11144518Sdavidxu * 2. Redistributions in binary form must reproduce the above copyright 12144518Sdavidxu * notice, this list of conditions and the following disclaimer in the 13144518Sdavidxu * documentation and/or other materials provided with the distribution. 14144518Sdavidxu * 15144518Sdavidxu * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16144518Sdavidxu * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17144518Sdavidxu * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18144518Sdavidxu * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19144518Sdavidxu * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20144518Sdavidxu * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21144518Sdavidxu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22144518Sdavidxu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23144518Sdavidxu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24144518Sdavidxu * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25144518Sdavidxu * 26144518Sdavidxu * $FreeBSD: head/lib/libthr/thread/thr_umtx.c 212077 2010-09-01 03:11:21Z davidxu $ 27144518Sdavidxu * 28144518Sdavidxu */ 29144518Sdavidxu 30144518Sdavidxu#include "thr_private.h" 31144518Sdavidxu#include "thr_umtx.h" 32144518Sdavidxu 33177853Sdavidxu#ifndef HAS__UMTX_OP_ERR 34177853Sdavidxuint _umtx_op_err(void *obj, int op, u_long val, void *uaddr, void *uaddr2) 35177853Sdavidxu{ 36177853Sdavidxu if (_umtx_op(obj, op, val, uaddr, uaddr2) == -1) 37177853Sdavidxu return (errno); 38177853Sdavidxu return (0); 39177853Sdavidxu} 40177853Sdavidxu#endif 41177853Sdavidxu 42163334Sdavidxuvoid 43163334Sdavidxu_thr_umutex_init(struct umutex *mtx) 44163334Sdavidxu{ 45163334Sdavidxu static struct umutex default_mtx = DEFAULT_UMUTEX; 46163334Sdavidxu 47163334Sdavidxu *mtx = default_mtx; 48163334Sdavidxu} 49163334Sdavidxu 50212077Sdavidxuvoid 51212077Sdavidxu_thr_urwlock_init(struct urwlock *rwl) 52212077Sdavidxu{ 53212077Sdavidxu static struct urwlock default_rwl = DEFAULT_URWLOCK; 54212077Sdavidxu *rwl = default_rwl; 55212077Sdavidxu} 56212077Sdavidxu 57144518Sdavidxuint 58179970Sdavidxu__thr_umutex_lock(struct umutex *mtx, uint32_t id) 59161680Sdavidxu{ 60179970Sdavidxu uint32_t owner; 61179970Sdavidxu 62179970Sdavidxu if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 63179970Sdavidxu for (;;) { 64179970Sdavidxu /* wait in kernel */ 65179970Sdavidxu _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); 66179970Sdavidxu 67179970Sdavidxu owner = mtx->m_owner; 68179970Sdavidxu if ((owner & ~UMUTEX_CONTESTED) == 0 && 69179970Sdavidxu atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) 70179970Sdavidxu return (0); 71179970Sdavidxu } 72179970Sdavidxu } 73179970Sdavidxu 74179970Sdavidxu return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); 75161680Sdavidxu} 76161680Sdavidxu 77161680Sdavidxuint 78179970Sdavidxu__thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 79179970Sdavidxu const struct timespec *ets) 80161680Sdavidxu{ 81179970Sdavidxu struct timespec timo, cts; 82179970Sdavidxu uint32_t owner; 83179970Sdavidxu int ret; 84179970Sdavidxu 85179970Sdavidxu clock_gettime(CLOCK_REALTIME, &cts); 86179970Sdavidxu TIMESPEC_SUB(&timo, ets, &cts); 87179970Sdavidxu 88179970Sdavidxu if (timo.tv_sec < 0) 89161680Sdavidxu return (ETIMEDOUT); 90179970Sdavidxu 91179970Sdavidxu for (;;) { 92179970Sdavidxu if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 93179970Sdavidxu 94179970Sdavidxu /* wait in kernel */ 95179970Sdavidxu ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, &timo); 96179970Sdavidxu 97179970Sdavidxu /* now try to lock it */ 98179970Sdavidxu owner = mtx->m_owner; 99179970Sdavidxu if ((owner & ~UMUTEX_CONTESTED) == 0 && 100179970Sdavidxu atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) 101179970Sdavidxu return (0); 102179970Sdavidxu } else { 103179970Sdavidxu ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, &timo); 104179970Sdavidxu if (ret == 0) 105179970Sdavidxu break; 106179970Sdavidxu } 107179970Sdavidxu if (ret == ETIMEDOUT) 108179970Sdavidxu break; 109179970Sdavidxu clock_gettime(CLOCK_REALTIME, &cts); 110179970Sdavidxu TIMESPEC_SUB(&timo, ets, &cts); 111179970Sdavidxu if (timo.tv_sec < 0 || (timo.tv_sec == 0 && timo.tv_nsec == 0)) { 112179970Sdavidxu ret = ETIMEDOUT; 113179970Sdavidxu break; 114179970Sdavidxu } 115179970Sdavidxu } 116179970Sdavidxu return (ret); 117161680Sdavidxu} 118161680Sdavidxu 119161680Sdavidxuint 120179970Sdavidxu__thr_umutex_unlock(struct umutex *mtx, uint32_t id) 121161680Sdavidxu{ 122200498Smarcel#ifndef __ia64__ 123200498Smarcel /* XXX this logic has a race-condition on ia64. */ 124179970Sdavidxu if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 125179970Sdavidxu atomic_cmpset_rel_32(&mtx->m_owner, id | UMUTEX_CONTESTED, UMUTEX_CONTESTED); 126179970Sdavidxu return _umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE, 0, 0, 0); 127179970Sdavidxu } 128200498Smarcel#endif /* __ia64__ */ 129177853Sdavidxu return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0); 130161680Sdavidxu} 131161680Sdavidxu 132161680Sdavidxuint 133163334Sdavidxu__thr_umutex_trylock(struct umutex *mtx) 134161680Sdavidxu{ 135177853Sdavidxu return _umtx_op_err(mtx, UMTX_OP_MUTEX_TRYLOCK, 0, 0, 0); 136161680Sdavidxu} 137161680Sdavidxu 138161680Sdavidxuint 139161680Sdavidxu__thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, 140161680Sdavidxu uint32_t *oldceiling) 141161680Sdavidxu{ 142177853Sdavidxu return _umtx_op_err(mtx, UMTX_OP_SET_CEILING, ceiling, oldceiling, 0); 143161680Sdavidxu} 144161680Sdavidxu 145161680Sdavidxuint 146173801Sdavidxu_thr_umtx_wait(volatile long *mtx, long id, const struct timespec *timeout) 147144518Sdavidxu{ 148144518Sdavidxu if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 149144518Sdavidxu timeout->tv_nsec <= 0))) 150144518Sdavidxu return (ETIMEDOUT); 151177853Sdavidxu return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAIT, id, 0, 152177853Sdavidxu __DECONST(void*, timeout)); 153144518Sdavidxu} 154144518Sdavidxu 155144518Sdavidxuint 156178647Sdavidxu_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout, int shared) 157144518Sdavidxu{ 158173801Sdavidxu if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 159173801Sdavidxu timeout->tv_nsec <= 0))) 160173801Sdavidxu return (ETIMEDOUT); 161178647Sdavidxu return _umtx_op_err(__DEVOLATILE(void *, mtx), 162178647Sdavidxu shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 0, 163178647Sdavidxu __DECONST(void*, timeout)); 164173801Sdavidxu} 165173801Sdavidxu 166173801Sdavidxuint 167178647Sdavidxu_thr_umtx_wake(volatile void *mtx, int nr_wakeup, int shared) 168173801Sdavidxu{ 169178647Sdavidxu return _umtx_op_err(__DEVOLATILE(void *, mtx), shared ? UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE, 170177853Sdavidxu nr_wakeup, 0, 0); 171144518Sdavidxu} 172164877Sdavidxu 173164902Sdavidxuvoid 174164902Sdavidxu_thr_ucond_init(struct ucond *cv) 175164902Sdavidxu{ 176164902Sdavidxu bzero(cv, sizeof(struct ucond)); 177164902Sdavidxu} 178164902Sdavidxu 179164877Sdavidxuint 180164877Sdavidxu_thr_ucond_wait(struct ucond *cv, struct umutex *m, 181164877Sdavidxu const struct timespec *timeout, int check_unparking) 182164877Sdavidxu{ 183164877Sdavidxu if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 184164877Sdavidxu timeout->tv_nsec <= 0))) { 185179970Sdavidxu struct pthread *curthread = _get_curthread(); 186179970Sdavidxu _thr_umutex_unlock(m, TID(curthread)); 187164877Sdavidxu return (ETIMEDOUT); 188164877Sdavidxu } 189177853Sdavidxu return _umtx_op_err(cv, UMTX_OP_CV_WAIT, 190164878Sdavidxu check_unparking ? UMTX_CHECK_UNPARKING : 0, 191177853Sdavidxu m, __DECONST(void*, timeout)); 192164877Sdavidxu} 193164877Sdavidxu 194164877Sdavidxuint 195164877Sdavidxu_thr_ucond_signal(struct ucond *cv) 196164877Sdavidxu{ 197165110Sdavidxu if (!cv->c_has_waiters) 198165110Sdavidxu return (0); 199177853Sdavidxu return _umtx_op_err(cv, UMTX_OP_CV_SIGNAL, 0, NULL, NULL); 200164877Sdavidxu} 201164877Sdavidxu 202164877Sdavidxuint 203164877Sdavidxu_thr_ucond_broadcast(struct ucond *cv) 204164877Sdavidxu{ 205165110Sdavidxu if (!cv->c_has_waiters) 206165110Sdavidxu return (0); 207177853Sdavidxu return _umtx_op_err(cv, UMTX_OP_CV_BROADCAST, 0, NULL, NULL); 208164877Sdavidxu} 209177850Sdavidxu 210177850Sdavidxuint 211177850Sdavidxu__thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp) 212177850Sdavidxu{ 213177853Sdavidxu return _umtx_op_err(rwlock, UMTX_OP_RW_RDLOCK, flags, NULL, tsp); 214177850Sdavidxu} 215177850Sdavidxu 216177850Sdavidxuint 217177850Sdavidxu__thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp) 218177850Sdavidxu{ 219177853Sdavidxu return _umtx_op_err(rwlock, UMTX_OP_RW_WRLOCK, 0, NULL, tsp); 220177850Sdavidxu} 221177850Sdavidxu 222177850Sdavidxuint 223177850Sdavidxu__thr_rwlock_unlock(struct urwlock *rwlock) 224177850Sdavidxu{ 225177853Sdavidxu return _umtx_op_err(rwlock, UMTX_OP_RW_UNLOCK, 0, NULL, NULL); 226177850Sdavidxu} 227212076Sdavidxu 228212076Sdavidxuvoid 229212076Sdavidxu_thr_rwl_rdlock(struct urwlock *rwlock) 230212076Sdavidxu{ 231212076Sdavidxu int ret; 232212076Sdavidxu 233212076Sdavidxu for (;;) { 234212076Sdavidxu if (_thr_rwlock_tryrdlock(rwlock, URWLOCK_PREFER_READER) == 0) 235212076Sdavidxu return; 236212076Sdavidxu ret = __thr_rwlock_rdlock(rwlock, URWLOCK_PREFER_READER, NULL); 237212076Sdavidxu if (ret == 0) 238212076Sdavidxu return; 239212076Sdavidxu if (ret != EINTR) 240212076Sdavidxu PANIC("rdlock error"); 241212076Sdavidxu } 242212076Sdavidxu} 243212076Sdavidxu 244212076Sdavidxuvoid 245212076Sdavidxu_thr_rwl_wrlock(struct urwlock *rwlock) 246212076Sdavidxu{ 247212076Sdavidxu int ret; 248212076Sdavidxu 249212076Sdavidxu for (;;) { 250212076Sdavidxu if (_thr_rwlock_trywrlock(rwlock) == 0) 251212076Sdavidxu return; 252212076Sdavidxu ret = __thr_rwlock_wrlock(rwlock, NULL); 253212076Sdavidxu if (ret == 0) 254212076Sdavidxu return; 255212076Sdavidxu if (ret != EINTR) 256212076Sdavidxu PANIC("wrlock error"); 257212076Sdavidxu } 258212076Sdavidxu} 259212076Sdavidxu 260212076Sdavidxuvoid 261212076Sdavidxu_thr_rwl_unlock(struct urwlock *rwlock) 262212076Sdavidxu{ 263212076Sdavidxu if (_thr_rwlock_unlock(rwlock)) 264212076Sdavidxu PANIC("unlock error"); 265212076Sdavidxu} 266