thr_umtx.c revision 297706
1/* 2 * Copyright (c) 2005 David Xu <davidxu@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/lib/libthr/thread/thr_umtx.c 297706 2016-04-08 11:15:26Z kib $"); 29 30#include "thr_private.h" 31#include "thr_umtx.h" 32 33#ifndef HAS__UMTX_OP_ERR 34int _umtx_op_err(void *obj, int op, u_long val, void *uaddr, void *uaddr2) 35{ 36 if (_umtx_op(obj, op, val, uaddr, uaddr2) == -1) 37 return (errno); 38 return (0); 39} 40#endif 41 42void 43_thr_umutex_init(struct umutex *mtx) 44{ 45 static const struct umutex default_mtx = DEFAULT_UMUTEX; 46 47 *mtx = default_mtx; 48} 49 50void 51_thr_urwlock_init(struct urwlock *rwl) 52{ 53 static const struct urwlock default_rwl = DEFAULT_URWLOCK; 54 55 *rwl = default_rwl; 56} 57 58int 59__thr_umutex_lock(struct umutex *mtx, uint32_t id) 60{ 61 uint32_t owner; 62 63 if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 64 for (;;) { 65 /* wait in kernel */ 66 _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); 67 68 owner = mtx->m_owner; 69 if ((owner & ~UMUTEX_CONTESTED) == 0 && 70 atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) 71 return (0); 72 } 73 } 74 75 return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); 76} 77 78#define SPINLOOPS 1000 79 80int 81__thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) 82{ 83 uint32_t owner; 84 85 if (!_thr_is_smp) 86 return __thr_umutex_lock(mtx, id); 87 88 if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 89 for (;;) { 90 int count = SPINLOOPS; 91 while (count--) { 92 owner = mtx->m_owner; 93 if ((owner & ~UMUTEX_CONTESTED) == 0) { 94 if (atomic_cmpset_acq_32( 95 &mtx->m_owner, 96 owner, id|owner)) { 97 return (0); 98 } 99 } 100 CPU_SPINWAIT; 101 } 102 103 /* wait in kernel */ 104 _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); 105 } 106 } 107 108 return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); 109} 110 111int 112__thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 113 const struct timespec *abstime) 114{ 115 struct _umtx_time *tm_p, timeout; 116 size_t tm_size; 117 uint32_t owner; 118 int ret; 119 120 if (abstime == NULL) { 121 tm_p = NULL; 122 tm_size = 0; 123 } else { 124 timeout._clockid = CLOCK_REALTIME; 125 timeout._flags = UMTX_ABSTIME; 126 timeout._timeout = *abstime; 127 tm_p = &timeout; 128 tm_size = sizeof(timeout); 129 } 130 131 for (;;) { 132 if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 133 134 /* wait in kernel */ 135 ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 136 (void *)tm_size, __DECONST(void *, tm_p)); 137 138 /* now try to lock it */ 139 owner = mtx->m_owner; 140 if ((owner & ~UMUTEX_CONTESTED) == 0 && 141 atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) 142 return (0); 143 } else { 144 ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 145 (void *)tm_size, __DECONST(void *, tm_p)); 146 if (ret == 0) 147 break; 148 } 149 if (ret == ETIMEDOUT) 150 break; 151 } 152 return (ret); 153} 154 155int 156__thr_umutex_unlock(struct umutex *mtx, uint32_t id) 157{ 158 return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0); 159} 160 161int 162__thr_umutex_trylock(struct umutex *mtx) 163{ 164 return _umtx_op_err(mtx, UMTX_OP_MUTEX_TRYLOCK, 0, 0, 0); 165} 166 167int 168__thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, 169 uint32_t *oldceiling) 170{ 171 return _umtx_op_err(mtx, UMTX_OP_SET_CEILING, ceiling, oldceiling, 0); 172} 173 174int 175_thr_umtx_wait(volatile long *mtx, long id, const struct timespec *timeout) 176{ 177 if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 178 timeout->tv_nsec <= 0))) 179 return (ETIMEDOUT); 180 return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAIT, id, 0, 181 __DECONST(void*, timeout)); 182} 183 184int 185_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout, int shared) 186{ 187 if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 188 timeout->tv_nsec <= 0))) 189 return (ETIMEDOUT); 190 return _umtx_op_err(__DEVOLATILE(void *, mtx), 191 shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 0, 192 __DECONST(void*, timeout)); 193} 194 195int 196_thr_umtx_timedwait_uint(volatile u_int *mtx, u_int id, int clockid, 197 const struct timespec *abstime, int shared) 198{ 199 struct _umtx_time *tm_p, timeout; 200 size_t tm_size; 201 202 if (abstime == NULL) { 203 tm_p = NULL; 204 tm_size = 0; 205 } else { 206 timeout._clockid = clockid; 207 timeout._flags = UMTX_ABSTIME; 208 timeout._timeout = *abstime; 209 tm_p = &timeout; 210 tm_size = sizeof(timeout); 211 } 212 213 return _umtx_op_err(__DEVOLATILE(void *, mtx), 214 shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 215 (void *)tm_size, __DECONST(void *, tm_p)); 216} 217 218int 219_thr_umtx_wake(volatile void *mtx, int nr_wakeup, int shared) 220{ 221 return _umtx_op_err(__DEVOLATILE(void *, mtx), shared ? UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE, 222 nr_wakeup, 0, 0); 223} 224 225void 226_thr_ucond_init(struct ucond *cv) 227{ 228 bzero(cv, sizeof(struct ucond)); 229} 230 231int 232_thr_ucond_wait(struct ucond *cv, struct umutex *m, 233 const struct timespec *timeout, int flags) 234{ 235 if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && 236 timeout->tv_nsec <= 0))) { 237 struct pthread *curthread = _get_curthread(); 238 _thr_umutex_unlock(m, TID(curthread)); 239 return (ETIMEDOUT); 240 } 241 return _umtx_op_err(cv, UMTX_OP_CV_WAIT, flags, 242 m, __DECONST(void*, timeout)); 243} 244 245int 246_thr_ucond_signal(struct ucond *cv) 247{ 248 if (!cv->c_has_waiters) 249 return (0); 250 return _umtx_op_err(cv, UMTX_OP_CV_SIGNAL, 0, NULL, NULL); 251} 252 253int 254_thr_ucond_broadcast(struct ucond *cv) 255{ 256 if (!cv->c_has_waiters) 257 return (0); 258 return _umtx_op_err(cv, UMTX_OP_CV_BROADCAST, 0, NULL, NULL); 259} 260 261int 262__thr_rwlock_rdlock(struct urwlock *rwlock, int flags, 263 const struct timespec *tsp) 264{ 265 struct _umtx_time timeout, *tm_p; 266 size_t tm_size; 267 268 if (tsp == NULL) { 269 tm_p = NULL; 270 tm_size = 0; 271 } else { 272 timeout._timeout = *tsp; 273 timeout._flags = UMTX_ABSTIME; 274 timeout._clockid = CLOCK_REALTIME; 275 tm_p = &timeout; 276 tm_size = sizeof(timeout); 277 } 278 return _umtx_op_err(rwlock, UMTX_OP_RW_RDLOCK, flags, (void *)tm_size, tm_p); 279} 280 281int 282__thr_rwlock_wrlock(struct urwlock *rwlock, const struct timespec *tsp) 283{ 284 struct _umtx_time timeout, *tm_p; 285 size_t tm_size; 286 287 if (tsp == NULL) { 288 tm_p = NULL; 289 tm_size = 0; 290 } else { 291 timeout._timeout = *tsp; 292 timeout._flags = UMTX_ABSTIME; 293 timeout._clockid = CLOCK_REALTIME; 294 tm_p = &timeout; 295 tm_size = sizeof(timeout); 296 } 297 return _umtx_op_err(rwlock, UMTX_OP_RW_WRLOCK, 0, (void *)tm_size, tm_p); 298} 299 300int 301__thr_rwlock_unlock(struct urwlock *rwlock) 302{ 303 return _umtx_op_err(rwlock, UMTX_OP_RW_UNLOCK, 0, NULL, NULL); 304} 305 306void 307_thr_rwl_rdlock(struct urwlock *rwlock) 308{ 309 int ret; 310 311 for (;;) { 312 if (_thr_rwlock_tryrdlock(rwlock, URWLOCK_PREFER_READER) == 0) 313 return; 314 ret = __thr_rwlock_rdlock(rwlock, URWLOCK_PREFER_READER, NULL); 315 if (ret == 0) 316 return; 317 if (ret != EINTR) 318 PANIC("rdlock error"); 319 } 320} 321 322void 323_thr_rwl_wrlock(struct urwlock *rwlock) 324{ 325 int ret; 326 327 for (;;) { 328 if (_thr_rwlock_trywrlock(rwlock) == 0) 329 return; 330 ret = __thr_rwlock_wrlock(rwlock, NULL); 331 if (ret == 0) 332 return; 333 if (ret != EINTR) 334 PANIC("wrlock error"); 335 } 336} 337 338void 339_thr_rwl_unlock(struct urwlock *rwlock) 340{ 341 if (_thr_rwlock_unlock(rwlock)) 342 PANIC("unlock error"); 343} 344