1144518Sdavidxu/*- 2144518Sdavidxu * Copyright (c) 2005 David Xu <davidxu@freebsd.org> 3144518Sdavidxu * All rights reserved. 4144518Sdavidxu * 5144518Sdavidxu * Redistribution and use in source and binary forms, with or without 6144518Sdavidxu * modification, are permitted provided that the following conditions 7144518Sdavidxu * are met: 8144518Sdavidxu * 1. Redistributions of source code must retain the above copyright 9144518Sdavidxu * notice, this list of conditions and the following disclaimer. 10144518Sdavidxu * 2. Redistributions in binary form must reproduce the above copyright 11144518Sdavidxu * notice, this list of conditions and the following disclaimer in the 12144518Sdavidxu * documentation and/or other materials provided with the distribution. 13144518Sdavidxu * 14144518Sdavidxu * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15144518Sdavidxu * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16144518Sdavidxu * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17144518Sdavidxu * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18144518Sdavidxu * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19144518Sdavidxu * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20144518Sdavidxu * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21144518Sdavidxu * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22144518Sdavidxu * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23144518Sdavidxu * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24144518Sdavidxu * SUCH DAMAGE. 25144518Sdavidxu * 26144518Sdavidxu * $FreeBSD: stable/11/lib/libthr/thread/thr_umtx.h 319430 2017-06-01 14:49:53Z vangyzen $ 27144518Sdavidxu */ 28144518Sdavidxu 29144518Sdavidxu#ifndef _THR_FBSD_UMTX_H_ 30144518Sdavidxu#define _THR_FBSD_UMTX_H_ 31144518Sdavidxu 32162061Sdavidxu#include <strings.h> 33144518Sdavidxu#include <sys/umtx.h> 34144518Sdavidxu 35300043Skib#ifdef __LP64__ 36300043Skib#define DEFAULT_UMUTEX {0,0,{0,0},0,{0,0}} 37300043Skib#else 38300043Skib#define DEFAULT_UMUTEX {0,0,{0,0},0,0,{0,0}} 39300043Skib#endif 40212077Sdavidxu#define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}} 41162061Sdavidxu 42233912Sdavidxuint _umtx_op_err(void *, int op, u_long, void *, void *) __hidden; 43179970Sdavidxuint __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden; 44216641Sdavidxuint __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden; 45179970Sdavidxuint __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 46161680Sdavidxu const struct timespec *timeout) __hidden; 47319430Svangyzenint __thr_umutex_unlock(struct umutex *mtx) __hidden; 48163334Sdavidxuint __thr_umutex_trylock(struct umutex *mtx) __hidden; 49161680Sdavidxuint __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, 50161680Sdavidxu uint32_t *oldceiling) __hidden; 51161680Sdavidxu 52163334Sdavidxuvoid _thr_umutex_init(struct umutex *mtx) __hidden; 53212077Sdavidxuvoid _thr_urwlock_init(struct urwlock *rwl) __hidden; 54212077Sdavidxu 55173801Sdavidxuint _thr_umtx_wait(volatile long *mtx, long exp, 56162061Sdavidxu const struct timespec *timeout) __hidden; 57173801Sdavidxuint _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp, 58178647Sdavidxu const struct timespec *timeout, int shared) __hidden; 59216641Sdavidxuint _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid, 60216641Sdavidxu const struct timespec *timeout, int shared) __hidden; 61178647Sdavidxuint _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden; 62164877Sdavidxuint _thr_ucond_wait(struct ucond *cv, struct umutex *m, 63249985Sjilles const struct timespec *timeout, int flags) __hidden; 64164902Sdavidxuvoid _thr_ucond_init(struct ucond *cv) __hidden; 65164902Sdavidxuint _thr_ucond_signal(struct ucond *cv) __hidden; 66164902Sdavidxuint _thr_ucond_broadcast(struct ucond *cv) __hidden; 67162061Sdavidxu 68232209Sdavidxuint __thr_rwlock_rdlock(struct urwlock *rwlock, int flags, 69232209Sdavidxu const struct timespec *tsp) __hidden; 70232209Sdavidxuint __thr_rwlock_wrlock(struct urwlock *rwlock, 71232209Sdavidxu const struct timespec *tsp) __hidden; 72177850Sdavidxuint __thr_rwlock_unlock(struct urwlock *rwlock) __hidden; 73177850Sdavidxu 74212076Sdavidxu/* Internal used only */ 75212076Sdavidxuvoid _thr_rwl_rdlock(struct urwlock *rwlock) __hidden; 76212076Sdavidxuvoid _thr_rwl_wrlock(struct urwlock *rwlock) __hidden; 77212076Sdavidxuvoid _thr_rwl_unlock(struct urwlock *rwlock) __hidden; 78212076Sdavidxu 79144518Sdavidxustatic inline int 80161680Sdavidxu_thr_umutex_trylock(struct umutex *mtx, uint32_t id) 81161680Sdavidxu{ 82300043Skib 83300043Skib if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) 84300043Skib return (0); 85300043Skib if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) && 86300043Skib atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD, 87300043Skib id | UMUTEX_CONTESTED)) 88300043Skib return (EOWNERDEAD); 89300043Skib if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV)) 90300043Skib return (ENOTRECOVERABLE); 91300043Skib if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0) 92300043Skib return (EBUSY); 93300043Skib return (__thr_umutex_trylock(mtx)); 94161680Sdavidxu} 95161680Sdavidxu 96161680Sdavidxustatic inline int 97165206Sdavidxu_thr_umutex_trylock2(struct umutex *mtx, uint32_t id) 98165206Sdavidxu{ 99300043Skib 100300043Skib if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0) 101179970Sdavidxu return (0); 102300043Skib if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED && 103300043Skib __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | 104300043Skib UMUTEX_PRIO_INHERIT)) == 0) && 105300043Skib atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, 106300043Skib id | UMUTEX_CONTESTED)) 107300043Skib return (0); 108300043Skib if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) && 109300043Skib atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD, 110300043Skib id | UMUTEX_CONTESTED)) 111300043Skib return (EOWNERDEAD); 112300043Skib if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV)) 113300043Skib return (ENOTRECOVERABLE); 114300043Skib return (EBUSY); 115165206Sdavidxu} 116165206Sdavidxu 117165206Sdavidxustatic inline int 118161680Sdavidxu_thr_umutex_lock(struct umutex *mtx, uint32_t id) 119161680Sdavidxu{ 120300043Skib 121300043Skib if (_thr_umutex_trylock2(mtx, id) == 0) 122300043Skib return (0); 123300043Skib return (__thr_umutex_lock(mtx, id)); 124161680Sdavidxu} 125161680Sdavidxu 126161680Sdavidxustatic inline int 127216641Sdavidxu_thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) 128216641Sdavidxu{ 129300043Skib 130300043Skib if (_thr_umutex_trylock2(mtx, id) == 0) 131300043Skib return (0); 132300043Skib return (__thr_umutex_lock_spin(mtx, id)); 133216641Sdavidxu} 134216641Sdavidxu 135216641Sdavidxustatic inline int 136161680Sdavidxu_thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 137300043Skib const struct timespec *timeout) 138161680Sdavidxu{ 139300043Skib 140300043Skib if (_thr_umutex_trylock2(mtx, id) == 0) 141300043Skib return (0); 142300043Skib return (__thr_umutex_timedlock(mtx, id, timeout)); 143161680Sdavidxu} 144161680Sdavidxu 145161680Sdavidxustatic inline int 146239200Sdavidxu_thr_umutex_unlock2(struct umutex *mtx, uint32_t id, int *defer) 147161680Sdavidxu{ 148300043Skib uint32_t flags, owner; 149300043Skib bool noncst; 150233912Sdavidxu 151300043Skib flags = mtx->m_flags; 152300043Skib noncst = (flags & UMUTEX_NONCONSISTENT) != 0; 153300043Skib 154300043Skib if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) { 155300043Skib if (atomic_cmpset_rel_32(&mtx->m_owner, id, noncst ? 156300043Skib UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED)) 157300043Skib return (0); 158319430Svangyzen return (__thr_umutex_unlock(mtx)); 159233912Sdavidxu } 160300043Skib 161300043Skib do { 162300043Skib owner = mtx->m_owner; 163300043Skib if (__predict_false((owner & ~UMUTEX_CONTESTED) != id)) 164300043Skib return (EPERM); 165300043Skib } while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, owner, 166300043Skib noncst ? UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED))); 167300043Skib if ((owner & UMUTEX_CONTESTED) != 0) { 168300043Skib if (defer == NULL || noncst) 169300043Skib (void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2, 170300043Skib flags, 0, 0); 171300043Skib else 172300043Skib *defer = 1; 173300043Skib } 174300043Skib return (0); 175161680Sdavidxu} 176161680Sdavidxu 177177850Sdavidxustatic inline int 178239200Sdavidxu_thr_umutex_unlock(struct umutex *mtx, uint32_t id) 179239200Sdavidxu{ 180300043Skib 181300043Skib return (_thr_umutex_unlock2(mtx, id, NULL)); 182239200Sdavidxu} 183239200Sdavidxu 184239200Sdavidxustatic inline int 185177850Sdavidxu_thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags) 186177850Sdavidxu{ 187300043Skib int32_t state, wrflags; 188177850Sdavidxu 189300043Skib if ((flags & URWLOCK_PREFER_READER) != 0 || 190300043Skib (rwlock->rw_flags & URWLOCK_PREFER_READER) != 0) 191177850Sdavidxu wrflags = URWLOCK_WRITE_OWNER; 192177850Sdavidxu else 193177850Sdavidxu wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS; 194177850Sdavidxu state = rwlock->rw_state; 195177850Sdavidxu while (!(state & wrflags)) { 196300043Skib if (__predict_false(URWLOCK_READER_COUNT(state) == 197300043Skib URWLOCK_MAX_READERS)) 198177850Sdavidxu return (EAGAIN); 199177850Sdavidxu if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1)) 200177850Sdavidxu return (0); 201177850Sdavidxu state = rwlock->rw_state; 202177850Sdavidxu } 203177850Sdavidxu 204177850Sdavidxu return (EBUSY); 205177850Sdavidxu} 206177850Sdavidxu 207177850Sdavidxustatic inline int 208177850Sdavidxu_thr_rwlock_trywrlock(struct urwlock *rwlock) 209177850Sdavidxu{ 210177850Sdavidxu int32_t state; 211177850Sdavidxu 212177850Sdavidxu state = rwlock->rw_state; 213300043Skib while ((state & URWLOCK_WRITE_OWNER) == 0 && 214300043Skib URWLOCK_READER_COUNT(state) == 0) { 215300043Skib if (atomic_cmpset_acq_32(&rwlock->rw_state, state, 216300043Skib state | URWLOCK_WRITE_OWNER)) 217177850Sdavidxu return (0); 218177850Sdavidxu state = rwlock->rw_state; 219177850Sdavidxu } 220177850Sdavidxu 221177850Sdavidxu return (EBUSY); 222177850Sdavidxu} 223177850Sdavidxu 224177850Sdavidxustatic inline int 225177850Sdavidxu_thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp) 226177850Sdavidxu{ 227300043Skib 228177850Sdavidxu if (_thr_rwlock_tryrdlock(rwlock, flags) == 0) 229177850Sdavidxu return (0); 230177850Sdavidxu return (__thr_rwlock_rdlock(rwlock, flags, tsp)); 231177850Sdavidxu} 232177850Sdavidxu 233177850Sdavidxustatic inline int 234177850Sdavidxu_thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp) 235177850Sdavidxu{ 236300043Skib 237177850Sdavidxu if (_thr_rwlock_trywrlock(rwlock) == 0) 238177850Sdavidxu return (0); 239177850Sdavidxu return (__thr_rwlock_wrlock(rwlock, tsp)); 240177850Sdavidxu} 241177850Sdavidxu 242177850Sdavidxustatic inline int 243177850Sdavidxu_thr_rwlock_unlock(struct urwlock *rwlock) 244177850Sdavidxu{ 245177850Sdavidxu int32_t state; 246177850Sdavidxu 247177850Sdavidxu state = rwlock->rw_state; 248300043Skib if ((state & URWLOCK_WRITE_OWNER) != 0) { 249300043Skib if (atomic_cmpset_rel_32(&rwlock->rw_state, 250300043Skib URWLOCK_WRITE_OWNER, 0)) 251177850Sdavidxu return (0); 252177850Sdavidxu } else { 253177850Sdavidxu for (;;) { 254177850Sdavidxu if (__predict_false(URWLOCK_READER_COUNT(state) == 0)) 255177850Sdavidxu return (EPERM); 256197445Sattilio if (!((state & (URWLOCK_WRITE_WAITERS | 257300043Skib URWLOCK_READ_WAITERS)) != 0 && 258197445Sattilio URWLOCK_READER_COUNT(state) == 1)) { 259197445Sattilio if (atomic_cmpset_rel_32(&rwlock->rw_state, 260300043Skib state, state - 1)) 261177850Sdavidxu return (0); 262177850Sdavidxu state = rwlock->rw_state; 263177850Sdavidxu } else { 264177850Sdavidxu break; 265177850Sdavidxu } 266177850Sdavidxu } 267177850Sdavidxu } 268177850Sdavidxu return (__thr_rwlock_unlock(rwlock)); 269177850Sdavidxu} 270144518Sdavidxu#endif 271