thr_umtx.h revision 233912
150974Swpaul/*- 250974Swpaul * Copyright (c) 2005 David Xu <davidxu@freebsd.org> 350974Swpaul * All rights reserved. 450974Swpaul * 550974Swpaul * Redistribution and use in source and binary forms, with or without 650974Swpaul * modification, are permitted provided that the following conditions 750974Swpaul * are met: 850974Swpaul * 1. Redistributions of source code must retain the above copyright 950974Swpaul * notice, this list of conditions and the following disclaimer. 1050974Swpaul * 2. Redistributions in binary form must reproduce the above copyright 1150974Swpaul * notice, this list of conditions and the following disclaimer in the 1250974Swpaul * documentation and/or other materials provided with the distribution. 1350974Swpaul * 1450974Swpaul * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1550974Swpaul * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1650974Swpaul * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1750974Swpaul * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1850974Swpaul * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1950974Swpaul * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2050974Swpaul * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2150974Swpaul * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2250974Swpaul * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2350974Swpaul * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2450974Swpaul * SUCH DAMAGE. 2550974Swpaul * 2650974Swpaul * $FreeBSD: head/lib/libthr/thread/thr_umtx.h 233912 2012-04-05 02:24:08Z davidxu $ 2750974Swpaul */ 2850974Swpaul 2950974Swpaul#ifndef _THR_FBSD_UMTX_H_ 3050974Swpaul#define _THR_FBSD_UMTX_H_ 3150974Swpaul 3250974Swpaul#include <strings.h> 33122678Sobrien#include <sys/umtx.h> 34122678Sobrien 35122678Sobrien#define DEFAULT_UMUTEX {0,0,{0,0},{0,0,0,0}} 3650974Swpaul#define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}} 3750974Swpaul 3850974Swpaulint _umtx_op_err(void *, int op, u_long, void *, void *) __hidden; 3950974Swpaulint __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden; 4064963Swpaulint __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden; 4164963Swpaulint __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 4264963Swpaul const struct timespec *timeout) __hidden; 4350974Swpaulint __thr_umutex_unlock(struct umutex *mtx, uint32_t id) __hidden; 4450974Swpaulint __thr_umutex_trylock(struct umutex *mtx) __hidden; 4550974Swpaulint __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, 4650974Swpaul uint32_t *oldceiling) __hidden; 4750974Swpaul 4850974Swpaulvoid _thr_umutex_init(struct umutex *mtx) __hidden; 4950974Swpaulvoid _thr_urwlock_init(struct urwlock *rwl) __hidden; 5050974Swpaul 5150974Swpaulint _thr_umtx_wait(volatile long *mtx, long exp, 5250974Swpaul const struct timespec *timeout) __hidden; 5350974Swpaulint _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp, 5450974Swpaul const struct timespec *timeout, int shared) __hidden; 5550974Swpaulint _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid, 5650974Swpaul const struct timespec *timeout, int shared) __hidden; 5750974Swpaulint _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden; 5850974Swpaulint _thr_ucond_wait(struct ucond *cv, struct umutex *m, 5950974Swpaul const struct timespec *timeout, int check_unpaking) __hidden; 6050974Swpaulvoid _thr_ucond_init(struct ucond *cv) __hidden; 6150974Swpaulint _thr_ucond_signal(struct ucond *cv) __hidden; 6250974Swpaulint _thr_ucond_broadcast(struct ucond *cv) __hidden; 6350974Swpaul 6450974Swpaulint __thr_rwlock_rdlock(struct urwlock *rwlock, int flags, 6550974Swpaul const struct timespec *tsp) __hidden; 66129876Sphkint __thr_rwlock_wrlock(struct urwlock *rwlock, 6750974Swpaul const struct timespec *tsp) __hidden; 6887059Sluigiint __thr_rwlock_unlock(struct urwlock *rwlock) __hidden; 6950974Swpaul 7050974Swpaul/* Internal used only */ 7150974Swpaulvoid _thr_rwl_rdlock(struct urwlock *rwlock) __hidden; 7250974Swpaulvoid _thr_rwl_wrlock(struct urwlock *rwlock) __hidden; 7350974Swpaulvoid _thr_rwl_unlock(struct urwlock *rwlock) __hidden; 7450974Swpaul 7587390Sjhaystatic inline int 7687390Sjhay_thr_umutex_trylock(struct umutex *mtx, uint32_t id) 7750974Swpaul{ 7850974Swpaul if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) 7950974Swpaul return (0); 8050974Swpaul if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0) 8150974Swpaul return (EBUSY); 8250974Swpaul return (__thr_umutex_trylock(mtx)); 8350974Swpaul} 8450974Swpaul 8550974Swpaulstatic inline int 8650974Swpaul_thr_umutex_trylock2(struct umutex *mtx, uint32_t id) 8750974Swpaul{ 8850974Swpaul if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0) 8950974Swpaul return (0); 90119288Simp if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED && 91119288Simp __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0)) 9250974Swpaul if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED)) 9350974Swpaul return (0); 9450974Swpaul return (EBUSY); 9550974Swpaul} 9650974Swpaul 97113506Smdoddstatic inline int 98113506Smdodd_thr_umutex_lock(struct umutex *mtx, uint32_t id) 9959758Speter{ 10059758Speter if (_thr_umutex_trylock2(mtx, id) == 0) 10151089Speter return (0); 10250974Swpaul return (__thr_umutex_lock(mtx, id)); 10350974Swpaul} 10450974Swpaul 10550974Swpaulstatic inline int 10650974Swpaul_thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) 10750974Swpaul{ 10850974Swpaul if (_thr_umutex_trylock2(mtx, id) == 0) 10950974Swpaul return (0); 110119712Sphk return (__thr_umutex_lock_spin(mtx, id)); 11150974Swpaul} 11250974Swpaul 11350974Swpaulstatic inline int 11492739Salfred_thr_umutex_timedlock(struct umutex *mtx, uint32_t id, 11592739Salfred const struct timespec *timeout) 11692739Salfred{ 11750974Swpaul if (_thr_umutex_trylock2(mtx, id) == 0) 11892739Salfred return (0); 11992739Salfred return (__thr_umutex_timedlock(mtx, id, timeout)); 12092739Salfred} 121121262Ssilby 12292739Salfredstatic inline int 12392739Salfred_thr_umutex_unlock(struct umutex *mtx, uint32_t id) 12492739Salfred{ 12592739Salfred uint32_t flags = mtx->m_flags; 12692739Salfred 12792739Salfred if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { 12892739Salfred uint32_t owner; 12992739Salfred do { 13092739Salfred owner = mtx->m_owner; 13192739Salfred if (__predict_false((owner & ~UMUTEX_CONTESTED) != id)) 13292739Salfred return (EPERM); 13392739Salfred } while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, 13492739Salfred owner, UMUTEX_UNOWNED))); 13550974Swpaul if ((owner & UMUTEX_CONTESTED)) 13692739Salfred (void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2, flags, 0, 0); 13792739Salfred return (0); 13892739Salfred } 13992739Salfred if (atomic_cmpset_rel_32(&mtx->m_owner, id, UMUTEX_UNOWNED)) 14092739Salfred return (0); 14192739Salfred return (__thr_umutex_unlock(mtx, id)); 14272197Swpaul} 14392739Salfred 14492739Salfredstatic inline int 14592739Salfred_thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags) 14692739Salfred{ 14772197Swpaul int32_t state; 14872197Swpaul int32_t wrflags; 149109060Smbr 150109060Smbr if (flags & URWLOCK_PREFER_READER || rwlock->rw_flags & URWLOCK_PREFER_READER) 151109060Smbr wrflags = URWLOCK_WRITE_OWNER; 152109060Smbr else 15392739Salfred wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS; 15492739Salfred state = rwlock->rw_state; 15592739Salfred while (!(state & wrflags)) { 15650974Swpaul if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) 15792739Salfred return (EAGAIN); 15892739Salfred if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1)) 159123289Sobrien return (0); 16092739Salfred state = rwlock->rw_state; 16192739Salfred } 16292739Salfred 16350974Swpaul return (EBUSY); 16492739Salfred} 16592739Salfred 16692739Salfredstatic inline int 16750974Swpaul_thr_rwlock_trywrlock(struct urwlock *rwlock) 16850974Swpaul{ 16950974Swpaul int32_t state; 17050974Swpaul 17151030Swpaul state = rwlock->rw_state; 17251030Swpaul while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { 17350974Swpaul if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER)) 17450974Swpaul return (0); 17550974Swpaul state = rwlock->rw_state; 17650974Swpaul } 17750974Swpaul 17850974Swpaul return (EBUSY); 17950974Swpaul} 18050974Swpaul 18150974Swpaulstatic inline int 18250974Swpaul_thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp) 18350974Swpaul{ 18450974Swpaul if (_thr_rwlock_tryrdlock(rwlock, flags) == 0) 18550974Swpaul return (0); 18650974Swpaul return (__thr_rwlock_rdlock(rwlock, flags, tsp)); 18750974Swpaul} 18850974Swpaul 18950974Swpaulstatic inline int 19050974Swpaul_thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp) 19150974Swpaul{ 19250974Swpaul if (_thr_rwlock_trywrlock(rwlock) == 0) 19350974Swpaul return (0); 19450974Swpaul return (__thr_rwlock_wrlock(rwlock, tsp)); 19551455Swpaul} 19650974Swpaul 19750974Swpaulstatic inline int 19850974Swpaul_thr_rwlock_unlock(struct urwlock *rwlock) 19950974Swpaul{ 20050974Swpaul int32_t state; 20150974Swpaul 202113506Smdodd state = rwlock->rw_state; 20351473Swpaul if (state & URWLOCK_WRITE_OWNER) { 20450974Swpaul if (atomic_cmpset_rel_32(&rwlock->rw_state, URWLOCK_WRITE_OWNER, 0)) 20550974Swpaul return (0); 20650974Swpaul } else { 20750974Swpaul for (;;) { 20850974Swpaul if (__predict_false(URWLOCK_READER_COUNT(state) == 0)) 20950974Swpaul return (EPERM); 21050974Swpaul if (!((state & (URWLOCK_WRITE_WAITERS | 21150974Swpaul URWLOCK_READ_WAITERS)) && 21250974Swpaul URWLOCK_READER_COUNT(state) == 1)) { 21350974Swpaul if (atomic_cmpset_rel_32(&rwlock->rw_state, 21450974Swpaul state, state-1)) 21550974Swpaul return (0); 21650974Swpaul state = rwlock->rw_state; 21750974Swpaul } else { 21850974Swpaul break; 21981713Swpaul } 22081713Swpaul } 22181713Swpaul } 22281713Swpaul return (__thr_rwlock_unlock(rwlock)); 22381713Swpaul} 22481713Swpaul#endif 22581713Swpaul