113546Sjulian/* 213546Sjulian * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. 313546Sjulian * All rights reserved. 413546Sjulian * 513546Sjulian * Redistribution and use in source and binary forms, with or without 613546Sjulian * modification, are permitted provided that the following conditions 713546Sjulian * are met: 813546Sjulian * 1. Redistributions of source code must retain the above copyright 913546Sjulian * notice, this list of conditions and the following disclaimer. 1013546Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1113546Sjulian * notice, this list of conditions and the following disclaimer in the 1213546Sjulian * documentation and/or other materials provided with the distribution. 13165967Simp * 3. Neither the name of the author nor the names of any co-contributors 1413546Sjulian * may be used to endorse or promote products derived from this software 1513546Sjulian * without specific prior written permission. 1613546Sjulian * 1713546Sjulian * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND 1813546Sjulian * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1913546Sjulian * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2044963Sjb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2113546Sjulian * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2213546Sjulian * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2313546Sjulian * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2413546Sjulian * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2513546Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2613546Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2713546Sjulian * SUCH DAMAGE. 2813546Sjulian * 2950476Speter * $FreeBSD$ 3013546Sjulian */ 31174112Sdeischen 32174112Sdeischen#include "namespace.h" 3313546Sjulian#include <stdlib.h> 3413546Sjulian#include <errno.h> 3536830Sjb#include <string.h> 3644963Sjb#include <sys/param.h> 3744963Sjb#include <sys/queue.h> 3813546Sjulian#include <pthread.h> 39176071Sdes#include <pthread_np.h> 40174112Sdeischen#include "un-namespace.h" 41103388Smini#include "thr_private.h" 4213546Sjulian 4348046Sjb#if defined(_PTHREADS_INVARIANTS) 44113658Sdeischen#define MUTEX_INIT_LINK(m) do { \ 4548046Sjb (m)->m_qe.tqe_prev = NULL; \ 4648046Sjb (m)->m_qe.tqe_next = NULL; \ 4748046Sjb} while (0) 48113658Sdeischen#define MUTEX_ASSERT_IS_OWNED(m) do { \ 4948046Sjb if ((m)->m_qe.tqe_prev == NULL) \ 5048046Sjb PANIC("mutex is not on list"); \ 5148046Sjb} while (0) 52113658Sdeischen#define MUTEX_ASSERT_NOT_OWNED(m) do { \ 5348046Sjb if (((m)->m_qe.tqe_prev != NULL) || \ 5448046Sjb ((m)->m_qe.tqe_next != NULL)) \ 5548046Sjb PANIC("mutex is on list"); \ 5648046Sjb} while (0) 57113658Sdeischen#define THR_ASSERT_NOT_IN_SYNCQ(thr) do { \ 58113658Sdeischen THR_ASSERT(((thr)->sflags & THR_FLAGS_IN_SYNCQ) == 0, \ 59113658Sdeischen "thread in syncq when it shouldn't be."); \ 60113658Sdeischen} while (0); 6148046Sjb#else 62113658Sdeischen#define MUTEX_INIT_LINK(m) 63113658Sdeischen#define MUTEX_ASSERT_IS_OWNED(m) 64113658Sdeischen#define MUTEX_ASSERT_NOT_OWNED(m) 65113658Sdeischen#define THR_ASSERT_NOT_IN_SYNCQ(thr) 6648046Sjb#endif 6744963Sjb 68119736Sdavidxu#define THR_IN_MUTEXQ(thr) (((thr)->sflags & THR_FLAGS_IN_SYNCQ) != 0) 69122070Sdeischen#define MUTEX_DESTROY(m) do { \ 70122070Sdeischen _lock_destroy(&(m)->m_lock); \ 71122070Sdeischen free(m); \ 72122070Sdeischen} while (0) 73119736Sdavidxu 74122070Sdeischen 7544963Sjb/* 7644963Sjb * Prototypes 7744963Sjb */ 78117907Sdeischenstatic struct kse_mailbox *mutex_handoff(struct pthread *, 79117907Sdeischen struct pthread_mutex *); 80174112Sdeischenstatic inline int mutex_self_trylock(pthread_mutex_t); 81113658Sdeischenstatic inline int mutex_self_lock(struct pthread *, pthread_mutex_t); 82113658Sdeischenstatic int mutex_unlock_common(pthread_mutex_t *, int); 83113658Sdeischenstatic void mutex_priority_adjust(struct pthread *, pthread_mutex_t); 84113658Sdeischenstatic void mutex_rescan_owned (struct pthread *, struct pthread *, 85113658Sdeischen struct pthread_mutex *); 8644963Sjbstatic inline pthread_t mutex_queue_deq(pthread_mutex_t); 8744963Sjbstatic inline void mutex_queue_remove(pthread_mutex_t, pthread_t); 8844963Sjbstatic inline void mutex_queue_enq(pthread_mutex_t, pthread_t); 89139023Sdeischenstatic void mutex_lock_backout(void *arg); 9044963Sjb 91174112Sdeischenint __pthread_mutex_init(pthread_mutex_t *mutex, 92174112Sdeischen const pthread_mutexattr_t *mutex_attr); 93174112Sdeischenint __pthread_mutex_trylock(pthread_mutex_t *mutex); 94174112Sdeischenint __pthread_mutex_lock(pthread_mutex_t *m); 95174112Sdeischenint __pthread_mutex_timedlock(pthread_mutex_t *m, 96174112Sdeischen const struct timespec *abs_timeout); 97174112Sdeischenint _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 98174112Sdeischen void *(calloc_cb)(size_t, size_t)); 99174112Sdeischen 100174112Sdeischen 10171581Sdeischenstatic struct pthread_mutex_attr static_mutex_attr = 10271581Sdeischen PTHREAD_MUTEXATTR_STATIC_INITIALIZER; 10371581Sdeischenstatic pthread_mutexattr_t static_mattr = &static_mutex_attr; 10471581Sdeischen 10571581Sdeischen/* Single underscore versions provided for libc internal usage: */ 106139023Sdeischen__weak_reference(__pthread_mutex_init, pthread_mutex_init); 107113658Sdeischen__weak_reference(__pthread_mutex_lock, pthread_mutex_lock); 108119736Sdavidxu__weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); 10975369Sdeischen__weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); 11071581Sdeischen 11171581Sdeischen/* No difference between libc and application usage of these: */ 11275369Sdeischen__weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy); 11375369Sdeischen__weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock); 114176060Sdes__weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np); 11571581Sdeischen 116173967Sjasonestatic int 117173967Sjasonethr_mutex_init(pthread_mutex_t *mutex, 118173967Sjasone const pthread_mutexattr_t *mutex_attr, void *(calloc_cb)(size_t, size_t)) 11913546Sjulian{ 120113658Sdeischen struct pthread_mutex *pmutex; 121113658Sdeischen enum pthread_mutextype type; 12244963Sjb int protocol; 12344963Sjb int ceiling; 12473057Sdeischen int flags; 12567097Sdeischen int ret = 0; 12613546Sjulian 12744963Sjb if (mutex == NULL) 12831402Salex ret = EINVAL; 12935509Sjb 13044963Sjb /* Check if default mutex attributes: */ 13144963Sjb else if (mutex_attr == NULL || *mutex_attr == NULL) { 13244963Sjb /* Default to a (error checking) POSIX mutex: */ 13344963Sjb type = PTHREAD_MUTEX_ERRORCHECK; 13444963Sjb protocol = PTHREAD_PRIO_NONE; 135113658Sdeischen ceiling = THR_MAX_PRIORITY; 13673057Sdeischen flags = 0; 13744963Sjb } 13817706Sjulian 13944963Sjb /* Check mutex type: */ 14044963Sjb else if (((*mutex_attr)->m_type < PTHREAD_MUTEX_ERRORCHECK) || 141149298Sstefanf ((*mutex_attr)->m_type >= PTHREAD_MUTEX_TYPE_MAX)) 14244963Sjb /* Return an invalid argument error: */ 14344963Sjb ret = EINVAL; 14417706Sjulian 14544963Sjb /* Check mutex protocol: */ 14644963Sjb else if (((*mutex_attr)->m_protocol < PTHREAD_PRIO_NONE) || 14744963Sjb ((*mutex_attr)->m_protocol > PTHREAD_MUTEX_RECURSIVE)) 14844963Sjb /* Return an invalid argument error: */ 14944963Sjb ret = EINVAL; 15017706Sjulian 15144963Sjb else { 15244963Sjb /* Use the requested mutex type and protocol: */ 15344963Sjb type = (*mutex_attr)->m_type; 15444963Sjb protocol = (*mutex_attr)->m_protocol; 15544963Sjb ceiling = (*mutex_attr)->m_ceiling; 15673057Sdeischen flags = (*mutex_attr)->m_flags; 15744963Sjb } 15817706Sjulian 15944963Sjb /* Check no errors so far: */ 16044963Sjb if (ret == 0) { 16144963Sjb if ((pmutex = (pthread_mutex_t) 162173967Sjasone calloc_cb(1, sizeof(struct pthread_mutex))) == NULL) 16344963Sjb ret = ENOMEM; 164113658Sdeischen else if (_lock_init(&pmutex->m_lock, LCK_ADAPTIVE, 165173967Sjasone _thr_lock_wait, _thr_lock_wakeup, calloc_cb) != 0) { 166113658Sdeischen free(pmutex); 167113658Sdeischen *mutex = NULL; 168113658Sdeischen ret = ENOMEM; 169113658Sdeischen } else { 17073057Sdeischen /* Set the mutex flags: */ 17173057Sdeischen pmutex->m_flags = flags; 17244963Sjb 17344963Sjb /* Process according to mutex type: */ 17444963Sjb switch (type) { 17544963Sjb /* case PTHREAD_MUTEX_DEFAULT: */ 17644963Sjb case PTHREAD_MUTEX_ERRORCHECK: 17744963Sjb case PTHREAD_MUTEX_NORMAL: 178173154Skris case PTHREAD_MUTEX_ADAPTIVE_NP: 17944963Sjb /* Nothing to do here. */ 18044963Sjb break; 18144963Sjb 18244963Sjb /* Single UNIX Spec 2 recursive mutex: */ 18344963Sjb case PTHREAD_MUTEX_RECURSIVE: 18444963Sjb /* Reset the mutex count: */ 185113658Sdeischen pmutex->m_count = 0; 18644963Sjb break; 18744963Sjb 18844963Sjb /* Trap invalid mutex types: */ 18944963Sjb default: 19044963Sjb /* Return an invalid argument error: */ 19144963Sjb ret = EINVAL; 19244963Sjb break; 19317706Sjulian } 19444963Sjb if (ret == 0) { 19544963Sjb /* Initialise the rest of the mutex: */ 19644963Sjb TAILQ_INIT(&pmutex->m_queue); 19744963Sjb pmutex->m_flags |= MUTEX_FLAGS_INITED; 19844963Sjb pmutex->m_owner = NULL; 19944963Sjb pmutex->m_type = type; 20044963Sjb pmutex->m_protocol = protocol; 20144963Sjb pmutex->m_refcount = 0; 20244963Sjb if (protocol == PTHREAD_PRIO_PROTECT) 20344963Sjb pmutex->m_prio = ceiling; 20444963Sjb else 205113658Sdeischen pmutex->m_prio = -1; 20644963Sjb pmutex->m_saved_prio = 0; 207113658Sdeischen MUTEX_INIT_LINK(pmutex); 20844963Sjb *mutex = pmutex; 20944963Sjb } else { 210115761Sdavidxu /* Free the mutex lock structure: */ 211122070Sdeischen MUTEX_DESTROY(pmutex); 21244963Sjb *mutex = NULL; 21344963Sjb } 21417706Sjulian } 21513546Sjulian } 21617706Sjulian /* Return the completion status: */ 217113658Sdeischen return (ret); 21817706Sjulian} 21913546Sjulian 220139023Sdeischenint 221173967Sjasone__pthread_mutex_init(pthread_mutex_t *mutex, 222173967Sjasone const pthread_mutexattr_t *mutex_attr) 223173967Sjasone{ 224173967Sjasone 225173967Sjasone return (thr_mutex_init(mutex, mutex_attr, calloc)); 226173967Sjasone} 227173967Sjasone 228173967Sjasoneint 229139023Sdeischen_pthread_mutex_init(pthread_mutex_t *mutex, 230139023Sdeischen const pthread_mutexattr_t *mutex_attr) 231139023Sdeischen{ 232139023Sdeischen struct pthread_mutex_attr mattr, *mattrp; 233139023Sdeischen 234139023Sdeischen if ((mutex_attr == NULL) || (*mutex_attr == NULL)) 235139023Sdeischen return (__pthread_mutex_init(mutex, &static_mattr)); 236139023Sdeischen else { 237139023Sdeischen mattr = **mutex_attr; 238139023Sdeischen mattr.m_flags |= MUTEX_FLAGS_PRIVATE; 239139023Sdeischen mattrp = &mattr; 240139023Sdeischen return (__pthread_mutex_init(mutex, &mattrp)); 241139023Sdeischen } 242139023Sdeischen} 243139023Sdeischen 244173967Sjasone/* This function is used internally by malloc. */ 245173967Sjasoneint 246173967Sjasone_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 247173967Sjasone void *(calloc_cb)(size_t, size_t)) 248173967Sjasone{ 249173967Sjasone static const struct pthread_mutex_attr attr = { 250173967Sjasone .m_type = PTHREAD_MUTEX_NORMAL, 251173967Sjasone .m_protocol = PTHREAD_PRIO_NONE, 252173967Sjasone .m_ceiling = 0, 253173967Sjasone .m_flags = 0 254173967Sjasone }; 255174001Sjasone static const struct pthread_mutex_attr *pattr = &attr; 256173967Sjasone 257174001Sjasone return (thr_mutex_init(mutex, (pthread_mutexattr_t *)&pattr, 258173967Sjasone calloc_cb)); 259173967Sjasone} 260173967Sjasone 261122070Sdeischenvoid 262122070Sdeischen_thr_mutex_reinit(pthread_mutex_t *mutex) 263122070Sdeischen{ 264122070Sdeischen _lock_reinit(&(*mutex)->m_lock, LCK_ADAPTIVE, 265122070Sdeischen _thr_lock_wait, _thr_lock_wakeup); 266122070Sdeischen TAILQ_INIT(&(*mutex)->m_queue); 267122070Sdeischen (*mutex)->m_owner = NULL; 268122070Sdeischen (*mutex)->m_count = 0; 269122070Sdeischen (*mutex)->m_refcount = 0; 270122070Sdeischen (*mutex)->m_prio = 0; 271122070Sdeischen (*mutex)->m_saved_prio = 0; 272122070Sdeischen} 273122070Sdeischen 27417706Sjulianint 275113658Sdeischen_pthread_mutex_destroy(pthread_mutex_t *mutex) 27617706Sjulian{ 277113658Sdeischen struct pthread *curthread = _get_curthread(); 278113658Sdeischen pthread_mutex_t m; 279113658Sdeischen int ret = 0; 28013546Sjulian 28135509Sjb if (mutex == NULL || *mutex == NULL) 28231402Salex ret = EINVAL; 28335509Sjb else { 28435509Sjb /* Lock the mutex structure: */ 285113658Sdeischen THR_LOCK_ACQUIRE(curthread, &(*mutex)->m_lock); 28613546Sjulian 28735509Sjb /* 28844963Sjb * Check to see if this mutex is in use: 28935509Sjb */ 29044963Sjb if (((*mutex)->m_owner != NULL) || 291157700Sdelphij (!TAILQ_EMPTY(&(*mutex)->m_queue)) || 29244963Sjb ((*mutex)->m_refcount != 0)) { 29344963Sjb ret = EBUSY; 29413546Sjulian 29544963Sjb /* Unlock the mutex structure: */ 296113658Sdeischen THR_LOCK_RELEASE(curthread, &(*mutex)->m_lock); 297113658Sdeischen } else { 29844963Sjb /* 299113658Sdeischen * Save a pointer to the mutex so it can be free'd 300113658Sdeischen * and set the caller's pointer to NULL: 30144963Sjb */ 302113658Sdeischen m = *mutex; 303113658Sdeischen *mutex = NULL; 30444963Sjb 305113658Sdeischen /* Unlock the mutex structure: */ 306113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 307113658Sdeischen 30844963Sjb /* 309113658Sdeischen * Free the memory allocated for the mutex 310113658Sdeischen * structure: 31144963Sjb */ 312113658Sdeischen MUTEX_ASSERT_NOT_OWNED(m); 313122070Sdeischen MUTEX_DESTROY(m); 31444963Sjb } 31513546Sjulian } 31613546Sjulian 31713546Sjulian /* Return the completion status: */ 31813546Sjulian return (ret); 31913546Sjulian} 32013546Sjulian 32138025Salexstatic int 322113658Sdeischeninit_static(struct pthread *thread, pthread_mutex_t *mutex) 32338025Salex{ 324113658Sdeischen int ret; 32538025Salex 326113658Sdeischen THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 32738025Salex 32838027Salex if (*mutex == NULL) 329174112Sdeischen ret = _pthread_mutex_init(mutex, NULL); 33038025Salex else 33138025Salex ret = 0; 33238025Salex 333113658Sdeischen THR_LOCK_RELEASE(thread, &_mutex_static_lock); 33438025Salex 335113658Sdeischen return (ret); 33638025Salex} 33738025Salex 33871581Sdeischenstatic int 339113658Sdeischeninit_static_private(struct pthread *thread, pthread_mutex_t *mutex) 34013546Sjulian{ 341113658Sdeischen int ret; 34271581Sdeischen 343113658Sdeischen THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); 34471581Sdeischen 34571581Sdeischen if (*mutex == NULL) 346174112Sdeischen ret = _pthread_mutex_init(mutex, &static_mattr); 34771581Sdeischen else 34871581Sdeischen ret = 0; 34971581Sdeischen 350113658Sdeischen THR_LOCK_RELEASE(thread, &_mutex_static_lock); 35171581Sdeischen 352113658Sdeischen return (ret); 35371581Sdeischen} 35471581Sdeischen 35571581Sdeischenstatic int 356113658Sdeischenmutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex) 35771581Sdeischen{ 358139023Sdeischen int private; 359113658Sdeischen int ret = 0; 36013546Sjulian 361113658Sdeischen THR_ASSERT((mutex != NULL) && (*mutex != NULL), 36271581Sdeischen "Uninitialized mutex in pthread_mutex_trylock_basic"); 36335027Sjb 36471581Sdeischen /* Lock the mutex structure: */ 365113658Sdeischen THR_LOCK_ACQUIRE(curthread, &(*mutex)->m_lock); 366139023Sdeischen private = (*mutex)->m_flags & MUTEX_FLAGS_PRIVATE; 36748046Sjb 36871581Sdeischen /* 36971581Sdeischen * If the mutex was statically allocated, properly 37071581Sdeischen * initialize the tail queue. 37171581Sdeischen */ 37271581Sdeischen if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) { 37371581Sdeischen TAILQ_INIT(&(*mutex)->m_queue); 374113658Sdeischen MUTEX_INIT_LINK(*mutex); 37571581Sdeischen (*mutex)->m_flags |= MUTEX_FLAGS_INITED; 37671581Sdeischen } 37747424Sjb 37871581Sdeischen /* Process according to mutex type: */ 37971581Sdeischen switch ((*mutex)->m_protocol) { 38071581Sdeischen /* Default POSIX mutex: */ 38171581Sdeischen case PTHREAD_PRIO_NONE: 38271581Sdeischen /* Check if this mutex is not locked: */ 38371581Sdeischen if ((*mutex)->m_owner == NULL) { 38471581Sdeischen /* Lock the mutex for the running thread: */ 38571581Sdeischen (*mutex)->m_owner = curthread; 38644963Sjb 38771581Sdeischen /* Add to the list of owned mutexes: */ 388113658Sdeischen MUTEX_ASSERT_NOT_OWNED(*mutex); 38971581Sdeischen TAILQ_INSERT_TAIL(&curthread->mutexq, 39071581Sdeischen (*mutex), m_qe); 39171581Sdeischen } else if ((*mutex)->m_owner == curthread) 392174112Sdeischen ret = mutex_self_trylock(*mutex); 39371581Sdeischen else 39471581Sdeischen /* Return a busy error: */ 39571581Sdeischen ret = EBUSY; 39671581Sdeischen break; 39717706Sjulian 39871581Sdeischen /* POSIX priority inheritence mutex: */ 39971581Sdeischen case PTHREAD_PRIO_INHERIT: 40071581Sdeischen /* Check if this mutex is not locked: */ 40171581Sdeischen if ((*mutex)->m_owner == NULL) { 40271581Sdeischen /* Lock the mutex for the running thread: */ 40371581Sdeischen (*mutex)->m_owner = curthread; 40444963Sjb 405114187Sdeischen THR_SCHED_LOCK(curthread, curthread); 40671581Sdeischen /* Track number of priority mutexes owned: */ 40771581Sdeischen curthread->priority_mutex_count++; 40844963Sjb 40971581Sdeischen /* 41071581Sdeischen * The mutex takes on the attributes of the 41171581Sdeischen * running thread when there are no waiters. 41271581Sdeischen */ 41371581Sdeischen (*mutex)->m_prio = curthread->active_priority; 41471581Sdeischen (*mutex)->m_saved_prio = 41571581Sdeischen curthread->inherited_priority; 416120403Sdavidxu curthread->inherited_priority = (*mutex)->m_prio; 417114187Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 41844963Sjb 41971581Sdeischen /* Add to the list of owned mutexes: */ 420113658Sdeischen MUTEX_ASSERT_NOT_OWNED(*mutex); 42171581Sdeischen TAILQ_INSERT_TAIL(&curthread->mutexq, 42271581Sdeischen (*mutex), m_qe); 42371581Sdeischen } else if ((*mutex)->m_owner == curthread) 424174112Sdeischen ret = mutex_self_trylock(*mutex); 42571581Sdeischen else 42671581Sdeischen /* Return a busy error: */ 42771581Sdeischen ret = EBUSY; 42871581Sdeischen break; 42944963Sjb 43071581Sdeischen /* POSIX priority protection mutex: */ 43171581Sdeischen case PTHREAD_PRIO_PROTECT: 43271581Sdeischen /* Check for a priority ceiling violation: */ 43371581Sdeischen if (curthread->active_priority > (*mutex)->m_prio) 43471581Sdeischen ret = EINVAL; 43544963Sjb 43671581Sdeischen /* Check if this mutex is not locked: */ 43771581Sdeischen else if ((*mutex)->m_owner == NULL) { 43871581Sdeischen /* Lock the mutex for the running thread: */ 43971581Sdeischen (*mutex)->m_owner = curthread; 44044963Sjb 441114187Sdeischen THR_SCHED_LOCK(curthread, curthread); 44271581Sdeischen /* Track number of priority mutexes owned: */ 44371581Sdeischen curthread->priority_mutex_count++; 44444963Sjb 44571581Sdeischen /* 44671581Sdeischen * The running thread inherits the ceiling 44771581Sdeischen * priority of the mutex and executes at that 44871581Sdeischen * priority. 44971581Sdeischen */ 45071581Sdeischen curthread->active_priority = (*mutex)->m_prio; 45171581Sdeischen (*mutex)->m_saved_prio = 45271581Sdeischen curthread->inherited_priority; 45371581Sdeischen curthread->inherited_priority = 45471581Sdeischen (*mutex)->m_prio; 455114187Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 45671581Sdeischen /* Add to the list of owned mutexes: */ 457113658Sdeischen MUTEX_ASSERT_NOT_OWNED(*mutex); 45871581Sdeischen TAILQ_INSERT_TAIL(&curthread->mutexq, 45971581Sdeischen (*mutex), m_qe); 46071581Sdeischen } else if ((*mutex)->m_owner == curthread) 461174112Sdeischen ret = mutex_self_trylock(*mutex); 46271581Sdeischen else 46371581Sdeischen /* Return a busy error: */ 46471581Sdeischen ret = EBUSY; 46571581Sdeischen break; 46617706Sjulian 46771581Sdeischen /* Trap invalid mutex types: */ 46871581Sdeischen default: 46971581Sdeischen /* Return an invalid argument error: */ 47071581Sdeischen ret = EINVAL; 47171581Sdeischen break; 47271581Sdeischen } 47313546Sjulian 474139023Sdeischen if (ret == 0 && private) 475139023Sdeischen THR_CRITICAL_ENTER(curthread); 476139023Sdeischen 47771581Sdeischen /* Unlock the mutex structure: */ 478113658Sdeischen THR_LOCK_RELEASE(curthread, &(*mutex)->m_lock); 47944963Sjb 48013546Sjulian /* Return the completion status: */ 48113546Sjulian return (ret); 48213546Sjulian} 48313546Sjulian 48413546Sjulianint 48571581Sdeischen__pthread_mutex_trylock(pthread_mutex_t *mutex) 48613546Sjulian{ 487113658Sdeischen struct pthread *curthread = _get_curthread(); 488113658Sdeischen int ret = 0; 48913546Sjulian 49035027Sjb if (mutex == NULL) 49171581Sdeischen ret = EINVAL; 49235027Sjb 49335027Sjb /* 49435027Sjb * If the mutex is statically initialized, perform the dynamic 49535027Sjb * initialization: 49635027Sjb */ 497113658Sdeischen else if ((*mutex != NULL) || 498113658Sdeischen ((ret = init_static(curthread, mutex)) == 0)) 499113658Sdeischen ret = mutex_trylock_common(curthread, mutex); 50068516Sdeischen 50171581Sdeischen return (ret); 50271581Sdeischen} 50371581Sdeischen 50471581Sdeischenint 50571581Sdeischen_pthread_mutex_trylock(pthread_mutex_t *mutex) 50671581Sdeischen{ 507113658Sdeischen struct pthread *curthread = _get_curthread(); 50871581Sdeischen int ret = 0; 50971581Sdeischen 51071581Sdeischen if (mutex == NULL) 51171581Sdeischen ret = EINVAL; 51271581Sdeischen 51371581Sdeischen /* 51471581Sdeischen * If the mutex is statically initialized, perform the dynamic 51571581Sdeischen * initialization marking the mutex private (delete safe): 51671581Sdeischen */ 517113658Sdeischen else if ((*mutex != NULL) || 518113658Sdeischen ((ret = init_static_private(curthread, mutex)) == 0)) 519113658Sdeischen ret = mutex_trylock_common(curthread, mutex); 52071581Sdeischen 52171581Sdeischen return (ret); 52271581Sdeischen} 52371581Sdeischen 52471581Sdeischenstatic int 525119736Sdavidxumutex_lock_common(struct pthread *curthread, pthread_mutex_t *m, 526119736Sdavidxu const struct timespec * abstime) 52771581Sdeischen{ 528139023Sdeischen int private; 52971581Sdeischen int ret = 0; 53071581Sdeischen 531113658Sdeischen THR_ASSERT((m != NULL) && (*m != NULL), 53271581Sdeischen "Uninitialized mutex in pthread_mutex_trylock_basic"); 53371581Sdeischen 534119736Sdavidxu if (abstime != NULL && (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || 535119736Sdavidxu abstime->tv_nsec >= 1000000000)) 536119736Sdavidxu return (EINVAL); 537119736Sdavidxu 53868516Sdeischen /* Reset the interrupted flag: */ 53971581Sdeischen curthread->interrupted = 0; 540119736Sdavidxu curthread->timeout = 0; 541119736Sdavidxu curthread->wakeup_time.tv_sec = -1; 54268516Sdeischen 543139023Sdeischen private = (*m)->m_flags & MUTEX_FLAGS_PRIVATE; 544139023Sdeischen 54568516Sdeischen /* 54668516Sdeischen * Enter a loop waiting to become the mutex owner. We need a 54768516Sdeischen * loop in case the waiting thread is interrupted by a signal 54868516Sdeischen * to execute a signal handler. It is not (currently) possible 54968516Sdeischen * to remain in the waiting queue while running a handler. 55068516Sdeischen * Instead, the thread is interrupted and backed out of the 55168516Sdeischen * waiting queue prior to executing the signal handler. 55268516Sdeischen */ 55368941Sdeischen do { 55448046Sjb /* Lock the mutex structure: */ 555113658Sdeischen THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 55648046Sjb 55748046Sjb /* 55847424Sjb * If the mutex was statically allocated, properly 55947424Sjb * initialize the tail queue. 56047424Sjb */ 561113658Sdeischen if (((*m)->m_flags & MUTEX_FLAGS_INITED) == 0) { 562113658Sdeischen TAILQ_INIT(&(*m)->m_queue); 563113658Sdeischen (*m)->m_flags |= MUTEX_FLAGS_INITED; 564113658Sdeischen MUTEX_INIT_LINK(*m); 56547424Sjb } 56647424Sjb 56717706Sjulian /* Process according to mutex type: */ 568113658Sdeischen switch ((*m)->m_protocol) { 56944963Sjb /* Default POSIX mutex: */ 57044963Sjb case PTHREAD_PRIO_NONE: 571113658Sdeischen if ((*m)->m_owner == NULL) { 57244963Sjb /* Lock the mutex for this thread: */ 573113658Sdeischen (*m)->m_owner = curthread; 57444963Sjb 57544963Sjb /* Add to the list of owned mutexes: */ 576113658Sdeischen MUTEX_ASSERT_NOT_OWNED(*m); 57771581Sdeischen TAILQ_INSERT_TAIL(&curthread->mutexq, 578113658Sdeischen (*m), m_qe); 579139023Sdeischen if (private) 580139023Sdeischen THR_CRITICAL_ENTER(curthread); 58144963Sjb 582113658Sdeischen /* Unlock the mutex structure: */ 583113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 584113658Sdeischen } else if ((*m)->m_owner == curthread) { 585113658Sdeischen ret = mutex_self_lock(curthread, *m); 586113658Sdeischen 587113658Sdeischen /* Unlock the mutex structure: */ 588113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 589113658Sdeischen } else { 59044963Sjb /* 59144963Sjb * Join the queue of threads waiting to lock 592113658Sdeischen * the mutex and save a pointer to the mutex. 59344963Sjb */ 594113658Sdeischen mutex_queue_enq(*m, curthread); 595113658Sdeischen curthread->data.mutex = *m; 596139023Sdeischen curthread->sigbackout = mutex_lock_backout; 59744963Sjb /* 598113658Sdeischen * This thread is active and is in a critical 599113658Sdeischen * region (holding the mutex lock); we should 600113658Sdeischen * be able to safely set the state. 60144963Sjb */ 602115080Sdeischen THR_SCHED_LOCK(curthread, curthread); 603155962Sdeischen /* Set the wakeup time: */ 604155962Sdeischen if (abstime) { 605155962Sdeischen curthread->wakeup_time.tv_sec = 606155962Sdeischen abstime->tv_sec; 607155962Sdeischen curthread->wakeup_time.tv_nsec = 608155962Sdeischen abstime->tv_nsec; 609155962Sdeischen } 610155962Sdeischen 611113658Sdeischen THR_SET_STATE(curthread, PS_MUTEX_WAIT); 612115080Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 61344963Sjb 614113658Sdeischen /* Unlock the mutex structure: */ 615113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 61644963Sjb 617113658Sdeischen /* Schedule the next thread: */ 618113658Sdeischen _thr_sched_switch(curthread); 619119736Sdavidxu 620119736Sdavidxu if (THR_IN_MUTEXQ(curthread)) { 621119736Sdavidxu THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 622119736Sdavidxu mutex_queue_remove(*m, curthread); 623119736Sdavidxu THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 624119736Sdavidxu } 625139023Sdeischen /* 626139023Sdeischen * Only clear these after assuring the 627139023Sdeischen * thread is dequeued. 628139023Sdeischen */ 629139023Sdeischen curthread->data.mutex = NULL; 630139023Sdeischen curthread->sigbackout = NULL; 63141390Seivind } 63244963Sjb break; 63313546Sjulian 63444963Sjb /* POSIX priority inheritence mutex: */ 63544963Sjb case PTHREAD_PRIO_INHERIT: 63644963Sjb /* Check if this mutex is not locked: */ 637113658Sdeischen if ((*m)->m_owner == NULL) { 63844963Sjb /* Lock the mutex for this thread: */ 639113658Sdeischen (*m)->m_owner = curthread; 64035509Sjb 641114187Sdeischen THR_SCHED_LOCK(curthread, curthread); 64244963Sjb /* Track number of priority mutexes owned: */ 64371581Sdeischen curthread->priority_mutex_count++; 64444963Sjb 64544963Sjb /* 64644963Sjb * The mutex takes on attributes of the 64744963Sjb * running thread when there are no waiters. 648113658Sdeischen * Make sure the thread's scheduling lock is 649113658Sdeischen * held while priorities are adjusted. 65044963Sjb */ 651113658Sdeischen (*m)->m_prio = curthread->active_priority; 652113658Sdeischen (*m)->m_saved_prio = 65371581Sdeischen curthread->inherited_priority; 654113658Sdeischen curthread->inherited_priority = (*m)->m_prio; 655113658Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 65644963Sjb 65744963Sjb /* Add to the list of owned mutexes: */ 658113658Sdeischen MUTEX_ASSERT_NOT_OWNED(*m); 65971581Sdeischen TAILQ_INSERT_TAIL(&curthread->mutexq, 660113658Sdeischen (*m), m_qe); 661139023Sdeischen if (private) 662139023Sdeischen THR_CRITICAL_ENTER(curthread); 66344963Sjb 664113658Sdeischen /* Unlock the mutex structure: */ 665113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 666113658Sdeischen } else if ((*m)->m_owner == curthread) { 667113658Sdeischen ret = mutex_self_lock(curthread, *m); 668113658Sdeischen 669113658Sdeischen /* Unlock the mutex structure: */ 670113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 671113658Sdeischen } else { 67244963Sjb /* 67344963Sjb * Join the queue of threads waiting to lock 674113658Sdeischen * the mutex and save a pointer to the mutex. 67544963Sjb */ 676113658Sdeischen mutex_queue_enq(*m, curthread); 677113658Sdeischen curthread->data.mutex = *m; 678139023Sdeischen curthread->sigbackout = mutex_lock_backout; 67944963Sjb 68044963Sjb /* 681113658Sdeischen * This thread is active and is in a critical 682113658Sdeischen * region (holding the mutex lock); we should 683113658Sdeischen * be able to safely set the state. 68444963Sjb */ 685113658Sdeischen if (curthread->active_priority > (*m)->m_prio) 68644963Sjb /* Adjust priorities: */ 687113658Sdeischen mutex_priority_adjust(curthread, *m); 68844963Sjb 689115080Sdeischen THR_SCHED_LOCK(curthread, curthread); 690155962Sdeischen /* Set the wakeup time: */ 691155962Sdeischen if (abstime) { 692155962Sdeischen curthread->wakeup_time.tv_sec = 693155962Sdeischen abstime->tv_sec; 694155962Sdeischen curthread->wakeup_time.tv_nsec = 695155962Sdeischen abstime->tv_nsec; 696155962Sdeischen } 697114187Sdeischen THR_SET_STATE(curthread, PS_MUTEX_WAIT); 698115080Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 699115080Sdeischen 700113658Sdeischen /* Unlock the mutex structure: */ 701113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 70244963Sjb 703113658Sdeischen /* Schedule the next thread: */ 704113658Sdeischen _thr_sched_switch(curthread); 705119736Sdavidxu 706119736Sdavidxu if (THR_IN_MUTEXQ(curthread)) { 707119736Sdavidxu THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 708119736Sdavidxu mutex_queue_remove(*m, curthread); 709119736Sdavidxu THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 710119736Sdavidxu } 711139023Sdeischen /* 712139023Sdeischen * Only clear these after assuring the 713139023Sdeischen * thread is dequeued. 714139023Sdeischen */ 715139023Sdeischen curthread->data.mutex = NULL; 716139023Sdeischen curthread->sigbackout = NULL; 71713546Sjulian } 71817706Sjulian break; 71913546Sjulian 72044963Sjb /* POSIX priority protection mutex: */ 72144963Sjb case PTHREAD_PRIO_PROTECT: 72244963Sjb /* Check for a priority ceiling violation: */ 723113658Sdeischen if (curthread->active_priority > (*m)->m_prio) { 724113658Sdeischen /* Unlock the mutex structure: */ 725113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 72644963Sjb ret = EINVAL; 727113658Sdeischen } 72844963Sjb /* Check if this mutex is not locked: */ 729113658Sdeischen else if ((*m)->m_owner == NULL) { 73044963Sjb /* 73144963Sjb * Lock the mutex for the running 73244963Sjb * thread: 73344963Sjb */ 734113658Sdeischen (*m)->m_owner = curthread; 73513546Sjulian 736114187Sdeischen THR_SCHED_LOCK(curthread, curthread); 73744963Sjb /* Track number of priority mutexes owned: */ 73871581Sdeischen curthread->priority_mutex_count++; 73935509Sjb 74044963Sjb /* 74144963Sjb * The running thread inherits the ceiling 74244963Sjb * priority of the mutex and executes at that 743113658Sdeischen * priority. Make sure the thread's 744113658Sdeischen * scheduling lock is held while priorities 745113658Sdeischen * are adjusted. 74644963Sjb */ 747113658Sdeischen curthread->active_priority = (*m)->m_prio; 748113658Sdeischen (*m)->m_saved_prio = 74971581Sdeischen curthread->inherited_priority; 750113658Sdeischen curthread->inherited_priority = (*m)->m_prio; 751113658Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 75244963Sjb 75344963Sjb /* Add to the list of owned mutexes: */ 754113658Sdeischen MUTEX_ASSERT_NOT_OWNED(*m); 75571581Sdeischen TAILQ_INSERT_TAIL(&curthread->mutexq, 756113658Sdeischen (*m), m_qe); 757139023Sdeischen if (private) 758139023Sdeischen THR_CRITICAL_ENTER(curthread); 759113658Sdeischen 760113658Sdeischen /* Unlock the mutex structure: */ 761113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 762113658Sdeischen } else if ((*m)->m_owner == curthread) { 763113658Sdeischen ret = mutex_self_lock(curthread, *m); 764113658Sdeischen 765113658Sdeischen /* Unlock the mutex structure: */ 766113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 767113658Sdeischen } else { 76844963Sjb /* 76944963Sjb * Join the queue of threads waiting to lock 770113658Sdeischen * the mutex and save a pointer to the mutex. 77144963Sjb */ 772113658Sdeischen mutex_queue_enq(*m, curthread); 773113658Sdeischen curthread->data.mutex = *m; 774139023Sdeischen curthread->sigbackout = mutex_lock_backout; 77544963Sjb 77644963Sjb /* Clear any previous error: */ 77771581Sdeischen curthread->error = 0; 77844963Sjb 77944963Sjb /* 780113658Sdeischen * This thread is active and is in a critical 781113658Sdeischen * region (holding the mutex lock); we should 782113658Sdeischen * be able to safely set the state. 78344963Sjb */ 784114187Sdeischen 785115080Sdeischen THR_SCHED_LOCK(curthread, curthread); 786155962Sdeischen /* Set the wakeup time: */ 787155962Sdeischen if (abstime) { 788155962Sdeischen curthread->wakeup_time.tv_sec = 789155962Sdeischen abstime->tv_sec; 790155962Sdeischen curthread->wakeup_time.tv_nsec = 791155962Sdeischen abstime->tv_nsec; 792155962Sdeischen } 793113658Sdeischen THR_SET_STATE(curthread, PS_MUTEX_WAIT); 794115080Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 79544963Sjb 796113658Sdeischen /* Unlock the mutex structure: */ 797113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 79844963Sjb 799113658Sdeischen /* Schedule the next thread: */ 800113658Sdeischen _thr_sched_switch(curthread); 801119736Sdavidxu 802119736Sdavidxu if (THR_IN_MUTEXQ(curthread)) { 803119736Sdavidxu THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 804119736Sdavidxu mutex_queue_remove(*m, curthread); 805119736Sdavidxu THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 806119736Sdavidxu } 807139023Sdeischen /* 808139023Sdeischen * Only clear these after assuring the 809139023Sdeischen * thread is dequeued. 810139023Sdeischen */ 811139023Sdeischen curthread->data.mutex = NULL; 812139023Sdeischen curthread->sigbackout = NULL; 813119736Sdavidxu 81444963Sjb /* 81544963Sjb * The threads priority may have changed while 81644963Sjb * waiting for the mutex causing a ceiling 81744963Sjb * violation. 81844963Sjb */ 81971581Sdeischen ret = curthread->error; 82071581Sdeischen curthread->error = 0; 82113546Sjulian } 82217706Sjulian break; 82313546Sjulian 82413546Sjulian /* Trap invalid mutex types: */ 82517706Sjulian default: 826113658Sdeischen /* Unlock the mutex structure: */ 827113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 828113658Sdeischen 82917706Sjulian /* Return an invalid argument error: */ 83031402Salex ret = EINVAL; 83117706Sjulian break; 83217706Sjulian } 83317706Sjulian 834113658Sdeischen } while (((*m)->m_owner != curthread) && (ret == 0) && 835119736Sdavidxu (curthread->interrupted == 0) && (curthread->timeout == 0)); 83653812Salfred 837123310Sdavidxu if (ret == 0 && (*m)->m_owner != curthread && curthread->timeout) 838119736Sdavidxu ret = ETIMEDOUT; 839119736Sdavidxu 840113658Sdeischen /* 841113658Sdeischen * Check to see if this thread was interrupted and 842113658Sdeischen * is still in the mutex queue of waiting threads: 843113658Sdeischen */ 844113658Sdeischen if (curthread->interrupted != 0) { 845113658Sdeischen /* Remove this thread from the mutex queue. */ 846113658Sdeischen THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 847113658Sdeischen if (THR_IN_SYNCQ(curthread)) 848113658Sdeischen mutex_queue_remove(*m, curthread); 849113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 85044963Sjb 851113658Sdeischen /* Check for asynchronous cancellation. */ 852113658Sdeischen if (curthread->continuation != NULL) 853113658Sdeischen curthread->continuation((void *) curthread); 854113658Sdeischen } 85513546Sjulian 85613546Sjulian /* Return the completion status: */ 85713546Sjulian return (ret); 85813546Sjulian} 85913546Sjulian 86013546Sjulianint 861113658Sdeischen__pthread_mutex_lock(pthread_mutex_t *m) 86213546Sjulian{ 863113658Sdeischen struct pthread *curthread; 86471581Sdeischen int ret = 0; 86571581Sdeischen 866113658Sdeischen if (_thr_initial == NULL) 867113658Sdeischen _libpthread_init(NULL); 86871581Sdeischen 869113658Sdeischen curthread = _get_curthread(); 870113658Sdeischen if (m == NULL) 87171581Sdeischen ret = EINVAL; 87271581Sdeischen 87371581Sdeischen /* 87471581Sdeischen * If the mutex is statically initialized, perform the dynamic 87571581Sdeischen * initialization: 87671581Sdeischen */ 877113658Sdeischen else if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0)) 878119736Sdavidxu ret = mutex_lock_common(curthread, m, NULL); 87971581Sdeischen 88071581Sdeischen return (ret); 88171581Sdeischen} 88271581Sdeischen 883115399Skan__strong_reference(__pthread_mutex_lock, _thr_mutex_lock); 884115399Skan 88571581Sdeischenint 886113658Sdeischen_pthread_mutex_lock(pthread_mutex_t *m) 88771581Sdeischen{ 888113658Sdeischen struct pthread *curthread; 88971581Sdeischen int ret = 0; 89071581Sdeischen 891113658Sdeischen if (_thr_initial == NULL) 892113658Sdeischen _libpthread_init(NULL); 893113658Sdeischen curthread = _get_curthread(); 89471581Sdeischen 895113658Sdeischen if (m == NULL) 89671581Sdeischen ret = EINVAL; 89771581Sdeischen 89871581Sdeischen /* 89971581Sdeischen * If the mutex is statically initialized, perform the dynamic 90071581Sdeischen * initialization marking it private (delete safe): 90171581Sdeischen */ 902113658Sdeischen else if ((*m != NULL) || 903113658Sdeischen ((ret = init_static_private(curthread, m)) == 0)) 904119736Sdavidxu ret = mutex_lock_common(curthread, m, NULL); 90571581Sdeischen 90671581Sdeischen return (ret); 90771581Sdeischen} 90871581Sdeischen 90971581Sdeischenint 910119736Sdavidxu__pthread_mutex_timedlock(pthread_mutex_t *m, 911119736Sdavidxu const struct timespec *abs_timeout) 912119736Sdavidxu{ 913119736Sdavidxu struct pthread *curthread; 914119736Sdavidxu int ret = 0; 915119736Sdavidxu 916119736Sdavidxu if (_thr_initial == NULL) 917119736Sdavidxu _libpthread_init(NULL); 918119736Sdavidxu 919119736Sdavidxu curthread = _get_curthread(); 920119736Sdavidxu if (m == NULL) 921119736Sdavidxu ret = EINVAL; 922119736Sdavidxu 923119736Sdavidxu /* 924119736Sdavidxu * If the mutex is statically initialized, perform the dynamic 925119736Sdavidxu * initialization: 926119736Sdavidxu */ 927119736Sdavidxu else if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0)) 928119736Sdavidxu ret = mutex_lock_common(curthread, m, abs_timeout); 929119736Sdavidxu 930119736Sdavidxu return (ret); 931119736Sdavidxu} 932119736Sdavidxu 933119736Sdavidxuint 934119736Sdavidxu_pthread_mutex_timedlock(pthread_mutex_t *m, 935119736Sdavidxu const struct timespec *abs_timeout) 936119736Sdavidxu{ 937119736Sdavidxu struct pthread *curthread; 938119736Sdavidxu int ret = 0; 939119736Sdavidxu 940119736Sdavidxu if (_thr_initial == NULL) 941119736Sdavidxu _libpthread_init(NULL); 942119736Sdavidxu curthread = _get_curthread(); 943119736Sdavidxu 944119736Sdavidxu if (m == NULL) 945119736Sdavidxu ret = EINVAL; 946119736Sdavidxu 947119736Sdavidxu /* 948119736Sdavidxu * If the mutex is statically initialized, perform the dynamic 949119736Sdavidxu * initialization marking it private (delete safe): 950119736Sdavidxu */ 951119736Sdavidxu else if ((*m != NULL) || 952119736Sdavidxu ((ret = init_static_private(curthread, m)) == 0)) 953119736Sdavidxu ret = mutex_lock_common(curthread, m, abs_timeout); 954119736Sdavidxu 955119736Sdavidxu return (ret); 956119736Sdavidxu} 957119736Sdavidxu 958119736Sdavidxuint 959113658Sdeischen_pthread_mutex_unlock(pthread_mutex_t *m) 96071581Sdeischen{ 961113658Sdeischen return (mutex_unlock_common(m, /* add reference */ 0)); 96244963Sjb} 96313546Sjulian 964115399Skan__strong_reference(_pthread_mutex_unlock, _thr_mutex_unlock); 965115399Skan 96644963Sjbint 967113658Sdeischen_mutex_cv_unlock(pthread_mutex_t *m) 96844963Sjb{ 969113658Sdeischen return (mutex_unlock_common(m, /* add reference */ 1)); 97044963Sjb} 97144963Sjb 97244963Sjbint 973113658Sdeischen_mutex_cv_lock(pthread_mutex_t *m) 97444963Sjb{ 975114187Sdeischen struct pthread *curthread; 97667097Sdeischen int ret; 977114187Sdeischen 978114187Sdeischen curthread = _get_curthread(); 979114187Sdeischen if ((ret = _pthread_mutex_lock(m)) == 0) { 980114187Sdeischen THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 981113658Sdeischen (*m)->m_refcount--; 982114187Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 983114187Sdeischen } 98444963Sjb return (ret); 98544963Sjb} 98644963Sjb 98744963Sjbstatic inline int 988174112Sdeischenmutex_self_trylock(pthread_mutex_t m) 98944963Sjb{ 99067097Sdeischen int ret = 0; 99144963Sjb 992113658Sdeischen switch (m->m_type) { 99344963Sjb /* case PTHREAD_MUTEX_DEFAULT: */ 99444963Sjb case PTHREAD_MUTEX_ERRORCHECK: 99544963Sjb case PTHREAD_MUTEX_NORMAL: 996173154Skris case PTHREAD_MUTEX_ADAPTIVE_NP: 997137095Sgreen ret = EBUSY; 99844963Sjb break; 99944963Sjb 100044963Sjb case PTHREAD_MUTEX_RECURSIVE: 100144963Sjb /* Increment the lock count: */ 1002113658Sdeischen m->m_count++; 100344963Sjb break; 100444963Sjb 100544963Sjb default: 100644963Sjb /* Trap invalid mutex types; */ 100744963Sjb ret = EINVAL; 100844963Sjb } 100944963Sjb 1010113658Sdeischen return (ret); 101144963Sjb} 101244963Sjb 101344963Sjbstatic inline int 1014113658Sdeischenmutex_self_lock(struct pthread *curthread, pthread_mutex_t m) 101544963Sjb{ 101644963Sjb int ret = 0; 101744963Sjb 1018139023Sdeischen /* 1019139023Sdeischen * Don't allow evil recursive mutexes for private use 1020139023Sdeischen * in libc and libpthread. 1021139023Sdeischen */ 1022139023Sdeischen if (m->m_flags & MUTEX_FLAGS_PRIVATE) 1023139023Sdeischen PANIC("Recurse on a private mutex."); 1024139023Sdeischen 1025113658Sdeischen switch (m->m_type) { 102644963Sjb /* case PTHREAD_MUTEX_DEFAULT: */ 102744963Sjb case PTHREAD_MUTEX_ERRORCHECK: 1028173154Skris case PTHREAD_MUTEX_ADAPTIVE_NP: 102944963Sjb /* 103044963Sjb * POSIX specifies that mutexes should return EDEADLK if a 103144963Sjb * recursive lock is detected. 103244963Sjb */ 103344963Sjb ret = EDEADLK; 103444963Sjb break; 103544963Sjb 103644963Sjb case PTHREAD_MUTEX_NORMAL: 103744963Sjb /* 103844963Sjb * What SS2 define as a 'normal' mutex. Intentionally 103944963Sjb * deadlock on attempts to get a lock you already own. 104044963Sjb */ 1041114187Sdeischen 1042115080Sdeischen THR_SCHED_LOCK(curthread, curthread); 1043113658Sdeischen THR_SET_STATE(curthread, PS_DEADLOCK); 1044115080Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 1045113658Sdeischen 1046113658Sdeischen /* Unlock the mutex structure: */ 1047113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 1048113658Sdeischen 1049113658Sdeischen /* Schedule the next thread: */ 1050113658Sdeischen _thr_sched_switch(curthread); 105144963Sjb break; 105244963Sjb 105344963Sjb case PTHREAD_MUTEX_RECURSIVE: 105444963Sjb /* Increment the lock count: */ 1055113658Sdeischen m->m_count++; 105644963Sjb break; 105744963Sjb 105844963Sjb default: 105944963Sjb /* Trap invalid mutex types; */ 106044963Sjb ret = EINVAL; 106144963Sjb } 106244963Sjb 1063113658Sdeischen return (ret); 106444963Sjb} 106544963Sjb 1066113658Sdeischenstatic int 1067113658Sdeischenmutex_unlock_common(pthread_mutex_t *m, int add_reference) 106844963Sjb{ 1069117714Sdeischen struct pthread *curthread = _get_curthread(); 1070117907Sdeischen struct kse_mailbox *kmbx = NULL; 1071117714Sdeischen int ret = 0; 107244963Sjb 1073113658Sdeischen if (m == NULL || *m == NULL) 107431402Salex ret = EINVAL; 1075113658Sdeischen else { 107635509Sjb /* Lock the mutex structure: */ 1077113658Sdeischen THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); 107813546Sjulian 107917706Sjulian /* Process according to mutex type: */ 1080113658Sdeischen switch ((*m)->m_protocol) { 108144963Sjb /* Default POSIX mutex: */ 108244963Sjb case PTHREAD_PRIO_NONE: 108344963Sjb /* 108444963Sjb * Check if the running thread is not the owner of the 108544963Sjb * mutex: 108644963Sjb */ 1087113658Sdeischen if ((*m)->m_owner != curthread) 1088124606Sdavidxu ret = EPERM; 1089113658Sdeischen else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && 1090113658Sdeischen ((*m)->m_count > 0)) 109144963Sjb /* Decrement the count: */ 1092113658Sdeischen (*m)->m_count--; 1093113658Sdeischen else { 109444963Sjb /* 1095113658Sdeischen * Clear the count in case this is a recursive 109644963Sjb * mutex. 109744963Sjb */ 1098113658Sdeischen (*m)->m_count = 0; 109944963Sjb 110044963Sjb /* Remove the mutex from the threads queue. */ 1101113658Sdeischen MUTEX_ASSERT_IS_OWNED(*m); 1102113658Sdeischen TAILQ_REMOVE(&(*m)->m_owner->mutexq, 1103113658Sdeischen (*m), m_qe); 1104113658Sdeischen MUTEX_INIT_LINK(*m); 110544963Sjb 110644963Sjb /* 1107113658Sdeischen * Hand off the mutex to the next waiting 1108113658Sdeischen * thread: 110944963Sjb */ 1110117907Sdeischen kmbx = mutex_handoff(curthread, *m); 111144963Sjb } 111244963Sjb break; 111344963Sjb 111444963Sjb /* POSIX priority inheritence mutex: */ 111544963Sjb case PTHREAD_PRIO_INHERIT: 111617706Sjulian /* 111744963Sjb * Check if the running thread is not the owner of the 111844963Sjb * mutex: 111917706Sjulian */ 1120113658Sdeischen if ((*m)->m_owner != curthread) 1121124606Sdavidxu ret = EPERM; 1122113658Sdeischen else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && 1123113658Sdeischen ((*m)->m_count > 0)) 112444963Sjb /* Decrement the count: */ 1125113658Sdeischen (*m)->m_count--; 1126113658Sdeischen else { 112744963Sjb /* 112844963Sjb * Clear the count in case this is recursive 112944963Sjb * mutex. 113044963Sjb */ 1131113658Sdeischen (*m)->m_count = 0; 113244963Sjb 113344963Sjb /* 113444963Sjb * Restore the threads inherited priority and 113544963Sjb * recompute the active priority (being careful 113644963Sjb * not to override changes in the threads base 113744963Sjb * priority subsequent to locking the mutex). 113844963Sjb */ 1139113658Sdeischen THR_SCHED_LOCK(curthread, curthread); 114071581Sdeischen curthread->inherited_priority = 1141113658Sdeischen (*m)->m_saved_prio; 114271581Sdeischen curthread->active_priority = 114371581Sdeischen MAX(curthread->inherited_priority, 114471581Sdeischen curthread->base_priority); 114544963Sjb 114644963Sjb /* 114744963Sjb * This thread now owns one less priority mutex. 114844963Sjb */ 114971581Sdeischen curthread->priority_mutex_count--; 1150114187Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 115144963Sjb 115244963Sjb /* Remove the mutex from the threads queue. */ 1153113658Sdeischen MUTEX_ASSERT_IS_OWNED(*m); 1154113658Sdeischen TAILQ_REMOVE(&(*m)->m_owner->mutexq, 1155113658Sdeischen (*m), m_qe); 1156113658Sdeischen MUTEX_INIT_LINK(*m); 115744963Sjb 115844963Sjb /* 1159113658Sdeischen * Hand off the mutex to the next waiting 1160113658Sdeischen * thread: 116144963Sjb */ 1162117907Sdeischen kmbx = mutex_handoff(curthread, *m); 116344963Sjb } 116417706Sjulian break; 116513546Sjulian 116644963Sjb /* POSIX priority ceiling mutex: */ 116744963Sjb case PTHREAD_PRIO_PROTECT: 116844963Sjb /* 116944963Sjb * Check if the running thread is not the owner of the 117044963Sjb * mutex: 117144963Sjb */ 1172113658Sdeischen if ((*m)->m_owner != curthread) 1173124606Sdavidxu ret = EPERM; 1174113658Sdeischen else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && 1175113658Sdeischen ((*m)->m_count > 0)) 117617706Sjulian /* Decrement the count: */ 1177113658Sdeischen (*m)->m_count--; 1178113658Sdeischen else { 117944963Sjb /* 1180113658Sdeischen * Clear the count in case this is a recursive 118144963Sjb * mutex. 118244963Sjb */ 1183113658Sdeischen (*m)->m_count = 0; 118444963Sjb 118541390Seivind /* 118644963Sjb * Restore the threads inherited priority and 118744963Sjb * recompute the active priority (being careful 118844963Sjb * not to override changes in the threads base 118944963Sjb * priority subsequent to locking the mutex). 119041390Seivind */ 1191113658Sdeischen THR_SCHED_LOCK(curthread, curthread); 119271581Sdeischen curthread->inherited_priority = 1193117714Sdeischen (*m)->m_saved_prio; 119471581Sdeischen curthread->active_priority = 119571581Sdeischen MAX(curthread->inherited_priority, 119671581Sdeischen curthread->base_priority); 119744963Sjb 119844963Sjb /* 119944963Sjb * This thread now owns one less priority mutex. 120044963Sjb */ 120171581Sdeischen curthread->priority_mutex_count--; 1202118206Sdeischen THR_SCHED_UNLOCK(curthread, curthread); 120344963Sjb 120444963Sjb /* Remove the mutex from the threads queue. */ 1205113658Sdeischen MUTEX_ASSERT_IS_OWNED(*m); 1206113658Sdeischen TAILQ_REMOVE(&(*m)->m_owner->mutexq, 1207113658Sdeischen (*m), m_qe); 1208113658Sdeischen MUTEX_INIT_LINK(*m); 120944963Sjb 121044963Sjb /* 1211113658Sdeischen * Hand off the mutex to the next waiting 1212113658Sdeischen * thread: 121344963Sjb */ 1214117907Sdeischen kmbx = mutex_handoff(curthread, *m); 121517706Sjulian } 121617706Sjulian break; 121717706Sjulian 121817706Sjulian /* Trap invalid mutex types: */ 121917706Sjulian default: 122013546Sjulian /* Return an invalid argument error: */ 122131402Salex ret = EINVAL; 122217706Sjulian break; 122313546Sjulian } 122413546Sjulian 1225113658Sdeischen if ((ret == 0) && (add_reference != 0)) 122644963Sjb /* Increment the reference count: */ 1227113658Sdeischen (*m)->m_refcount++; 122844963Sjb 1229139023Sdeischen /* Leave the critical region if this is a private mutex. */ 1230139023Sdeischen if ((ret == 0) && ((*m)->m_flags & MUTEX_FLAGS_PRIVATE)) 1231139023Sdeischen THR_CRITICAL_LEAVE(curthread); 1232139023Sdeischen 123335509Sjb /* Unlock the mutex structure: */ 1234113658Sdeischen THR_LOCK_RELEASE(curthread, &(*m)->m_lock); 1235139023Sdeischen 1236117907Sdeischen if (kmbx != NULL) 1237117907Sdeischen kse_wakeup(kmbx); 123813546Sjulian } 123913546Sjulian 124013546Sjulian /* Return the completion status: */ 124113546Sjulian return (ret); 124213546Sjulian} 124344963Sjb 124444963Sjb 124544963Sjb/* 124648046Sjb * This function is called when a change in base priority occurs for 124748046Sjb * a thread that is holding or waiting for a priority protection or 124848046Sjb * inheritence mutex. A change in a threads base priority can effect 124948046Sjb * changes to active priorities of other threads and to the ordering 125048046Sjb * of mutex locking by waiting threads. 125144963Sjb * 1252113658Sdeischen * This must be called without the target thread's scheduling lock held. 125344963Sjb */ 125444963Sjbvoid 1255113658Sdeischen_mutex_notify_priochange(struct pthread *curthread, struct pthread *pthread, 1256113658Sdeischen int propagate_prio) 125744963Sjb{ 1258113658Sdeischen struct pthread_mutex *m; 1259113658Sdeischen 126044963Sjb /* Adjust the priorites of any owned priority mutexes: */ 126144963Sjb if (pthread->priority_mutex_count > 0) { 126244963Sjb /* 126344963Sjb * Rescan the mutexes owned by this thread and correct 126444963Sjb * their priorities to account for this threads change 126544963Sjb * in priority. This has the side effect of changing 126644963Sjb * the threads active priority. 1267113658Sdeischen * 1268113658Sdeischen * Be sure to lock the first mutex in the list of owned 1269113658Sdeischen * mutexes. This acts as a barrier against another 1270113658Sdeischen * simultaneous call to change the threads priority 1271113658Sdeischen * and from the owning thread releasing the mutex. 127244963Sjb */ 1273113658Sdeischen m = TAILQ_FIRST(&pthread->mutexq); 1274113658Sdeischen if (m != NULL) { 1275113658Sdeischen THR_LOCK_ACQUIRE(curthread, &m->m_lock); 1276113658Sdeischen /* 1277113658Sdeischen * Make sure the thread still owns the lock. 1278113658Sdeischen */ 1279113658Sdeischen if (m == TAILQ_FIRST(&pthread->mutexq)) 1280113658Sdeischen mutex_rescan_owned(curthread, pthread, 1281113658Sdeischen /* rescan all owned */ NULL); 1282113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 1283113658Sdeischen } 128444963Sjb } 128544963Sjb 128644963Sjb /* 128744963Sjb * If this thread is waiting on a priority inheritence mutex, 128844963Sjb * check for priority adjustments. A change in priority can 1289113658Sdeischen * also cause a ceiling violation(*) for a thread waiting on 129044963Sjb * a priority protection mutex; we don't perform the check here 129144963Sjb * as it is done in pthread_mutex_unlock. 129244963Sjb * 129344963Sjb * (*) It should be noted that a priority change to a thread 129444963Sjb * _after_ taking and owning a priority ceiling mutex 129544963Sjb * does not affect ownership of that mutex; the ceiling 129644963Sjb * priority is only checked before mutex ownership occurs. 129744963Sjb */ 1298113658Sdeischen if (propagate_prio != 0) { 129944963Sjb /* 1300113658Sdeischen * Lock the thread's scheduling queue. This is a bit 1301113658Sdeischen * convoluted; the "in synchronization queue flag" can 1302113658Sdeischen * only be cleared with both the thread's scheduling and 1303113658Sdeischen * mutex locks held. The thread's pointer to the wanted 1304113658Sdeischen * mutex is guaranteed to be valid during this time. 130544963Sjb */ 1306113658Sdeischen THR_SCHED_LOCK(curthread, pthread); 1307113658Sdeischen 1308113658Sdeischen if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) == 0) || 1309113658Sdeischen ((m = pthread->data.mutex) == NULL)) 1310113658Sdeischen THR_SCHED_UNLOCK(curthread, pthread); 1311113658Sdeischen else { 131244963Sjb /* 1313113658Sdeischen * This thread is currently waiting on a mutex; unlock 1314113658Sdeischen * the scheduling queue lock and lock the mutex. We 1315113658Sdeischen * can't hold both at the same time because the locking 1316113658Sdeischen * order could cause a deadlock. 131744963Sjb */ 1318113658Sdeischen THR_SCHED_UNLOCK(curthread, pthread); 1319113658Sdeischen THR_LOCK_ACQUIRE(curthread, &m->m_lock); 132044963Sjb 1321113658Sdeischen /* 1322113658Sdeischen * Check to make sure this thread is still in the 1323113658Sdeischen * same state (the lock above can yield the CPU to 1324113658Sdeischen * another thread or the thread may be running on 1325113658Sdeischen * another CPU). 1326113658Sdeischen */ 1327113658Sdeischen if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && 1328113658Sdeischen (pthread->data.mutex == m)) { 1329113658Sdeischen /* 1330113658Sdeischen * Remove and reinsert this thread into 1331113658Sdeischen * the list of waiting threads to preserve 1332113658Sdeischen * decreasing priority order. 1333113658Sdeischen */ 1334113658Sdeischen mutex_queue_remove(m, pthread); 1335113658Sdeischen mutex_queue_enq(m, pthread); 1336113658Sdeischen 1337113658Sdeischen if (m->m_protocol == PTHREAD_PRIO_INHERIT) 1338113658Sdeischen /* Adjust priorities: */ 1339113658Sdeischen mutex_priority_adjust(curthread, m); 134044963Sjb } 1341113658Sdeischen 1342113658Sdeischen /* Unlock the mutex structure: */ 1343113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 134444963Sjb } 134544963Sjb } 134644963Sjb} 134744963Sjb 134844963Sjb/* 134944963Sjb * Called when a new thread is added to the mutex waiting queue or 135044963Sjb * when a threads priority changes that is already in the mutex 135144963Sjb * waiting queue. 1352113658Sdeischen * 1353113658Sdeischen * This must be called with the mutex locked by the current thread. 135444963Sjb */ 135544963Sjbstatic void 1356113658Sdeischenmutex_priority_adjust(struct pthread *curthread, pthread_mutex_t mutex) 135744963Sjb{ 135844963Sjb pthread_mutex_t m = mutex; 1359113658Sdeischen struct pthread *pthread_next, *pthread = mutex->m_owner; 1360113658Sdeischen int done, temp_prio; 136144963Sjb 136244963Sjb /* 136344963Sjb * Calculate the mutex priority as the maximum of the highest 136444963Sjb * active priority of any waiting threads and the owning threads 136544963Sjb * active priority(*). 136644963Sjb * 136744963Sjb * (*) Because the owning threads current active priority may 136844963Sjb * reflect priority inherited from this mutex (and the mutex 136944963Sjb * priority may have changed) we must recalculate the active 137044963Sjb * priority based on the threads saved inherited priority 137144963Sjb * and its base priority. 137244963Sjb */ 137344963Sjb pthread_next = TAILQ_FIRST(&m->m_queue); /* should never be NULL */ 137444963Sjb temp_prio = MAX(pthread_next->active_priority, 137544963Sjb MAX(m->m_saved_prio, pthread->base_priority)); 137644963Sjb 137744963Sjb /* See if this mutex really needs adjusting: */ 137844963Sjb if (temp_prio == m->m_prio) 137944963Sjb /* No need to propagate the priority: */ 138044963Sjb return; 138144963Sjb 138244963Sjb /* Set new priority of the mutex: */ 138344963Sjb m->m_prio = temp_prio; 138444963Sjb 1385113658Sdeischen /* 1386113658Sdeischen * Don't unlock the mutex passed in as an argument. It is 1387113658Sdeischen * expected to be locked and unlocked by the caller. 1388113658Sdeischen */ 1389113658Sdeischen done = 1; 1390113658Sdeischen do { 139144963Sjb /* 139244963Sjb * Save the threads priority before rescanning the 139344963Sjb * owned mutexes: 139444963Sjb */ 139544963Sjb temp_prio = pthread->active_priority; 139644963Sjb 139744963Sjb /* 1398113658Sdeischen * Fix the priorities for all mutexes held by the owning 1399113658Sdeischen * thread since taking this mutex. This also has a 140044963Sjb * potential side-effect of changing the threads priority. 1401113658Sdeischen * 1402113658Sdeischen * At this point the mutex is locked by the current thread. 1403113658Sdeischen * The owning thread can't release the mutex until it is 1404113658Sdeischen * unlocked, so we should be able to safely walk its list 1405113658Sdeischen * of owned mutexes. 140644963Sjb */ 1407113658Sdeischen mutex_rescan_owned(curthread, pthread, m); 140844963Sjb 140944963Sjb /* 1410113658Sdeischen * If this isn't the first time through the loop, 1411113658Sdeischen * the current mutex needs to be unlocked. 1412113658Sdeischen */ 1413113658Sdeischen if (done == 0) 1414113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 1415113658Sdeischen 1416113658Sdeischen /* Assume we're done unless told otherwise: */ 1417113658Sdeischen done = 1; 1418113658Sdeischen 1419113658Sdeischen /* 142044963Sjb * If the thread is currently waiting on a mutex, check 142144963Sjb * to see if the threads new priority has affected the 142244963Sjb * priority of the mutex. 142344963Sjb */ 142444963Sjb if ((temp_prio != pthread->active_priority) && 1425113658Sdeischen ((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && 1426113658Sdeischen ((m = pthread->data.mutex) != NULL) && 1427113658Sdeischen (m->m_protocol == PTHREAD_PRIO_INHERIT)) { 1428113658Sdeischen /* Lock the mutex structure: */ 1429113658Sdeischen THR_LOCK_ACQUIRE(curthread, &m->m_lock); 143044963Sjb 143144963Sjb /* 1432113658Sdeischen * Make sure the thread is still waiting on the 1433113658Sdeischen * mutex: 143444963Sjb */ 1435113658Sdeischen if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && 1436113658Sdeischen (m == pthread->data.mutex)) { 1437113658Sdeischen /* 1438113658Sdeischen * The priority for this thread has changed. 1439113658Sdeischen * Remove and reinsert this thread into the 1440113658Sdeischen * list of waiting threads to preserve 1441113658Sdeischen * decreasing priority order. 1442113658Sdeischen */ 1443113658Sdeischen mutex_queue_remove(m, pthread); 1444113658Sdeischen mutex_queue_enq(m, pthread); 144544963Sjb 1446113658Sdeischen /* 1447113658Sdeischen * Grab the waiting thread with highest 1448113658Sdeischen * priority: 1449113658Sdeischen */ 1450113658Sdeischen pthread_next = TAILQ_FIRST(&m->m_queue); 145144963Sjb 145244963Sjb /* 1453113658Sdeischen * Calculate the mutex priority as the maximum 1454113658Sdeischen * of the highest active priority of any 1455113658Sdeischen * waiting threads and the owning threads 1456113658Sdeischen * active priority. 145744963Sjb */ 1458113658Sdeischen temp_prio = MAX(pthread_next->active_priority, 1459113658Sdeischen MAX(m->m_saved_prio, 1460113658Sdeischen m->m_owner->base_priority)); 1461113658Sdeischen 1462113658Sdeischen if (temp_prio != m->m_prio) { 1463113658Sdeischen /* 1464113658Sdeischen * The priority needs to be propagated 1465113658Sdeischen * to the mutex this thread is waiting 1466113658Sdeischen * on and up to the owner of that mutex. 1467113658Sdeischen */ 1468113658Sdeischen m->m_prio = temp_prio; 1469113658Sdeischen pthread = m->m_owner; 1470113658Sdeischen 1471113658Sdeischen /* We're not done yet: */ 1472113658Sdeischen done = 0; 1473113658Sdeischen } 147444963Sjb } 1475113658Sdeischen /* Only release the mutex if we're done: */ 1476113658Sdeischen if (done != 0) 1477113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 147844963Sjb } 1479113658Sdeischen } while (done == 0); 148044963Sjb} 148144963Sjb 148244963Sjbstatic void 1483113658Sdeischenmutex_rescan_owned(struct pthread *curthread, struct pthread *pthread, 1484113658Sdeischen struct pthread_mutex *mutex) 148544963Sjb{ 1486113658Sdeischen struct pthread_mutex *m; 1487113658Sdeischen struct pthread *pthread_next; 1488113658Sdeischen int active_prio, inherited_prio; 148944963Sjb 149044963Sjb /* 149144963Sjb * Start walking the mutexes the thread has taken since 149244963Sjb * taking this mutex. 149344963Sjb */ 149444963Sjb if (mutex == NULL) { 149544963Sjb /* 149644963Sjb * A null mutex means start at the beginning of the owned 149744963Sjb * mutex list. 149844963Sjb */ 149944963Sjb m = TAILQ_FIRST(&pthread->mutexq); 150044963Sjb 150144963Sjb /* There is no inherited priority yet. */ 150244963Sjb inherited_prio = 0; 1503113658Sdeischen } else { 150444963Sjb /* 150544963Sjb * The caller wants to start after a specific mutex. It 150644963Sjb * is assumed that this mutex is a priority inheritence 150744963Sjb * mutex and that its priority has been correctly 150844963Sjb * calculated. 150944963Sjb */ 151044963Sjb m = TAILQ_NEXT(mutex, m_qe); 151144963Sjb 151244963Sjb /* Start inheriting priority from the specified mutex. */ 151344963Sjb inherited_prio = mutex->m_prio; 151444963Sjb } 151544963Sjb active_prio = MAX(inherited_prio, pthread->base_priority); 151644963Sjb 1517113658Sdeischen for (; m != NULL; m = TAILQ_NEXT(m, m_qe)) { 151844963Sjb /* 151944963Sjb * We only want to deal with priority inheritence 152044963Sjb * mutexes. This might be optimized by only placing 152144963Sjb * priority inheritence mutexes into the owned mutex 152244963Sjb * list, but it may prove to be useful having all 152344963Sjb * owned mutexes in this list. Consider a thread 152444963Sjb * exiting while holding mutexes... 152544963Sjb */ 152644963Sjb if (m->m_protocol == PTHREAD_PRIO_INHERIT) { 152744963Sjb /* 152844963Sjb * Fix the owners saved (inherited) priority to 152944963Sjb * reflect the priority of the previous mutex. 153044963Sjb */ 153144963Sjb m->m_saved_prio = inherited_prio; 153244963Sjb 153344963Sjb if ((pthread_next = TAILQ_FIRST(&m->m_queue)) != NULL) 153444963Sjb /* Recalculate the priority of the mutex: */ 153544963Sjb m->m_prio = MAX(active_prio, 153644963Sjb pthread_next->active_priority); 153744963Sjb else 153844963Sjb m->m_prio = active_prio; 153944963Sjb 154044963Sjb /* Recalculate new inherited and active priorities: */ 154144963Sjb inherited_prio = m->m_prio; 154244963Sjb active_prio = MAX(m->m_prio, pthread->base_priority); 154344963Sjb } 154444963Sjb } 154544963Sjb 154644963Sjb /* 154744963Sjb * Fix the threads inherited priority and recalculate its 154844963Sjb * active priority. 154944963Sjb */ 155044963Sjb pthread->inherited_priority = inherited_prio; 155144963Sjb active_prio = MAX(inherited_prio, pthread->base_priority); 155244963Sjb 155344963Sjb if (active_prio != pthread->active_priority) { 1554113658Sdeischen /* Lock the thread's scheduling queue: */ 1555113658Sdeischen THR_SCHED_LOCK(curthread, pthread); 1556113658Sdeischen 1557113658Sdeischen if ((pthread->flags & THR_FLAGS_IN_RUNQ) == 0) { 155844963Sjb /* 1559113658Sdeischen * This thread is not in a run queue. Just set 1560113658Sdeischen * its active priority. 156144963Sjb */ 1562113658Sdeischen pthread->active_priority = active_prio; 1563113658Sdeischen } 1564113658Sdeischen else { 1565113658Sdeischen /* 1566113658Sdeischen * This thread is in a run queue. Remove it from 1567113658Sdeischen * the queue before changing its priority: 1568113658Sdeischen */ 1569113658Sdeischen THR_RUNQ_REMOVE(pthread); 157044963Sjb 157144963Sjb /* 157244963Sjb * POSIX states that if the priority is being 157344963Sjb * lowered, the thread must be inserted at the 157444963Sjb * head of the queue for its priority if it owns 157544963Sjb * any priority protection or inheritence mutexes. 157644963Sjb */ 157744963Sjb if ((active_prio < pthread->active_priority) && 157844963Sjb (pthread->priority_mutex_count > 0)) { 157944963Sjb /* Set the new active priority. */ 158044963Sjb pthread->active_priority = active_prio; 158144963Sjb 1582113658Sdeischen THR_RUNQ_INSERT_HEAD(pthread); 1583113658Sdeischen } else { 158444963Sjb /* Set the new active priority. */ 158544963Sjb pthread->active_priority = active_prio; 158644963Sjb 1587113658Sdeischen THR_RUNQ_INSERT_TAIL(pthread); 158844963Sjb } 158944963Sjb } 1590113658Sdeischen THR_SCHED_UNLOCK(curthread, pthread); 159144963Sjb } 159244963Sjb} 159344963Sjb 159453812Salfredvoid 159553812Salfred_mutex_unlock_private(pthread_t pthread) 159653812Salfred{ 159753812Salfred struct pthread_mutex *m, *m_next; 159853812Salfred 159953812Salfred for (m = TAILQ_FIRST(&pthread->mutexq); m != NULL; m = m_next) { 160053812Salfred m_next = TAILQ_NEXT(m, m_qe); 160153812Salfred if ((m->m_flags & MUTEX_FLAGS_PRIVATE) != 0) 1602174112Sdeischen _pthread_mutex_unlock(&m); 160353812Salfred } 160453812Salfred} 160553812Salfred 1606113658Sdeischen/* 1607113658Sdeischen * This is called by the current thread when it wants to back out of a 1608113658Sdeischen * mutex_lock in order to run a signal handler. 1609113658Sdeischen */ 1610139023Sdeischenstatic void 1611139023Sdeischenmutex_lock_backout(void *arg) 161267097Sdeischen{ 1613139023Sdeischen struct pthread *curthread = (struct pthread *)arg; 1614113658Sdeischen struct pthread_mutex *m; 161567097Sdeischen 1616113658Sdeischen if ((curthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) { 1617113658Sdeischen /* 1618113658Sdeischen * Any other thread may clear the "in sync queue flag", 1619113658Sdeischen * but only the current thread can clear the pointer 1620113658Sdeischen * to the mutex. So if the flag is set, we can 1621113658Sdeischen * guarantee that the pointer to the mutex is valid. 1622113658Sdeischen * The only problem may be if the mutex is destroyed 1623113658Sdeischen * out from under us, but that should be considered 1624113658Sdeischen * an application bug. 1625113658Sdeischen */ 1626113658Sdeischen m = curthread->data.mutex; 162767097Sdeischen 162867097Sdeischen /* Lock the mutex structure: */ 1629113658Sdeischen THR_LOCK_ACQUIRE(curthread, &m->m_lock); 163067097Sdeischen 163167097Sdeischen 1632113658Sdeischen /* 1633113658Sdeischen * Check to make sure this thread doesn't already own 1634113658Sdeischen * the mutex. Since mutexes are unlocked with direct 1635113658Sdeischen * handoffs, it is possible the previous owner gave it 1636113658Sdeischen * to us after we checked the sync queue flag and before 1637113658Sdeischen * we locked the mutex structure. 1638113658Sdeischen */ 1639113658Sdeischen if (m->m_owner == curthread) { 1640113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 1641113658Sdeischen mutex_unlock_common(&m, /* add_reference */ 0); 1642113658Sdeischen } else { 1643113658Sdeischen /* 1644113658Sdeischen * Remove ourselves from the mutex queue and 1645113658Sdeischen * clear the pointer to the mutex. We may no 1646113658Sdeischen * longer be in the mutex queue, but the removal 1647113658Sdeischen * function will DTRT. 1648113658Sdeischen */ 1649113658Sdeischen mutex_queue_remove(m, curthread); 1650113658Sdeischen curthread->data.mutex = NULL; 1651113658Sdeischen THR_LOCK_RELEASE(curthread, &m->m_lock); 1652113658Sdeischen } 1653113658Sdeischen } 1654139023Sdeischen /* No need to call this again. */ 1655139023Sdeischen curthread->sigbackout = NULL; 1656113658Sdeischen} 1657113658Sdeischen 1658113658Sdeischen/* 1659113658Sdeischen * Dequeue a waiting thread from the head of a mutex queue in descending 1660113658Sdeischen * priority order. 1661113658Sdeischen * 1662113658Sdeischen * In order to properly dequeue a thread from the mutex queue and 1663113658Sdeischen * make it runnable without the possibility of errant wakeups, it 1664113658Sdeischen * is necessary to lock the thread's scheduling queue while also 1665113658Sdeischen * holding the mutex lock. 1666113658Sdeischen */ 1667117907Sdeischenstatic struct kse_mailbox * 1668113658Sdeischenmutex_handoff(struct pthread *curthread, struct pthread_mutex *mutex) 1669113658Sdeischen{ 1670117907Sdeischen struct kse_mailbox *kmbx = NULL; 1671117714Sdeischen struct pthread *pthread; 1672113658Sdeischen 1673113658Sdeischen /* Keep dequeueing until we find a valid thread: */ 1674113658Sdeischen mutex->m_owner = NULL; 1675113658Sdeischen pthread = TAILQ_FIRST(&mutex->m_queue); 1676113658Sdeischen while (pthread != NULL) { 1677113658Sdeischen /* Take the thread's scheduling lock: */ 1678113658Sdeischen THR_SCHED_LOCK(curthread, pthread); 1679113658Sdeischen 1680113658Sdeischen /* Remove the thread from the mutex queue: */ 1681113658Sdeischen TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); 1682113658Sdeischen pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; 1683113658Sdeischen 1684113658Sdeischen /* 1685113658Sdeischen * Only exit the loop if the thread hasn't been 1686113658Sdeischen * cancelled. 1687113658Sdeischen */ 1688113658Sdeischen switch (mutex->m_protocol) { 1689113658Sdeischen case PTHREAD_PRIO_NONE: 1690113658Sdeischen /* 1691113658Sdeischen * Assign the new owner and add the mutex to the 1692113658Sdeischen * thread's list of owned mutexes. 1693113658Sdeischen */ 1694113658Sdeischen mutex->m_owner = pthread; 1695113658Sdeischen TAILQ_INSERT_TAIL(&pthread->mutexq, mutex, m_qe); 1696113658Sdeischen break; 169767097Sdeischen 1698113658Sdeischen case PTHREAD_PRIO_INHERIT: 1699113658Sdeischen /* 1700113658Sdeischen * Assign the new owner and add the mutex to the 1701113658Sdeischen * thread's list of owned mutexes. 1702113658Sdeischen */ 1703113658Sdeischen mutex->m_owner = pthread; 1704113658Sdeischen TAILQ_INSERT_TAIL(&pthread->mutexq, mutex, m_qe); 1705113658Sdeischen 1706113658Sdeischen /* Track number of priority mutexes owned: */ 1707113658Sdeischen pthread->priority_mutex_count++; 1708113658Sdeischen 1709113658Sdeischen /* 1710113658Sdeischen * Set the priority of the mutex. Since our waiting 1711113658Sdeischen * threads are in descending priority order, the 1712113658Sdeischen * priority of the mutex becomes the active priority 1713113658Sdeischen * of the thread we just dequeued. 1714113658Sdeischen */ 1715113658Sdeischen mutex->m_prio = pthread->active_priority; 1716113658Sdeischen 1717113658Sdeischen /* Save the owning threads inherited priority: */ 1718113658Sdeischen mutex->m_saved_prio = pthread->inherited_priority; 1719113658Sdeischen 1720113658Sdeischen /* 1721113658Sdeischen * The owning threads inherited priority now becomes 1722113658Sdeischen * his active priority (the priority of the mutex). 1723113658Sdeischen */ 1724113658Sdeischen pthread->inherited_priority = mutex->m_prio; 1725113658Sdeischen break; 1726113658Sdeischen 1727113658Sdeischen case PTHREAD_PRIO_PROTECT: 1728113658Sdeischen if (pthread->active_priority > mutex->m_prio) { 1729113658Sdeischen /* 1730113658Sdeischen * Either the mutex ceiling priority has 1731113658Sdeischen * been lowered and/or this threads priority 1732113658Sdeischen * has been raised subsequent to the thread 1733113658Sdeischen * being queued on the waiting list. 1734113658Sdeischen */ 1735113658Sdeischen pthread->error = EINVAL; 1736113658Sdeischen } 1737113658Sdeischen else { 1738113658Sdeischen /* 1739113658Sdeischen * Assign the new owner and add the mutex 1740113658Sdeischen * to the thread's list of owned mutexes. 1741113658Sdeischen */ 1742113658Sdeischen mutex->m_owner = pthread; 1743113658Sdeischen TAILQ_INSERT_TAIL(&pthread->mutexq, 1744113658Sdeischen mutex, m_qe); 1745113658Sdeischen 1746113658Sdeischen /* Track number of priority mutexes owned: */ 1747113658Sdeischen pthread->priority_mutex_count++; 1748113658Sdeischen 1749113658Sdeischen /* 1750113658Sdeischen * Save the owning threads inherited 1751113658Sdeischen * priority: 1752113658Sdeischen */ 1753113658Sdeischen mutex->m_saved_prio = 1754113658Sdeischen pthread->inherited_priority; 1755113658Sdeischen 1756113658Sdeischen /* 1757113658Sdeischen * The owning thread inherits the ceiling 1758113658Sdeischen * priority of the mutex and executes at 1759113658Sdeischen * that priority: 1760113658Sdeischen */ 1761113658Sdeischen pthread->inherited_priority = mutex->m_prio; 1762113658Sdeischen pthread->active_priority = mutex->m_prio; 1763113658Sdeischen 1764113658Sdeischen } 1765113658Sdeischen break; 1766113658Sdeischen } 1767113658Sdeischen 1768113658Sdeischen /* Make the thread runnable and unlock the scheduling queue: */ 1769117907Sdeischen kmbx = _thr_setrunnable_unlocked(pthread); 1770117714Sdeischen 1771117714Sdeischen /* Add a preemption point. */ 1772117714Sdeischen if ((curthread->kseg == pthread->kseg) && 1773117714Sdeischen (pthread->active_priority > curthread->active_priority)) 1774117714Sdeischen curthread->critical_yield = 1; 1775117714Sdeischen 1776139023Sdeischen if (mutex->m_owner == pthread) { 1777113658Sdeischen /* We're done; a valid owner was found. */ 1778139023Sdeischen if (mutex->m_flags & MUTEX_FLAGS_PRIVATE) 1779139023Sdeischen THR_CRITICAL_ENTER(pthread); 1780139023Sdeischen THR_SCHED_UNLOCK(curthread, pthread); 1781113658Sdeischen break; 1782139023Sdeischen } 1783139023Sdeischen THR_SCHED_UNLOCK(curthread, pthread); 1784139023Sdeischen /* Get the next thread from the waiting queue: */ 1785139023Sdeischen pthread = TAILQ_NEXT(pthread, sqe); 178667097Sdeischen } 1787113658Sdeischen 1788113658Sdeischen if ((pthread == NULL) && (mutex->m_protocol == PTHREAD_PRIO_INHERIT)) 1789113658Sdeischen /* This mutex has no priority: */ 1790113658Sdeischen mutex->m_prio = 0; 1791117907Sdeischen return (kmbx); 179267097Sdeischen} 179367097Sdeischen 179444963Sjb/* 179544963Sjb * Dequeue a waiting thread from the head of a mutex queue in descending 179644963Sjb * priority order. 179744963Sjb */ 179844963Sjbstatic inline pthread_t 1799113658Sdeischenmutex_queue_deq(struct pthread_mutex *mutex) 180044963Sjb{ 180144963Sjb pthread_t pthread; 180244963Sjb 180353812Salfred while ((pthread = TAILQ_FIRST(&mutex->m_queue)) != NULL) { 180467097Sdeischen TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); 1805113658Sdeischen pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; 180644963Sjb 180753812Salfred /* 180853812Salfred * Only exit the loop if the thread hasn't been 180953812Salfred * cancelled. 181053812Salfred */ 181153812Salfred if (pthread->interrupted == 0) 181253812Salfred break; 181353812Salfred } 181453812Salfred 1815113658Sdeischen return (pthread); 181644963Sjb} 181744963Sjb 181844963Sjb/* 181944963Sjb * Remove a waiting thread from a mutex queue in descending priority order. 182044963Sjb */ 182144963Sjbstatic inline void 182244963Sjbmutex_queue_remove(pthread_mutex_t mutex, pthread_t pthread) 182344963Sjb{ 1824113658Sdeischen if ((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) { 182567097Sdeischen TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); 1826113658Sdeischen pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; 182753812Salfred } 182844963Sjb} 182944963Sjb 183044963Sjb/* 183144963Sjb * Enqueue a waiting thread to a queue in descending priority order. 183244963Sjb */ 183344963Sjbstatic inline void 183444963Sjbmutex_queue_enq(pthread_mutex_t mutex, pthread_t pthread) 183544963Sjb{ 183644963Sjb pthread_t tid = TAILQ_LAST(&mutex->m_queue, mutex_head); 183744963Sjb 1838113658Sdeischen THR_ASSERT_NOT_IN_SYNCQ(pthread); 183944963Sjb /* 184044963Sjb * For the common case of all threads having equal priority, 184144963Sjb * we perform a quick check against the priority of the thread 184244963Sjb * at the tail of the queue. 184344963Sjb */ 184444963Sjb if ((tid == NULL) || (pthread->active_priority <= tid->active_priority)) 184567097Sdeischen TAILQ_INSERT_TAIL(&mutex->m_queue, pthread, sqe); 184644963Sjb else { 184744963Sjb tid = TAILQ_FIRST(&mutex->m_queue); 184844963Sjb while (pthread->active_priority <= tid->active_priority) 184967097Sdeischen tid = TAILQ_NEXT(tid, sqe); 185067097Sdeischen TAILQ_INSERT_BEFORE(tid, pthread, sqe); 185144963Sjb } 1852113658Sdeischen pthread->sflags |= THR_FLAGS_IN_SYNCQ; 185344963Sjb} 1854176060Sdes 1855176060Sdesint 1856176060Sdes_pthread_mutex_isowned_np(pthread_mutex_t *mutex) 1857176060Sdes{ 1858176060Sdes struct pthread *curthread = _get_curthread(); 1859176060Sdes 1860176060Sdes return ((*mutex)->m_owner == curthread); 1861176060Sdes} 1862176060Sdes 1863