kern_umtx.c revision 112967
1112904Sjeff/* 2112904Sjeff * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 3112904Sjeff * All rights reserved. 4112904Sjeff * 5112904Sjeff * Redistribution and use in source and binary forms, with or without 6112904Sjeff * modification, are permitted provided that the following conditions 7112904Sjeff * are met: 8112904Sjeff * 1. Redistributions of source code must retain the above copyright 9112904Sjeff * notice unmodified, this list of conditions, and the following 10112904Sjeff * disclaimer. 11112904Sjeff * 2. Redistributions in binary form must reproduce the above copyright 12112904Sjeff * notice, this list of conditions and the following disclaimer in the 13112904Sjeff * documentation and/or other materials provided with the distribution. 14112904Sjeff * 15112904Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16112904Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17112904Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18112904Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19112904Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20112904Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21112904Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22112904Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23112904Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24112904Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25112904Sjeff * 26112904Sjeff * $FreeBSD: head/sys/kern/kern_umtx.c 112967 2003-04-02 08:02:27Z jake $ 27112904Sjeff * 28112904Sjeff */ 29112904Sjeff 30112904Sjeff#include <sys/param.h> 31112904Sjeff#include <sys/kernel.h> 32112904Sjeff#include <sys/lock.h> 33112904Sjeff#include <sys/mutex.h> 34112904Sjeff#include <sys/proc.h> 35112904Sjeff#include <sys/signalvar.h> 36112904Sjeff#include <sys/sysent.h> 37112904Sjeff#include <sys/systm.h> 38112904Sjeff#include <sys/sysproto.h> 39112904Sjeff#include <sys/thr.h> 40112904Sjeff#include <sys/umtx.h> 41112904Sjeff 42112904Sjeffint 43112904Sjeff_umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 44112904Sjeff /* struct umtx *umtx */ 45112904Sjeff{ 46112904Sjeff struct umtx *umtx; 47112904Sjeff struct thread *blocked; 48112904Sjeff intptr_t owner; 49112967Sjake intptr_t old; 50112904Sjeff int error; 51112904Sjeff 52112904Sjeff error = 0; 53112904Sjeff 54112904Sjeff /* 55112904Sjeff * Care must be exercised when dealing with this structure. It 56112904Sjeff * can fault on any access. 57112904Sjeff */ 58112904Sjeff umtx = uap->umtx; 59112904Sjeff 60112904Sjeff PROC_LOCK(td->td_proc); 61112904Sjeff 62112904Sjeff for (;;) { 63112904Sjeff /* 64112904Sjeff * Try the uncontested case. This should be done in userland. 65112904Sjeff */ 66112904Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, 67112904Sjeff UMTX_UNOWNED, (intptr_t)td); 68112904Sjeff 69112904Sjeff /* The acquire succeeded. */ 70112967Sjake if (owner == UMTX_UNOWNED) { 71112904Sjeff error = 0; 72112904Sjeff goto out; 73112904Sjeff } 74112904Sjeff 75112904Sjeff /* The address was invalid. */ 76112904Sjeff if (owner == -1) { 77112904Sjeff error = EFAULT; 78112904Sjeff goto out; 79112904Sjeff } 80112904Sjeff 81112904Sjeff if (owner & UMTX_CONTESTED) 82112904Sjeff break; 83112904Sjeff 84112904Sjeff /* 85112904Sjeff * Set the contested bit so that a release in user space 86112904Sjeff * knows to use the system call for unlock. If this fails 87112904Sjeff * either some one else has acquired the lock or it has been 88112904Sjeff * released. 89112904Sjeff */ 90112967Sjake old = casuptr((intptr_t *)&umtx->u_owner, owner, 91112967Sjake owner | UMTX_CONTESTED); 92112904Sjeff 93112967Sjake /* We set the contested bit. */ 94112967Sjake if (old == owner) 95112904Sjeff break; 96112904Sjeff 97112904Sjeff /* The address was invalid. */ 98112967Sjake if (old == -1) { 99112904Sjeff error = EFAULT; 100112904Sjeff goto out; 101112904Sjeff } 102112904Sjeff /* We didn't set the contested bit, try again. */ 103112904Sjeff } 104112904Sjeff 105112904Sjeff /* 106112904Sjeff * We are now protected from further races via the proc lock. 107112904Sjeff * If userland messes with their mutex without using cmpset 108112904Sjeff * they will deadlock themselves but they will still be 109112904Sjeff * killable via signals. 110112904Sjeff */ 111112904Sjeff 112112904Sjeff if ((owner = fuword(&umtx->u_blocked)) == -1) { 113112904Sjeff error = EFAULT; 114112904Sjeff goto out; 115112904Sjeff } 116112904Sjeff 117112904Sjeff if (owner == UMTX_UNOWNED) { 118112904Sjeff if (suword(&umtx->u_blocked, (long)td) == -1) { 119112904Sjeff error = EFAULT; 120112904Sjeff goto out; 121112904Sjeff } 122112904Sjeff /* 123112904Sjeff * Other blocked threads will reside here. 124112904Sjeff */ 125112904Sjeff STAILQ_INIT(&td->td_umtxq); 126112904Sjeff } else { 127112904Sjeff FOREACH_THREAD_IN_PROC(td->td_proc, blocked) 128112904Sjeff if (blocked == (struct thread *)(owner)) 129112904Sjeff break; 130112904Sjeff 131112904Sjeff if (blocked == NULL) { 132112904Sjeff error = EINVAL; 133112904Sjeff goto out; 134112904Sjeff } 135112904Sjeff /* 136112904Sjeff * Insert us onto the end of the TAILQ. 137112904Sjeff */ 138112904Sjeff STAILQ_INSERT_TAIL(&blocked->td_umtxq, td, td_umtx); 139112904Sjeff } 140112904Sjeff 141112904Sjeff for (;;) { 142112904Sjeff /* 143112904Sjeff * Sleep until we can acquire the lock. We must still deliver 144112904Sjeff * signals so that they are not deferred until we acquire the 145112904Sjeff * lock which may be never. The threads actual priority is 146112904Sjeff * used to maintain proper ordering. 147112904Sjeff */ 148112904Sjeff 149112904Sjeff error = msleep(&td->td_umtx, &td->td_proc->p_mtx, 150112904Sjeff td->td_priority | PCATCH, "umtx", 0); 151112904Sjeff 152112904Sjeff /* 153112904Sjeff * When we are woken up we need to see if we now own the lock 154112904Sjeff * even if a signal was delivered. 155112904Sjeff */ 156112904Sjeff if ((owner = fuword(&umtx->u_owner)) == -1) { 157112904Sjeff error = EFAULT; 158112904Sjeff break; 159112904Sjeff } 160112904Sjeff owner &= ~UMTX_CONTESTED; 161112904Sjeff if ((struct thread *)owner == td) { 162112904Sjeff error = 0; 163112904Sjeff break; 164112904Sjeff } 165112904Sjeff 166112904Sjeff /* 167112904Sjeff * We may have signals to deliver. 168112904Sjeff */ 169112904Sjeff if (error) 170112904Sjeff break; 171112904Sjeff } 172112904Sjeff 173112904Sjeffout: 174112904Sjeff PROC_UNLOCK(td->td_proc); 175112904Sjeff 176112904Sjeff return (error); 177112904Sjeff} 178112904Sjeff 179112904Sjeffint 180112904Sjeff_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 181112904Sjeff /* struct umtx *umtx */ 182112904Sjeff{ 183112904Sjeff struct thread *td0; 184112904Sjeff struct umtx *umtx; 185112904Sjeff intptr_t owner; 186112904Sjeff intptr_t blocked; 187112967Sjake intptr_t old; 188112904Sjeff int error; 189112904Sjeff 190112904Sjeff error = 0; 191112904Sjeff umtx = uap->umtx; 192112904Sjeff 193112904Sjeff PROC_LOCK(td->td_proc); 194112904Sjeff 195112904Sjeff /* 196112904Sjeff * Make sure we own this mtx. 197112904Sjeff * 198112904Sjeff * XXX Need a {fu,su}ptr this is not correct on arch where 199112904Sjeff * sizeof(intptr_t) != sizeof(long). 200112904Sjeff */ 201112904Sjeff if ((owner = fuword(&umtx->u_owner)) == -1) { 202112904Sjeff error = EFAULT; 203112904Sjeff goto out; 204112904Sjeff } 205112904Sjeff if ((struct thread *)(owner & ~UMTX_CONTESTED) != td) { 206112904Sjeff error = EPERM; 207112904Sjeff goto out; 208112904Sjeff } 209112904Sjeff /* 210112904Sjeff * If we own it but it isn't contested then we can just release and 211112904Sjeff * return. 212112904Sjeff */ 213112904Sjeff if ((owner & UMTX_CONTESTED) == 0) { 214112904Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, 215112904Sjeff (intptr_t)td, UMTX_UNOWNED); 216112904Sjeff 217112904Sjeff if (owner == -1) 218112904Sjeff error = EFAULT; 219112904Sjeff /* 220112904Sjeff * If this failed someone modified the memory without going 221112904Sjeff * through this api. 222112904Sjeff */ 223112967Sjake else if (owner != (intptr_t)td) 224112904Sjeff error = EINVAL; 225112904Sjeff else 226112904Sjeff error = 0; 227112904Sjeff 228112904Sjeff goto out; 229112904Sjeff } 230112904Sjeff 231112904Sjeff /* 232112904Sjeff * Since we own the mutex and the proc lock we are free to inspect 233112904Sjeff * the blocked queue. It must have one valid entry since the 234112904Sjeff * CONTESTED bit was set. 235112904Sjeff */ 236112904Sjeff blocked = fuword(&umtx->u_blocked); 237112967Sjake if (blocked == -1) { 238112904Sjeff error = EFAULT; 239112904Sjeff goto out; 240112904Sjeff } 241112904Sjeff if (blocked == 0) { 242112904Sjeff error = EINVAL; 243112904Sjeff goto out; 244112904Sjeff } 245112904Sjeff 246112904Sjeff FOREACH_THREAD_IN_PROC(td->td_proc, td0) 247112904Sjeff if (td0 == (struct thread *)blocked) 248112904Sjeff break; 249112904Sjeff 250112904Sjeff if (td0 == NULL) { 251112904Sjeff error = EINVAL; 252112904Sjeff goto out; 253112904Sjeff } 254112904Sjeff 255112904Sjeff if (!STAILQ_EMPTY(&td0->td_umtxq)) { 256112904Sjeff struct thread *next; 257112904Sjeff 258112904Sjeff blocked |= UMTX_CONTESTED; 259112904Sjeff next = STAILQ_FIRST(&td0->td_umtxq); 260112904Sjeff if (suword(&umtx->u_blocked, (long)next) == -1) { 261112904Sjeff error = EFAULT; 262112904Sjeff goto out; 263112904Sjeff } 264112904Sjeff STAILQ_REMOVE_HEAD(&td0->td_umtxq, td_umtx); 265112904Sjeff 266112904Sjeff /* 267112904Sjeff * Switch the queue over to the next blocked thread. 268112904Sjeff */ 269112904Sjeff if (!STAILQ_EMPTY(&td0->td_umtxq)) { 270112904Sjeff next->td_umtxq = td0->td_umtxq; 271112904Sjeff STAILQ_INIT(&td0->td_umtxq); 272112904Sjeff } else 273112904Sjeff STAILQ_INIT(&next->td_umtxq); 274112904Sjeff } else { 275112904Sjeff if (suword(&umtx->u_blocked, UMTX_UNOWNED) == -1) { 276112904Sjeff error = EFAULT; 277112904Sjeff goto out; 278112904Sjeff } 279112904Sjeff } 280112904Sjeff /* 281112904Sjeff * Now directly assign this mutex to the first thread that was 282112904Sjeff * blocked on it. 283112904Sjeff */ 284112967Sjake old = casuptr((intptr_t *)&umtx->u_owner, owner, blocked); 285112904Sjeff 286112904Sjeff /* 287112904Sjeff * This will only happen if someone modifies the lock without going 288112904Sjeff * through this api. 289112904Sjeff */ 290112967Sjake if (old != owner) { 291112904Sjeff error = EINVAL; 292112904Sjeff goto out; 293112904Sjeff } 294112967Sjake if (old == -1) { 295112904Sjeff error = EFAULT; 296112904Sjeff goto out; 297112904Sjeff } 298112904Sjeff /* Success. */ 299112904Sjeff error = 0; 300112904Sjeff wakeup(&td0->td_umtx); 301112904Sjeff 302112904Sjeffout: 303112904Sjeff PROC_UNLOCK(td->td_proc); 304112904Sjeff 305112904Sjeff return (error); 306112904Sjeff} 307