kern_umtx.c revision 112904
1112904Sjeff/* 2112904Sjeff * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 3112904Sjeff * All rights reserved. 4112904Sjeff * 5112904Sjeff * Redistribution and use in source and binary forms, with or without 6112904Sjeff * modification, are permitted provided that the following conditions 7112904Sjeff * are met: 8112904Sjeff * 1. Redistributions of source code must retain the above copyright 9112904Sjeff * notice unmodified, this list of conditions, and the following 10112904Sjeff * disclaimer. 11112904Sjeff * 2. Redistributions in binary form must reproduce the above copyright 12112904Sjeff * notice, this list of conditions and the following disclaimer in the 13112904Sjeff * documentation and/or other materials provided with the distribution. 14112904Sjeff * 15112904Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16112904Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17112904Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18112904Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19112904Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20112904Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21112904Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22112904Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23112904Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24112904Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25112904Sjeff * 26112904Sjeff * $FreeBSD: head/sys/kern/kern_umtx.c 112904 2003-04-01 01:10:42Z jeff $ 27112904Sjeff * 28112904Sjeff */ 29112904Sjeff 30112904Sjeff#include <sys/param.h> 31112904Sjeff#include <sys/kernel.h> 32112904Sjeff#include <sys/lock.h> 33112904Sjeff#include <sys/mutex.h> 34112904Sjeff#include <sys/proc.h> 35112904Sjeff#include <sys/signalvar.h> 36112904Sjeff#include <sys/sysent.h> 37112904Sjeff#include <sys/systm.h> 38112904Sjeff#include <sys/sysproto.h> 39112904Sjeff#include <sys/thr.h> 40112904Sjeff#include <sys/umtx.h> 41112904Sjeff 42112904Sjeffint 43112904Sjeff_umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 44112904Sjeff /* struct umtx *umtx */ 45112904Sjeff{ 46112904Sjeff struct umtx *umtx; 47112904Sjeff struct thread *blocked; 48112904Sjeff intptr_t owner; 49112904Sjeff int error; 50112904Sjeff 51112904Sjeff error = 0; 52112904Sjeff 53112904Sjeff /* 54112904Sjeff * Care must be exercised when dealing with this structure. It 55112904Sjeff * can fault on any access. 56112904Sjeff */ 57112904Sjeff umtx = uap->umtx; 58112904Sjeff 59112904Sjeff PROC_LOCK(td->td_proc); 60112904Sjeff 61112904Sjeff for (;;) { 62112904Sjeff /* 63112904Sjeff * Try the uncontested case. This should be done in userland. 64112904Sjeff */ 65112904Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, 66112904Sjeff UMTX_UNOWNED, (intptr_t)td); 67112904Sjeff 68112904Sjeff /* The acquire succeeded. */ 69112904Sjeff if (owner == (intptr_t)td) { 70112904Sjeff error = 0; 71112904Sjeff goto out; 72112904Sjeff } 73112904Sjeff 74112904Sjeff /* The address was invalid. */ 75112904Sjeff if (owner == -1) { 76112904Sjeff error = EFAULT; 77112904Sjeff goto out; 78112904Sjeff } 79112904Sjeff 80112904Sjeff if (owner & UMTX_CONTESTED) 81112904Sjeff break; 82112904Sjeff 83112904Sjeff /* 84112904Sjeff * Set the contested bit so that a release in user space 85112904Sjeff * knows to use the system call for unlock. If this fails 86112904Sjeff * either some one else has acquired the lock or it has been 87112904Sjeff * released. 88112904Sjeff */ 89112904Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, owner, owner | UMTX_CONTESTED); 90112904Sjeff 91112904Sjeff /* The contested bit was set. */ 92112904Sjeff if (owner & UMTX_CONTESTED) 93112904Sjeff break; 94112904Sjeff 95112904Sjeff /* The address was invalid. */ 96112904Sjeff if (owner == -1) { 97112904Sjeff error = EFAULT; 98112904Sjeff goto out; 99112904Sjeff } 100112904Sjeff /* We didn't set the contested bit, try again. */ 101112904Sjeff } 102112904Sjeff 103112904Sjeff /* 104112904Sjeff * We are now protected from further races via the proc lock. 105112904Sjeff * If userland messes with their mutex without using cmpset 106112904Sjeff * they will deadlock themselves but they will still be 107112904Sjeff * killable via signals. 108112904Sjeff */ 109112904Sjeff 110112904Sjeff if ((owner = fuword(&umtx->u_blocked)) == -1) { 111112904Sjeff error = EFAULT; 112112904Sjeff goto out; 113112904Sjeff } 114112904Sjeff 115112904Sjeff if (owner == UMTX_UNOWNED) { 116112904Sjeff if (suword(&umtx->u_blocked, (long)td) == -1) { 117112904Sjeff error = EFAULT; 118112904Sjeff goto out; 119112904Sjeff } 120112904Sjeff /* 121112904Sjeff * Other blocked threads will reside here. 122112904Sjeff */ 123112904Sjeff STAILQ_INIT(&td->td_umtxq); 124112904Sjeff } else { 125112904Sjeff FOREACH_THREAD_IN_PROC(td->td_proc, blocked) 126112904Sjeff if (blocked == (struct thread *)(owner)) 127112904Sjeff break; 128112904Sjeff 129112904Sjeff if (blocked == NULL) { 130112904Sjeff error = EINVAL; 131112904Sjeff goto out; 132112904Sjeff } 133112904Sjeff /* 134112904Sjeff * Insert us onto the end of the TAILQ. 135112904Sjeff */ 136112904Sjeff STAILQ_INSERT_TAIL(&blocked->td_umtxq, td, td_umtx); 137112904Sjeff } 138112904Sjeff 139112904Sjeff for (;;) { 140112904Sjeff /* 141112904Sjeff * Sleep until we can acquire the lock. We must still deliver 142112904Sjeff * signals so that they are not deferred until we acquire the 143112904Sjeff * lock which may be never. The threads actual priority is 144112904Sjeff * used to maintain proper ordering. 145112904Sjeff */ 146112904Sjeff 147112904Sjeff error = msleep(&td->td_umtx, &td->td_proc->p_mtx, 148112904Sjeff td->td_priority | PCATCH, "umtx", 0); 149112904Sjeff 150112904Sjeff /* 151112904Sjeff * When we are woken up we need to see if we now own the lock 152112904Sjeff * even if a signal was delivered. 153112904Sjeff */ 154112904Sjeff if ((owner = fuword(&umtx->u_owner)) == -1) { 155112904Sjeff error = EFAULT; 156112904Sjeff break; 157112904Sjeff } 158112904Sjeff owner &= ~UMTX_CONTESTED; 159112904Sjeff if ((struct thread *)owner == td) { 160112904Sjeff error = 0; 161112904Sjeff break; 162112904Sjeff } 163112904Sjeff 164112904Sjeff /* 165112904Sjeff * We may have signals to deliver. 166112904Sjeff */ 167112904Sjeff if (error) 168112904Sjeff break; 169112904Sjeff } 170112904Sjeff 171112904Sjeffout: 172112904Sjeff PROC_UNLOCK(td->td_proc); 173112904Sjeff 174112904Sjeff return (error); 175112904Sjeff} 176112904Sjeff 177112904Sjeffint 178112904Sjeff_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 179112904Sjeff /* struct umtx *umtx */ 180112904Sjeff{ 181112904Sjeff struct thread *td0; 182112904Sjeff struct umtx *umtx; 183112904Sjeff intptr_t owner; 184112904Sjeff intptr_t blocked; 185112904Sjeff int error; 186112904Sjeff 187112904Sjeff error = 0; 188112904Sjeff umtx = uap->umtx; 189112904Sjeff 190112904Sjeff PROC_LOCK(td->td_proc); 191112904Sjeff 192112904Sjeff /* 193112904Sjeff * Make sure we own this mtx. 194112904Sjeff * 195112904Sjeff * XXX Need a {fu,su}ptr this is not correct on arch where 196112904Sjeff * sizeof(intptr_t) != sizeof(long). 197112904Sjeff */ 198112904Sjeff if ((owner = fuword(&umtx->u_owner)) == -1) { 199112904Sjeff error = EFAULT; 200112904Sjeff goto out; 201112904Sjeff } 202112904Sjeff if ((struct thread *)(owner & ~UMTX_CONTESTED) != td) { 203112904Sjeff error = EPERM; 204112904Sjeff goto out; 205112904Sjeff } 206112904Sjeff /* 207112904Sjeff * If we own it but it isn't contested then we can just release and 208112904Sjeff * return. 209112904Sjeff */ 210112904Sjeff if ((owner & UMTX_CONTESTED) == 0) { 211112904Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, 212112904Sjeff (intptr_t)td, UMTX_UNOWNED); 213112904Sjeff 214112904Sjeff if (owner == -1) 215112904Sjeff error = EFAULT; 216112904Sjeff /* 217112904Sjeff * If this failed someone modified the memory without going 218112904Sjeff * through this api. 219112904Sjeff */ 220112904Sjeff else if (owner != UMTX_UNOWNED) 221112904Sjeff error = EINVAL; 222112904Sjeff else 223112904Sjeff error = 0; 224112904Sjeff 225112904Sjeff goto out; 226112904Sjeff } 227112904Sjeff 228112904Sjeff /* 229112904Sjeff * Since we own the mutex and the proc lock we are free to inspect 230112904Sjeff * the blocked queue. It must have one valid entry since the 231112904Sjeff * CONTESTED bit was set. 232112904Sjeff */ 233112904Sjeff blocked = fuword(&umtx->u_blocked); 234112904Sjeff if (blocked == -1){ 235112904Sjeff error = EFAULT; 236112904Sjeff goto out; 237112904Sjeff } 238112904Sjeff if (blocked == 0) { 239112904Sjeff error = EINVAL; 240112904Sjeff goto out; 241112904Sjeff } 242112904Sjeff 243112904Sjeff FOREACH_THREAD_IN_PROC(td->td_proc, td0) 244112904Sjeff if (td0 == (struct thread *)blocked) 245112904Sjeff break; 246112904Sjeff 247112904Sjeff if (td0 == NULL) { 248112904Sjeff error = EINVAL; 249112904Sjeff goto out; 250112904Sjeff } 251112904Sjeff 252112904Sjeff if (!STAILQ_EMPTY(&td0->td_umtxq)) { 253112904Sjeff struct thread *next; 254112904Sjeff 255112904Sjeff blocked |= UMTX_CONTESTED; 256112904Sjeff next = STAILQ_FIRST(&td0->td_umtxq); 257112904Sjeff if (suword(&umtx->u_blocked, (long)next) == -1) { 258112904Sjeff error = EFAULT; 259112904Sjeff goto out; 260112904Sjeff } 261112904Sjeff STAILQ_REMOVE_HEAD(&td0->td_umtxq, td_umtx); 262112904Sjeff 263112904Sjeff /* 264112904Sjeff * Switch the queue over to the next blocked thread. 265112904Sjeff */ 266112904Sjeff if (!STAILQ_EMPTY(&td0->td_umtxq)) { 267112904Sjeff next->td_umtxq = td0->td_umtxq; 268112904Sjeff STAILQ_INIT(&td0->td_umtxq); 269112904Sjeff } else 270112904Sjeff STAILQ_INIT(&next->td_umtxq); 271112904Sjeff } else { 272112904Sjeff if (suword(&umtx->u_blocked, UMTX_UNOWNED) == -1) { 273112904Sjeff error = EFAULT; 274112904Sjeff goto out; 275112904Sjeff } 276112904Sjeff } 277112904Sjeff /* 278112904Sjeff * Now directly assign this mutex to the first thread that was 279112904Sjeff * blocked on it. 280112904Sjeff */ 281112904Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, owner, blocked); 282112904Sjeff 283112904Sjeff /* 284112904Sjeff * This will only happen if someone modifies the lock without going 285112904Sjeff * through this api. 286112904Sjeff */ 287112904Sjeff if (owner != blocked) { 288112904Sjeff error = EINVAL; 289112904Sjeff goto out; 290112904Sjeff } 291112904Sjeff if (owner == -1) { 292112904Sjeff error = EFAULT; 293112904Sjeff goto out; 294112904Sjeff } 295112904Sjeff /* Success. */ 296112904Sjeff error = 0; 297112904Sjeff wakeup(&td0->td_umtx); 298112904Sjeff 299112904Sjeffout: 300112904Sjeff PROC_UNLOCK(td->td_proc); 301112904Sjeff 302112904Sjeff return (error); 303112904Sjeff} 304