1166970Snetchild/* $NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $ */ 2161304Snetchild 3161304Snetchild/*- 4161304Snetchild * Copyright (c) 2005 Emmanuel Dreyfus, all rights reserved. 5161304Snetchild * 6161304Snetchild * Redistribution and use in source and binary forms, with or without 7161304Snetchild * modification, are permitted provided that the following conditions 8161304Snetchild * are met: 9161304Snetchild * 1. Redistributions of source code must retain the above copyright 10161304Snetchild * notice, this list of conditions and the following disclaimer. 11161304Snetchild * 2. Redistributions in binary form must reproduce the above copyright 12161304Snetchild * notice, this list of conditions and the following disclaimer in the 13161304Snetchild * documentation and/or other materials provided with the distribution. 14161304Snetchild * 3. All advertising materials mentioning features or use of this software 15161304Snetchild * must display the following acknowledgement: 16161304Snetchild * This product includes software developed by Emmanuel Dreyfus 17166969Snetchild * 4. The name of the author may not be used to endorse or promote 18166969Snetchild * products derived from this software without specific prior written 19161304Snetchild * permission. 20161304Snetchild * 21166969Snetchild * THIS SOFTWARE IS PROVIDED BY THE THE AUTHOR AND CONTRIBUTORS ``AS IS'' 22166969Snetchild * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 23161304Snetchild * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24166969Snetchild * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 25161304Snetchild * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26161304Snetchild * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27161304Snetchild * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28161304Snetchild * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29161304Snetchild * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30161304Snetchild * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31161304Snetchild * POSSIBILITY OF SUCH DAMAGE. 32161304Snetchild */ 33161304Snetchild 34161304Snetchild#include <sys/cdefs.h> 35161304Snetchild__FBSDID("$FreeBSD$"); 36161304Snetchild#if 0 37166970Snetchild__KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $"); 38161304Snetchild#endif 39161304Snetchild 40161304Snetchild#include "opt_compat.h" 41246290Sdchagin#include "opt_kdtrace.h" 42161304Snetchild 43161304Snetchild#include <sys/param.h> 44161304Snetchild#include <sys/systm.h> 45178976Srdivacky#include <sys/imgact.h> 46189867Sdchagin#include <sys/kernel.h> 47191887Sdchagin#include <sys/ktr.h> 48161304Snetchild#include <sys/lock.h> 49189867Sdchagin#include <sys/malloc.h> 50161304Snetchild#include <sys/mutex.h> 51178976Srdivacky#include <sys/priv.h> 52189867Sdchagin#include <sys/proc.h> 53189867Sdchagin#include <sys/queue.h> 54178976Srdivacky#include <sys/sched.h> 55246290Sdchagin#include <sys/sdt.h> 56162182Snetchild#include <sys/sx.h> 57218970Sjhb#include <sys/umtx.h> 58161304Snetchild 59161304Snetchild#ifdef COMPAT_LINUX32 60161304Snetchild#include <machine/../linux32/linux.h> 61161304Snetchild#include <machine/../linux32/linux32_proto.h> 62161304Snetchild#else 63161304Snetchild#include <machine/../linux/linux.h> 64161304Snetchild#include <machine/../linux/linux_proto.h> 65161304Snetchild#endif 66246290Sdchagin#include <compat/linux/linux_dtrace.h> 67218668Sdchagin#include <compat/linux/linux_emul.h> 68189861Sdchagin#include <compat/linux/linux_futex.h> 69191887Sdchagin#include <compat/linux/linux_util.h> 70161304Snetchild 71246290Sdchagin/* DTrace init */ 72246290SdchaginLIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE); 73191719Sdchagin 74246290Sdchagin/* Linuxulator-global DTrace probes */ 75246290SdchaginLIN_SDT_PROBE_DECLARE(locks, emul_lock, locked); 76246290SdchaginLIN_SDT_PROBE_DECLARE(locks, emul_lock, unlock); 77246290Sdchagin 78246290Sdchagin/** 79246290Sdchagin * Futex part for the special DTrace module "locks". 80246290Sdchagin */ 81246290SdchaginLIN_SDT_PROBE_DEFINE1(locks, futex_mtx, locked, "struct mtx *"); 82246290SdchaginLIN_SDT_PROBE_DEFINE1(locks, futex_mtx, unlock, "struct mtx *"); 83246290Sdchagin 84246290Sdchagin/** 85246290Sdchagin * Per futex probes. 86246290Sdchagin */ 87246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex, create, "struct sx *"); 88246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex, destroy, "struct sx *"); 89246290Sdchagin 90246290Sdchagin/** 91246290Sdchagin * DTrace probes in this module. 92246290Sdchagin */ 93246290SdchaginLIN_SDT_PROBE_DEFINE2(futex, futex_put, entry, "struct futex *", 94246290Sdchagin "struct waiting_proc *"); 95246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_put, destroy, "uint32_t *", "uint32_t", 96246290Sdchagin "int"); 97246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_put, unlock, "uint32_t *", "uint32_t", 98246290Sdchagin "int"); 99246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, futex_put, return); 100246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_get0, entry, "uint32_t *", "struct futex **", 101246290Sdchagin "uint32_t"); 102246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_get0, umtx_key_get_error, "int"); 103246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_get0, shared, "uint32_t *", "uint32_t", 104246290Sdchagin "int"); 105246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_get0, null, "uint32_t *"); 106246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_get0, new, "uint32_t *", "uint32_t", "int"); 107246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_get0, return, "int"); 108246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_get, entry, "uint32_t *", 109246290Sdchagin "struct waiting_proc **", "struct futex **"); 110246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, futex_get, error); 111246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_get, return, "int"); 112246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_sleep, entry, "struct futex *", 113246290Sdchagin "struct waiting_proc **", "int"); 114246290SdchaginLIN_SDT_PROBE_DEFINE5(futex, futex_sleep, requeue_error, "int", "uint32_t *", 115246290Sdchagin "struct waiting_proc *", "uint32_t *", "uint32_t"); 116246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_sleep, sleep_error, "int", "uint32_t *", 117246290Sdchagin "struct waiting_proc *"); 118246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_sleep, return, "int"); 119246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_wake, entry, "struct futex *", "int", 120246290Sdchagin "uint32_t"); 121246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_wake, iterate, "uint32_t", 122262056Savg "struct waiting_proc *", "uint32_t"); 123246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_wake, wakeup, "struct waiting_proc *"); 124246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_wake, return, "int"); 125246290SdchaginLIN_SDT_PROBE_DEFINE4(futex, futex_requeue, entry, "struct futex *", "int", 126246290Sdchagin "struct futex *", "int"); 127246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_requeue, wakeup, "struct waiting_proc *"); 128246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_requeue, requeue, "uint32_t *", 129246290Sdchagin "struct waiting_proc *", "uint32_t"); 130246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_requeue, return, "int"); 131246290SdchaginLIN_SDT_PROBE_DEFINE4(futex, futex_wait, entry, "struct futex *", 132267006Sdchagin "struct waiting_proc **", "int", "uint32_t"); 133246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_wait, sleep_error, "int"); 134246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_wait, return, "int"); 135246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, futex_atomic_op, entry, "struct thread *", 136246290Sdchagin "int", "uint32_t"); 137246290SdchaginLIN_SDT_PROBE_DEFINE4(futex, futex_atomic_op, decoded_op, "int", "int", "int", 138246290Sdchagin "int"); 139246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, futex_atomic_op, missing_access_check); 140246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_op, "int"); 141246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_cmp, "int"); 142246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, return, "int"); 143246290SdchaginLIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, entry, "struct thread *", 144246290Sdchagin "struct linux_sys_futex_args *"); 145246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_clockswitch); 146267006SdchaginLIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, itimerfix_error, "int"); 147246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, copyin_error, "int"); 148246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, invalid_cmp_requeue_use); 149246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wait, "uint32_t *", 150246290Sdchagin "uint32_t", "uint32_t"); 151246290SdchaginLIN_SDT_PROBE_DEFINE4(futex, linux_sys_futex, debug_wait_value_neq, 152246290Sdchagin "uint32_t *", "uint32_t", "int", "uint32_t"); 153246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wake, "uint32_t *", 154246290Sdchagin "uint32_t", "uint32_t"); 155246290SdchaginLIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_cmp_requeue, "uint32_t *", 156246290Sdchagin "uint32_t", "uint32_t", "uint32_t *", "struct l_timespec *"); 157246290SdchaginLIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, debug_cmp_requeue_value_neq, 158246290Sdchagin "uint32_t", "int"); 159246290SdchaginLIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_wake_op, "uint32_t *", 160246290Sdchagin "int", "uint32_t", "uint32_t *", "uint32_t"); 161246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unhandled_efault); 162246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_lock_pi); 163246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_unlock_pi); 164246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_trylock_pi); 165246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, deprecated_requeue); 166246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_wait_requeue_pi); 167246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_cmp_requeue_pi); 168246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, unknown_operation, "int"); 169246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, return, "int"); 170246290SdchaginLIN_SDT_PROBE_DEFINE2(futex, linux_set_robust_list, entry, "struct thread *", 171246290Sdchagin "struct linux_set_robust_list_args *"); 172246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, linux_set_robust_list, size_error); 173246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, linux_set_robust_list, return, "int"); 174246290SdchaginLIN_SDT_PROBE_DEFINE2(futex, linux_get_robust_list, entry, "struct thread *", 175246290Sdchagin "struct linux_get_robust_list_args *"); 176246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, copyout_error, "int"); 177246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, return, "int"); 178246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, handle_futex_death, entry, "struct proc *", 179246290Sdchagin "uint32_t *", "int"); 180246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, copyin_error, "int"); 181246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, return, "int"); 182246290SdchaginLIN_SDT_PROBE_DEFINE3(futex, fetch_robust_entry, entry, 183246290Sdchagin "struct linux_robust_list **", "struct linux_robust_list **", "int *"); 184246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, copyin_error, "int"); 185246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, return, "int"); 186246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, release_futexes, entry, "struct proc *"); 187246290SdchaginLIN_SDT_PROBE_DEFINE1(futex, release_futexes, copyin_error, "int"); 188246290SdchaginLIN_SDT_PROBE_DEFINE0(futex, release_futexes, return); 189246290Sdchagin 190246290Sdchaginstatic MALLOC_DEFINE(M_FUTEX, "futex", "Linux futexes"); 191246290Sdchaginstatic MALLOC_DEFINE(M_FUTEX_WP, "futex wp", "Linux futexes wp"); 192246290Sdchagin 193161304Snetchildstruct futex; 194161304Snetchild 195161304Snetchildstruct waiting_proc { 196191719Sdchagin uint32_t wp_flags; 197191719Sdchagin struct futex *wp_futex; 198161304Snetchild TAILQ_ENTRY(waiting_proc) wp_list; 199161304Snetchild}; 200191719Sdchagin 201161304Snetchildstruct futex { 202191719Sdchagin struct sx f_lck; 203218970Sjhb uint32_t *f_uaddr; /* user-supplied value, for debug */ 204218970Sjhb struct umtx_key f_key; 205191719Sdchagin uint32_t f_refcount; 206218117Sdchagin uint32_t f_bitset; 207161304Snetchild LIST_ENTRY(futex) f_list; 208161304Snetchild TAILQ_HEAD(lf_waiting_proc, waiting_proc) f_waiting_proc; 209161304Snetchild}; 210161304Snetchild 211191741Sdchaginstruct futex_list futex_list; 212161304Snetchild 213191719Sdchagin#define FUTEX_LOCK(f) sx_xlock(&(f)->f_lck) 214191719Sdchagin#define FUTEX_UNLOCK(f) sx_xunlock(&(f)->f_lck) 215246290Sdchagin#define FUTEX_INIT(f) do { \ 216246290Sdchagin sx_init_flags(&(f)->f_lck, "ftlk", \ 217246290Sdchagin SX_DUPOK); \ 218246290Sdchagin LIN_SDT_PROBE1(futex, futex, create, \ 219246290Sdchagin &(f)->f_lck); \ 220246290Sdchagin } while (0) 221246290Sdchagin#define FUTEX_DESTROY(f) do { \ 222246290Sdchagin LIN_SDT_PROBE1(futex, futex, destroy, \ 223246290Sdchagin &(f)->f_lck); \ 224246290Sdchagin sx_destroy(&(f)->f_lck); \ 225246290Sdchagin } while (0) 226191719Sdchagin#define FUTEX_ASSERT_LOCKED(f) sx_assert(&(f)->f_lck, SA_XLOCKED) 227161304Snetchild 228191719Sdchaginstruct mtx futex_mtx; /* protects the futex list */ 229246290Sdchagin#define FUTEXES_LOCK do { \ 230246290Sdchagin mtx_lock(&futex_mtx); \ 231246290Sdchagin LIN_SDT_PROBE1(locks, futex_mtx, \ 232246290Sdchagin locked, &futex_mtx); \ 233246290Sdchagin } while (0) 234246290Sdchagin#define FUTEXES_UNLOCK do { \ 235246290Sdchagin LIN_SDT_PROBE1(locks, futex_mtx, \ 236246290Sdchagin unlock, &futex_mtx); \ 237246290Sdchagin mtx_unlock(&futex_mtx); \ 238246290Sdchagin } while (0) 239161304Snetchild 240191719Sdchagin/* flags for futex_get() */ 241191719Sdchagin#define FUTEX_CREATE_WP 0x1 /* create waiting_proc */ 242191719Sdchagin#define FUTEX_DONTCREATE 0x2 /* don't create futex if not exists */ 243191719Sdchagin#define FUTEX_DONTEXISTS 0x4 /* return EINVAL if futex exists */ 244218970Sjhb#define FUTEX_SHARED 0x8 /* shared futex */ 245161304Snetchild 246191719Sdchagin/* wp_flags */ 247191719Sdchagin#define FUTEX_WP_REQUEUED 0x1 /* wp requeued - wp moved from wp_list 248191719Sdchagin * of futex where thread sleep to wp_list 249191719Sdchagin * of another futex. 250191719Sdchagin */ 251191719Sdchagin#define FUTEX_WP_REMOVED 0x2 /* wp is woken up and removed from futex 252191719Sdchagin * wp_list to prevent double wakeup. 253191719Sdchagin */ 254161304Snetchild 255161304Snetchild/* support.s */ 256191719Sdchaginint futex_xchgl(int oparg, uint32_t *uaddr, int *oldval); 257191719Sdchaginint futex_addl(int oparg, uint32_t *uaddr, int *oldval); 258191719Sdchaginint futex_orl(int oparg, uint32_t *uaddr, int *oldval); 259191719Sdchaginint futex_andl(int oparg, uint32_t *uaddr, int *oldval); 260191719Sdchaginint futex_xorl(int oparg, uint32_t *uaddr, int *oldval); 261161304Snetchild 262191719Sdchaginstatic void 263191719Sdchaginfutex_put(struct futex *f, struct waiting_proc *wp) 264161304Snetchild{ 265246290Sdchagin LIN_SDT_PROBE2(futex, futex_put, entry, f, wp); 266161304Snetchild 267191719Sdchagin FUTEX_ASSERT_LOCKED(f); 268191719Sdchagin if (wp != NULL) { 269191719Sdchagin if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0) 270191719Sdchagin TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 271191719Sdchagin free(wp, M_FUTEX_WP); 272191719Sdchagin } 273161304Snetchild 274191719Sdchagin FUTEXES_LOCK; 275191719Sdchagin if (--f->f_refcount == 0) { 276191719Sdchagin LIST_REMOVE(f, f_list); 277191719Sdchagin FUTEXES_UNLOCK; 278191719Sdchagin FUTEX_UNLOCK(f); 279177460Srdivacky 280246290Sdchagin LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr, 281246290Sdchagin f->f_refcount, f->f_key.shared); 282219242Sdchagin LINUX_CTR3(sys_futex, "futex_put destroy uaddr %p ref %d " 283219242Sdchagin "shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared); 284218970Sjhb umtx_key_release(&f->f_key); 285191719Sdchagin FUTEX_DESTROY(f); 286191719Sdchagin free(f, M_FUTEX); 287246290Sdchagin 288246290Sdchagin LIN_SDT_PROBE0(futex, futex_put, return); 289191719Sdchagin return; 290191719Sdchagin } 291161304Snetchild 292246290Sdchagin LIN_SDT_PROBE3(futex, futex_put, unlock, f->f_uaddr, f->f_refcount, 293246290Sdchagin f->f_key.shared); 294219242Sdchagin LINUX_CTR3(sys_futex, "futex_put uaddr %p ref %d shared %d", 295219242Sdchagin f->f_uaddr, f->f_refcount, f->f_key.shared); 296191719Sdchagin FUTEXES_UNLOCK; 297191719Sdchagin FUTEX_UNLOCK(f); 298246290Sdchagin 299246290Sdchagin LIN_SDT_PROBE0(futex, futex_put, return); 300191719Sdchagin} 301161304Snetchild 302191719Sdchaginstatic int 303191719Sdchaginfutex_get0(uint32_t *uaddr, struct futex **newf, uint32_t flags) 304191719Sdchagin{ 305191719Sdchagin struct futex *f, *tmpf; 306218970Sjhb struct umtx_key key; 307218970Sjhb int error; 308161304Snetchild 309246290Sdchagin LIN_SDT_PROBE3(futex, futex_get0, entry, uaddr, newf, flags); 310246290Sdchagin 311191719Sdchagin *newf = tmpf = NULL; 312191719Sdchagin 313218970Sjhb error = umtx_key_get(uaddr, TYPE_FUTEX, (flags & FUTEX_SHARED) ? 314219240Sdchagin AUTO_SHARE : THREAD_SHARE, &key); 315246290Sdchagin if (error) { 316246290Sdchagin LIN_SDT_PROBE1(futex, futex_get0, umtx_key_get_error, error); 317246290Sdchagin LIN_SDT_PROBE1(futex, futex_get0, return, error); 318218970Sjhb return (error); 319246290Sdchagin } 320191719Sdchaginretry: 321191719Sdchagin FUTEXES_LOCK; 322191719Sdchagin LIST_FOREACH(f, &futex_list, f_list) { 323218970Sjhb if (umtx_key_match(&f->f_key, &key)) { 324191719Sdchagin if (tmpf != NULL) { 325191719Sdchagin FUTEX_UNLOCK(tmpf); 326191719Sdchagin FUTEX_DESTROY(tmpf); 327191719Sdchagin free(tmpf, M_FUTEX); 328161304Snetchild } 329191719Sdchagin if (flags & FUTEX_DONTEXISTS) { 330191719Sdchagin FUTEXES_UNLOCK; 331218970Sjhb umtx_key_release(&key); 332246290Sdchagin 333246290Sdchagin LIN_SDT_PROBE1(futex, futex_get0, return, 334246290Sdchagin EINVAL); 335191719Sdchagin return (EINVAL); 336191719Sdchagin } 337161304Snetchild 338191719Sdchagin /* 339191719Sdchagin * Increment refcount of the found futex to 340191719Sdchagin * prevent it from deallocation before FUTEX_LOCK() 341191719Sdchagin */ 342191719Sdchagin ++f->f_refcount; 343191719Sdchagin FUTEXES_UNLOCK; 344218970Sjhb umtx_key_release(&key); 345161304Snetchild 346191719Sdchagin FUTEX_LOCK(f); 347191719Sdchagin *newf = f; 348246290Sdchagin LIN_SDT_PROBE3(futex, futex_get0, shared, uaddr, 349246290Sdchagin f->f_refcount, f->f_key.shared); 350219242Sdchagin LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d", 351219242Sdchagin uaddr, f->f_refcount, f->f_key.shared); 352246290Sdchagin 353246290Sdchagin LIN_SDT_PROBE1(futex, futex_get0, return, 0); 354191719Sdchagin return (0); 355161304Snetchild } 356191719Sdchagin } 357161304Snetchild 358191719Sdchagin if (flags & FUTEX_DONTCREATE) { 359191719Sdchagin FUTEXES_UNLOCK; 360218970Sjhb umtx_key_release(&key); 361246290Sdchagin LIN_SDT_PROBE1(futex, futex_get0, null, uaddr); 362191887Sdchagin LINUX_CTR1(sys_futex, "futex_get uaddr %p null", uaddr); 363246290Sdchagin 364246290Sdchagin LIN_SDT_PROBE1(futex, futex_get0, return, 0); 365191719Sdchagin return (0); 366191719Sdchagin } 367166969Snetchild 368191719Sdchagin if (tmpf == NULL) { 369191719Sdchagin FUTEXES_UNLOCK; 370191719Sdchagin tmpf = malloc(sizeof(*tmpf), M_FUTEX, M_WAITOK | M_ZERO); 371191719Sdchagin tmpf->f_uaddr = uaddr; 372218970Sjhb tmpf->f_key = key; 373191719Sdchagin tmpf->f_refcount = 1; 374218646Sdchagin tmpf->f_bitset = FUTEX_BITSET_MATCH_ANY; 375191719Sdchagin FUTEX_INIT(tmpf); 376191719Sdchagin TAILQ_INIT(&tmpf->f_waiting_proc); 377161304Snetchild 378166969Snetchild /* 379191719Sdchagin * Lock the new futex before an insert into the futex_list 380191719Sdchagin * to prevent futex usage by other. 381161304Snetchild */ 382191719Sdchagin FUTEX_LOCK(tmpf); 383191719Sdchagin goto retry; 384191719Sdchagin } 385161304Snetchild 386191719Sdchagin LIST_INSERT_HEAD(&futex_list, tmpf, f_list); 387191719Sdchagin FUTEXES_UNLOCK; 388161304Snetchild 389246290Sdchagin LIN_SDT_PROBE3(futex, futex_get0, new, uaddr, tmpf->f_refcount, 390246290Sdchagin tmpf->f_key.shared); 391219242Sdchagin LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d new", 392219242Sdchagin uaddr, tmpf->f_refcount, tmpf->f_key.shared); 393191719Sdchagin *newf = tmpf; 394246290Sdchagin 395246290Sdchagin LIN_SDT_PROBE1(futex, futex_get0, return, 0); 396191719Sdchagin return (0); 397191719Sdchagin} 398161304Snetchild 399191719Sdchaginstatic int 400191719Sdchaginfutex_get(uint32_t *uaddr, struct waiting_proc **wp, struct futex **f, 401191719Sdchagin uint32_t flags) 402191719Sdchagin{ 403191719Sdchagin int error; 404161304Snetchild 405246290Sdchagin LIN_SDT_PROBE3(futex, futex_get, entry, uaddr, wp, f); 406246290Sdchagin 407191719Sdchagin if (flags & FUTEX_CREATE_WP) { 408191719Sdchagin *wp = malloc(sizeof(struct waiting_proc), M_FUTEX_WP, M_WAITOK); 409191719Sdchagin (*wp)->wp_flags = 0; 410191719Sdchagin } 411191719Sdchagin error = futex_get0(uaddr, f, flags); 412191719Sdchagin if (error) { 413246290Sdchagin LIN_SDT_PROBE0(futex, futex_get, error); 414246290Sdchagin 415191719Sdchagin if (flags & FUTEX_CREATE_WP) 416191719Sdchagin free(*wp, M_FUTEX_WP); 417246290Sdchagin 418246290Sdchagin LIN_SDT_PROBE1(futex, futex_get, return, error); 419191719Sdchagin return (error); 420191719Sdchagin } 421191719Sdchagin if (flags & FUTEX_CREATE_WP) { 422191719Sdchagin TAILQ_INSERT_HEAD(&(*f)->f_waiting_proc, *wp, wp_list); 423191719Sdchagin (*wp)->wp_futex = *f; 424191719Sdchagin } 425161304Snetchild 426246290Sdchagin LIN_SDT_PROBE1(futex, futex_get, return, error); 427191719Sdchagin return (error); 428161304Snetchild} 429161304Snetchild 430191719Sdchaginstatic int 431213490Sjkimfutex_sleep(struct futex *f, struct waiting_proc *wp, int timeout) 432161304Snetchild{ 433191719Sdchagin int error; 434161304Snetchild 435191719Sdchagin FUTEX_ASSERT_LOCKED(f); 436246290Sdchagin LIN_SDT_PROBE3(futex, futex_sleep, entry, f, wp, timeout); 437213490Sjkim LINUX_CTR4(sys_futex, "futex_sleep enter uaddr %p wp %p timo %d ref %d", 438191887Sdchagin f->f_uaddr, wp, timeout, f->f_refcount); 439191719Sdchagin error = sx_sleep(wp, &f->f_lck, PCATCH, "futex", timeout); 440191719Sdchagin if (wp->wp_flags & FUTEX_WP_REQUEUED) { 441191719Sdchagin KASSERT(f != wp->wp_futex, ("futex != wp_futex")); 442246290Sdchagin 443246290Sdchagin if (error) { 444246290Sdchagin LIN_SDT_PROBE5(futex, futex_sleep, requeue_error, error, 445246290Sdchagin f->f_uaddr, wp, wp->wp_futex->f_uaddr, 446246290Sdchagin wp->wp_futex->f_refcount); 447246290Sdchagin } 448246290Sdchagin 449246290Sdchagin LINUX_CTR5(sys_futex, "futex_sleep out error %d uaddr %p wp" 450191887Sdchagin " %p requeued uaddr %p ref %d", 451191887Sdchagin error, f->f_uaddr, wp, wp->wp_futex->f_uaddr, 452191887Sdchagin wp->wp_futex->f_refcount); 453191719Sdchagin futex_put(f, NULL); 454191719Sdchagin f = wp->wp_futex; 455191719Sdchagin FUTEX_LOCK(f); 456246290Sdchagin } else { 457246290Sdchagin if (error) { 458246290Sdchagin LIN_SDT_PROBE3(futex, futex_sleep, sleep_error, error, 459246290Sdchagin f->f_uaddr, wp); 460246290Sdchagin } 461191887Sdchagin LINUX_CTR3(sys_futex, "futex_sleep out error %d uaddr %p wp %p", 462191887Sdchagin error, f->f_uaddr, wp); 463246290Sdchagin } 464161304Snetchild 465191719Sdchagin futex_put(f, wp); 466246290Sdchagin 467246290Sdchagin LIN_SDT_PROBE1(futex, futex_sleep, return, error); 468191719Sdchagin return (error); 469161304Snetchild} 470161304Snetchild 471191719Sdchaginstatic int 472218117Sdchaginfutex_wake(struct futex *f, int n, uint32_t bitset) 473161304Snetchild{ 474191719Sdchagin struct waiting_proc *wp, *wpt; 475191719Sdchagin int count = 0; 476191719Sdchagin 477246290Sdchagin LIN_SDT_PROBE3(futex, futex_wake, entry, f, n, bitset); 478246290Sdchagin 479246290Sdchagin if (bitset == 0) { 480246290Sdchagin LIN_SDT_PROBE1(futex, futex_wake, return, EINVAL); 481218117Sdchagin return (EINVAL); 482246290Sdchagin } 483218117Sdchagin 484191719Sdchagin FUTEX_ASSERT_LOCKED(f); 485191719Sdchagin TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) { 486246290Sdchagin LIN_SDT_PROBE3(futex, futex_wake, iterate, f->f_uaddr, wp, 487246290Sdchagin f->f_refcount); 488191887Sdchagin LINUX_CTR3(sys_futex, "futex_wake uaddr %p wp %p ref %d", 489191887Sdchagin f->f_uaddr, wp, f->f_refcount); 490218117Sdchagin /* 491218117Sdchagin * Unless we find a matching bit in 492218117Sdchagin * the bitset, continue searching. 493218117Sdchagin */ 494218117Sdchagin if (!(wp->wp_futex->f_bitset & bitset)) 495218117Sdchagin continue; 496218117Sdchagin 497191719Sdchagin wp->wp_flags |= FUTEX_WP_REMOVED; 498191719Sdchagin TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 499246290Sdchagin LIN_SDT_PROBE1(futex, futex_wake, wakeup, wp); 500191719Sdchagin wakeup_one(wp); 501191719Sdchagin if (++count == n) 502191719Sdchagin break; 503161304Snetchild } 504161304Snetchild 505246290Sdchagin LIN_SDT_PROBE1(futex, futex_wake, return, count); 506191719Sdchagin return (count); 507161304Snetchild} 508161304Snetchild 509166969Snetchildstatic int 510191719Sdchaginfutex_requeue(struct futex *f, int n, struct futex *f2, int n2) 511161304Snetchild{ 512191719Sdchagin struct waiting_proc *wp, *wpt; 513191719Sdchagin int count = 0; 514161304Snetchild 515246290Sdchagin LIN_SDT_PROBE4(futex, futex_requeue, entry, f, n, f2, n2); 516246290Sdchagin 517191719Sdchagin FUTEX_ASSERT_LOCKED(f); 518191719Sdchagin FUTEX_ASSERT_LOCKED(f2); 519161304Snetchild 520191719Sdchagin TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) { 521191719Sdchagin if (++count <= n) { 522191887Sdchagin LINUX_CTR2(sys_futex, "futex_req_wake uaddr %p wp %p", 523191887Sdchagin f->f_uaddr, wp); 524191719Sdchagin wp->wp_flags |= FUTEX_WP_REMOVED; 525191719Sdchagin TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 526246290Sdchagin LIN_SDT_PROBE1(futex, futex_requeue, wakeup, wp); 527191719Sdchagin wakeup_one(wp); 528191719Sdchagin } else { 529246290Sdchagin LIN_SDT_PROBE3(futex, futex_requeue, requeue, 530246290Sdchagin f->f_uaddr, wp, f2->f_uaddr); 531191887Sdchagin LINUX_CTR3(sys_futex, "futex_requeue uaddr %p wp %p to %p", 532191887Sdchagin f->f_uaddr, wp, f2->f_uaddr); 533191719Sdchagin wp->wp_flags |= FUTEX_WP_REQUEUED; 534191719Sdchagin /* Move wp to wp_list of f2 futex */ 535191719Sdchagin TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 536191719Sdchagin TAILQ_INSERT_HEAD(&f2->f_waiting_proc, wp, wp_list); 537161304Snetchild 538191719Sdchagin /* 539191719Sdchagin * Thread which sleeps on wp after waking should 540191719Sdchagin * acquire f2 lock, so increment refcount of f2 to 541191719Sdchagin * prevent it from premature deallocation. 542191719Sdchagin */ 543191719Sdchagin wp->wp_futex = f2; 544191719Sdchagin FUTEXES_LOCK; 545191719Sdchagin ++f2->f_refcount; 546191719Sdchagin FUTEXES_UNLOCK; 547191719Sdchagin if (count - n >= n2) 548191719Sdchagin break; 549191719Sdchagin } 550161304Snetchild } 551161304Snetchild 552246290Sdchagin LIN_SDT_PROBE1(futex, futex_requeue, return, count); 553191719Sdchagin return (count); 554161304Snetchild} 555161304Snetchild 556161304Snetchildstatic int 557267006Sdchaginfutex_wait(struct futex *f, struct waiting_proc *wp, int timeout_hz, 558218117Sdchagin uint32_t bitset) 559161304Snetchild{ 560191719Sdchagin int error; 561161304Snetchild 562267006Sdchagin LIN_SDT_PROBE4(futex, futex_wait, entry, f, wp, timeout_hz, bitset); 563246290Sdchagin 564246290Sdchagin if (bitset == 0) { 565246290Sdchagin LIN_SDT_PROBE1(futex, futex_wait, return, EINVAL); 566218117Sdchagin return (EINVAL); 567246290Sdchagin } 568246290Sdchagin 569218117Sdchagin f->f_bitset = bitset; 570191719Sdchagin error = futex_sleep(f, wp, timeout_hz); 571267006Sdchagin if (error) 572246290Sdchagin LIN_SDT_PROBE1(futex, futex_wait, sleep_error, error); 573191719Sdchagin if (error == EWOULDBLOCK) 574191719Sdchagin error = ETIMEDOUT; 575161304Snetchild 576246290Sdchagin LIN_SDT_PROBE1(futex, futex_wait, return, error); 577191719Sdchagin return (error); 578161304Snetchild} 579161304Snetchild 580161304Snetchildstatic int 581191719Sdchaginfutex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr) 582161304Snetchild{ 583168037Sjkim int op = (encoded_op >> 28) & 7; 584168037Sjkim int cmp = (encoded_op >> 24) & 15; 585168037Sjkim int oparg = (encoded_op << 8) >> 20; 586168037Sjkim int cmparg = (encoded_op << 20) >> 20; 587168037Sjkim int oldval = 0, ret; 588161304Snetchild 589246290Sdchagin LIN_SDT_PROBE3(futex, futex_atomic_op, entry, td, encoded_op, uaddr); 590246290Sdchagin 591161304Snetchild if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) 592168037Sjkim oparg = 1 << oparg; 593161304Snetchild 594246290Sdchagin LIN_SDT_PROBE4(futex, futex_atomic_op, decoded_op, op, cmp, oparg, 595246290Sdchagin cmparg); 596246290Sdchagin 597219460Sjhb /* XXX: Linux verifies access here and returns EFAULT */ 598246290Sdchagin LIN_SDT_PROBE0(futex, futex_atomic_op, missing_access_check); 599161304Snetchild 600168037Sjkim switch (op) { 601168037Sjkim case FUTEX_OP_SET: 602168037Sjkim ret = futex_xchgl(oparg, uaddr, &oldval); 603168037Sjkim break; 604168037Sjkim case FUTEX_OP_ADD: 605168037Sjkim ret = futex_addl(oparg, uaddr, &oldval); 606168037Sjkim break; 607168037Sjkim case FUTEX_OP_OR: 608168037Sjkim ret = futex_orl(oparg, uaddr, &oldval); 609168037Sjkim break; 610168037Sjkim case FUTEX_OP_ANDN: 611168037Sjkim ret = futex_andl(~oparg, uaddr, &oldval); 612168037Sjkim break; 613168037Sjkim case FUTEX_OP_XOR: 614168037Sjkim ret = futex_xorl(oparg, uaddr, &oldval); 615168037Sjkim break; 616168037Sjkim default: 617246290Sdchagin LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_op, op); 618168037Sjkim ret = -ENOSYS; 619169895Skib break; 620168037Sjkim } 621161304Snetchild 622246290Sdchagin if (ret) { 623246290Sdchagin LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret); 624169895Skib return (ret); 625246290Sdchagin } 626168037Sjkim 627169895Skib switch (cmp) { 628169895Skib case FUTEX_OP_CMP_EQ: 629246290Sdchagin ret = (oldval == cmparg); 630246290Sdchagin break; 631169895Skib case FUTEX_OP_CMP_NE: 632246290Sdchagin ret = (oldval != cmparg); 633246290Sdchagin break; 634169895Skib case FUTEX_OP_CMP_LT: 635246290Sdchagin ret = (oldval < cmparg); 636246290Sdchagin break; 637169895Skib case FUTEX_OP_CMP_GE: 638246290Sdchagin ret = (oldval >= cmparg); 639246290Sdchagin break; 640169895Skib case FUTEX_OP_CMP_LE: 641246290Sdchagin ret = (oldval <= cmparg); 642246290Sdchagin break; 643169895Skib case FUTEX_OP_CMP_GT: 644246290Sdchagin ret = (oldval > cmparg); 645246290Sdchagin break; 646169895Skib default: 647246290Sdchagin LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_cmp, cmp); 648246290Sdchagin ret = -ENOSYS; 649161304Snetchild } 650246290Sdchagin 651246290Sdchagin LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret); 652246290Sdchagin return (ret); 653168037Sjkim} 654178976Srdivacky 655178976Srdivackyint 656191719Sdchaginlinux_sys_futex(struct thread *td, struct linux_sys_futex_args *args) 657191719Sdchagin{ 658266980Sdchagin int clockrt, nrwake, op_ret, ret; 659191719Sdchagin struct linux_emuldata *em; 660191719Sdchagin struct waiting_proc *wp; 661218686Sdchagin struct futex *f, *f2; 662267006Sdchagin struct l_timespec timeout; 663267006Sdchagin struct timeval utv, ctv; 664267006Sdchagin int timeout_hz; 665218686Sdchagin int error; 666266980Sdchagin uint32_t flags, val; 667191719Sdchagin 668246290Sdchagin LIN_SDT_PROBE2(futex, linux_sys_futex, entry, td, args); 669246290Sdchagin 670218970Sjhb if (args->op & LINUX_FUTEX_PRIVATE_FLAG) { 671218970Sjhb flags = 0; 672218970Sjhb args->op &= ~LINUX_FUTEX_PRIVATE_FLAG; 673218970Sjhb } else 674218970Sjhb flags = FUTEX_SHARED; 675191719Sdchagin 676215338Snetchild /* 677215338Snetchild * Currently support for switching between CLOCK_MONOTONIC and 678215338Snetchild * CLOCK_REALTIME is not present. However Linux forbids the use of 679215338Snetchild * FUTEX_CLOCK_REALTIME with any op except FUTEX_WAIT_BITSET and 680215338Snetchild * FUTEX_WAIT_REQUEUE_PI. 681215338Snetchild */ 682215338Snetchild clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME; 683215338Snetchild args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME; 684215338Snetchild if (clockrt && args->op != LINUX_FUTEX_WAIT_BITSET && 685246290Sdchagin args->op != LINUX_FUTEX_WAIT_REQUEUE_PI) { 686246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, 687246290Sdchagin unimplemented_clockswitch); 688246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 689215338Snetchild return (ENOSYS); 690246290Sdchagin } 691215338Snetchild 692218686Sdchagin error = 0; 693218686Sdchagin f = f2 = NULL; 694218686Sdchagin 695191719Sdchagin switch (args->op) { 696191719Sdchagin case LINUX_FUTEX_WAIT: 697218117Sdchagin args->val3 = FUTEX_BITSET_MATCH_ANY; 698218117Sdchagin /* FALLTHROUGH */ 699191719Sdchagin 700218117Sdchagin case LINUX_FUTEX_WAIT_BITSET: 701246290Sdchagin LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wait, args->uaddr, 702246290Sdchagin args->val, args->val3); 703266980Sdchagin LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x", 704218117Sdchagin args->uaddr, args->val, args->val3); 705246290Sdchagin 706218970Sjhb error = futex_get(args->uaddr, &wp, &f, 707218970Sjhb flags | FUTEX_CREATE_WP); 708246290Sdchagin if (error) { 709246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 710191719Sdchagin return (error); 711246290Sdchagin } 712246290Sdchagin 713191719Sdchagin error = copyin(args->uaddr, &val, sizeof(val)); 714191719Sdchagin if (error) { 715246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, 716246290Sdchagin error); 717191887Sdchagin LINUX_CTR1(sys_futex, "WAIT copyin failed %d", 718191887Sdchagin error); 719191719Sdchagin futex_put(f, wp); 720246290Sdchagin 721246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 722191719Sdchagin return (error); 723191719Sdchagin } 724191719Sdchagin if (val != args->val) { 725246290Sdchagin LIN_SDT_PROBE4(futex, linux_sys_futex, 726246290Sdchagin debug_wait_value_neq, args->uaddr, args->val, val, 727246290Sdchagin args->val3); 728266980Sdchagin LINUX_CTR3(sys_futex, 729266980Sdchagin "WAIT uaddr %p val 0x%x != uval 0x%x", 730266980Sdchagin args->uaddr, args->val, val); 731191719Sdchagin futex_put(f, wp); 732246290Sdchagin 733246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, 734246290Sdchagin EWOULDBLOCK); 735191719Sdchagin return (EWOULDBLOCK); 736191719Sdchagin } 737191719Sdchagin 738267006Sdchagin if (args->timeout != NULL) { 739267006Sdchagin error = copyin(args->timeout, &timeout, sizeof(timeout)); 740267006Sdchagin if (error) { 741267006Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, 742267006Sdchagin error); 743267006Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 744267006Sdchagin futex_put(f, wp); 745267006Sdchagin return (error); 746267006Sdchagin } 747267006Sdchagin TIMESPEC_TO_TIMEVAL(&utv, &timeout); 748267006Sdchagin error = itimerfix(&utv); 749267006Sdchagin if (error) { 750267006Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, itimerfix_error, 751267006Sdchagin error); 752267006Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 753267006Sdchagin futex_put(f, wp); 754267006Sdchagin return (error); 755267006Sdchagin } 756267006Sdchagin if (clockrt) { 757267006Sdchagin microtime(&ctv); 758267006Sdchagin timevalsub(&utv, &ctv); 759267006Sdchagin } else if (args->op == LINUX_FUTEX_WAIT_BITSET) { 760267006Sdchagin microuptime(&ctv); 761267006Sdchagin timevalsub(&utv, &ctv); 762267006Sdchagin } 763267006Sdchagin if (utv.tv_sec < 0) 764267006Sdchagin timevalclear(&utv); 765267006Sdchagin timeout_hz = tvtohz(&utv); 766267006Sdchagin } else 767267006Sdchagin timeout_hz = 0; 768267006Sdchagin 769267006Sdchagin error = futex_wait(f, wp, timeout_hz, args->val3); 770191719Sdchagin break; 771191719Sdchagin 772191719Sdchagin case LINUX_FUTEX_WAKE: 773218117Sdchagin args->val3 = FUTEX_BITSET_MATCH_ANY; 774218117Sdchagin /* FALLTHROUGH */ 775191719Sdchagin 776218117Sdchagin case LINUX_FUTEX_WAKE_BITSET: 777246290Sdchagin LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wake, args->uaddr, 778246290Sdchagin args->val, args->val3); 779266980Sdchagin LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x", 780218117Sdchagin args->uaddr, args->val, args->val3); 781218117Sdchagin 782218970Sjhb error = futex_get(args->uaddr, NULL, &f, 783218970Sjhb flags | FUTEX_DONTCREATE); 784246290Sdchagin if (error) { 785246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 786191719Sdchagin return (error); 787246290Sdchagin } 788246290Sdchagin 789191719Sdchagin if (f == NULL) { 790191719Sdchagin td->td_retval[0] = 0; 791246290Sdchagin 792246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 793201758Smbr return (error); 794191719Sdchagin } 795218117Sdchagin td->td_retval[0] = futex_wake(f, args->val, args->val3); 796191719Sdchagin futex_put(f, NULL); 797191719Sdchagin break; 798191719Sdchagin 799191719Sdchagin case LINUX_FUTEX_CMP_REQUEUE: 800246290Sdchagin LIN_SDT_PROBE5(futex, linux_sys_futex, debug_cmp_requeue, 801246290Sdchagin args->uaddr, args->val, args->val3, args->uaddr2, 802246290Sdchagin args->timeout); 803191887Sdchagin LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p " 804266980Sdchagin "nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x", 805191887Sdchagin args->uaddr, args->val, args->val3, args->uaddr2, 806266980Sdchagin args->timeout); 807191887Sdchagin 808191719Sdchagin /* 809191719Sdchagin * Linux allows this, we would not, it is an incorrect 810191719Sdchagin * usage of declared ABI, so return EINVAL. 811191719Sdchagin */ 812246290Sdchagin if (args->uaddr == args->uaddr2) { 813246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, 814246290Sdchagin invalid_cmp_requeue_use); 815246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL); 816191719Sdchagin return (EINVAL); 817246290Sdchagin } 818246290Sdchagin 819218970Sjhb error = futex_get(args->uaddr, NULL, &f, flags); 820246290Sdchagin if (error) { 821246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 822191719Sdchagin return (error); 823246290Sdchagin } 824191719Sdchagin 825191719Sdchagin /* 826191719Sdchagin * To avoid deadlocks return EINVAL if second futex 827218655Sdchagin * exists at this time. 828191719Sdchagin * 829191719Sdchagin * Glibc fall back to FUTEX_WAKE in case of any error 830191719Sdchagin * returned by FUTEX_CMP_REQUEUE. 831191719Sdchagin */ 832218970Sjhb error = futex_get(args->uaddr2, NULL, &f2, 833218970Sjhb flags | FUTEX_DONTEXISTS); 834191719Sdchagin if (error) { 835191719Sdchagin futex_put(f, NULL); 836246290Sdchagin 837246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 838191719Sdchagin return (error); 839191719Sdchagin } 840191719Sdchagin error = copyin(args->uaddr, &val, sizeof(val)); 841191719Sdchagin if (error) { 842246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, 843246290Sdchagin error); 844191887Sdchagin LINUX_CTR1(sys_futex, "CMP_REQUEUE copyin failed %d", 845191887Sdchagin error); 846191719Sdchagin futex_put(f2, NULL); 847191719Sdchagin futex_put(f, NULL); 848246290Sdchagin 849246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 850191719Sdchagin return (error); 851191719Sdchagin } 852191719Sdchagin if (val != args->val3) { 853246290Sdchagin LIN_SDT_PROBE2(futex, linux_sys_futex, 854246290Sdchagin debug_cmp_requeue_value_neq, args->val, val); 855266980Sdchagin LINUX_CTR2(sys_futex, "CMP_REQUEUE val 0x%x != uval 0x%x", 856191887Sdchagin args->val, val); 857191719Sdchagin futex_put(f2, NULL); 858191719Sdchagin futex_put(f, NULL); 859246290Sdchagin 860246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, EAGAIN); 861191719Sdchagin return (EAGAIN); 862191719Sdchagin } 863191719Sdchagin 864191719Sdchagin nrwake = (int)(unsigned long)args->timeout; 865191719Sdchagin td->td_retval[0] = futex_requeue(f, args->val, f2, nrwake); 866191719Sdchagin futex_put(f2, NULL); 867191719Sdchagin futex_put(f, NULL); 868191719Sdchagin break; 869191719Sdchagin 870191719Sdchagin case LINUX_FUTEX_WAKE_OP: 871246290Sdchagin LIN_SDT_PROBE5(futex, linux_sys_futex, debug_wake_op, 872246290Sdchagin args->uaddr, args->op, args->val, args->uaddr2, args->val3); 873191887Sdchagin LINUX_CTR5(sys_futex, "WAKE_OP " 874266980Sdchagin "uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x", 875266980Sdchagin args->uaddr, args->val, args->uaddr2, args->val3, 876266980Sdchagin args->timeout); 877191887Sdchagin 878218970Sjhb error = futex_get(args->uaddr, NULL, &f, flags); 879246290Sdchagin if (error) { 880246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 881191719Sdchagin return (error); 882246290Sdchagin } 883246290Sdchagin 884191719Sdchagin if (args->uaddr != args->uaddr2) 885218970Sjhb error = futex_get(args->uaddr2, NULL, &f2, flags); 886191719Sdchagin if (error) { 887191719Sdchagin futex_put(f, NULL); 888246290Sdchagin 889246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 890191719Sdchagin return (error); 891191719Sdchagin } 892191719Sdchagin 893191719Sdchagin /* 894191719Sdchagin * This function returns positive number as results and 895191719Sdchagin * negative as errors 896191719Sdchagin */ 897191719Sdchagin op_ret = futex_atomic_op(td, args->val3, args->uaddr2); 898191719Sdchagin 899266980Sdchagin LINUX_CTR2(sys_futex, "WAKE_OP atomic_op uaddr %p ret 0x%x", 900266980Sdchagin args->uaddr, op_ret); 901266980Sdchagin 902191719Sdchagin if (op_ret < 0) { 903191719Sdchagin /* XXX: We don't handle the EFAULT yet. */ 904191719Sdchagin if (op_ret != -EFAULT) { 905191719Sdchagin if (f2 != NULL) 906191719Sdchagin futex_put(f2, NULL); 907191719Sdchagin futex_put(f, NULL); 908246290Sdchagin 909246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, 910246290Sdchagin -op_ret); 911191719Sdchagin return (-op_ret); 912246290Sdchagin } else { 913246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, 914246290Sdchagin unhandled_efault); 915191719Sdchagin } 916191719Sdchagin if (f2 != NULL) 917191719Sdchagin futex_put(f2, NULL); 918191719Sdchagin futex_put(f, NULL); 919246290Sdchagin 920246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, EFAULT); 921191719Sdchagin return (EFAULT); 922191719Sdchagin } 923191719Sdchagin 924218117Sdchagin ret = futex_wake(f, args->val, args->val3); 925191719Sdchagin 926191719Sdchagin if (op_ret > 0) { 927191719Sdchagin op_ret = 0; 928191719Sdchagin nrwake = (int)(unsigned long)args->timeout; 929191719Sdchagin 930191719Sdchagin if (f2 != NULL) 931218117Sdchagin op_ret += futex_wake(f2, nrwake, args->val3); 932191719Sdchagin else 933218117Sdchagin op_ret += futex_wake(f, nrwake, args->val3); 934191719Sdchagin ret += op_ret; 935191719Sdchagin 936191719Sdchagin } 937191719Sdchagin if (f2 != NULL) 938191719Sdchagin futex_put(f2, NULL); 939191719Sdchagin futex_put(f, NULL); 940191719Sdchagin td->td_retval[0] = ret; 941191719Sdchagin break; 942191719Sdchagin 943191719Sdchagin case LINUX_FUTEX_LOCK_PI: 944191719Sdchagin /* not yet implemented */ 945215338Snetchild linux_msg(td, 946215338Snetchild "linux_sys_futex: " 947215666Snetchild "op LINUX_FUTEX_LOCK_PI not implemented\n"); 948246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, unimplemented_lock_pi); 949246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 950191719Sdchagin return (ENOSYS); 951191719Sdchagin 952191719Sdchagin case LINUX_FUTEX_UNLOCK_PI: 953191719Sdchagin /* not yet implemented */ 954215338Snetchild linux_msg(td, 955215338Snetchild "linux_sys_futex: " 956215666Snetchild "op LINUX_FUTEX_UNLOCK_PI not implemented\n"); 957246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, unimplemented_unlock_pi); 958246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 959191719Sdchagin return (ENOSYS); 960191719Sdchagin 961191719Sdchagin case LINUX_FUTEX_TRYLOCK_PI: 962191719Sdchagin /* not yet implemented */ 963215338Snetchild linux_msg(td, 964215338Snetchild "linux_sys_futex: " 965215666Snetchild "op LINUX_FUTEX_TRYLOCK_PI not implemented\n"); 966246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, 967246290Sdchagin unimplemented_trylock_pi); 968246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 969191719Sdchagin return (ENOSYS); 970191719Sdchagin 971191719Sdchagin case LINUX_FUTEX_REQUEUE: 972191719Sdchagin 973191719Sdchagin /* 974191719Sdchagin * Glibc does not use this operation since version 2.3.3, 975191719Sdchagin * as it is racy and replaced by FUTEX_CMP_REQUEUE operation. 976191719Sdchagin * Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when 977191719Sdchagin * FUTEX_REQUEUE returned EINVAL. 978191719Sdchagin */ 979191719Sdchagin em = em_find(td->td_proc, EMUL_DONTLOCK); 980218621Sdchagin if ((em->flags & LINUX_XDEPR_REQUEUEOP) == 0) { 981215338Snetchild linux_msg(td, 982215338Snetchild "linux_sys_futex: " 983215338Snetchild "unsupported futex_requeue op\n"); 984218621Sdchagin em->flags |= LINUX_XDEPR_REQUEUEOP; 985246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, 986246290Sdchagin deprecated_requeue); 987191719Sdchagin } 988246290Sdchagin 989246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL); 990191719Sdchagin return (EINVAL); 991191719Sdchagin 992215338Snetchild case LINUX_FUTEX_WAIT_REQUEUE_PI: 993215338Snetchild /* not yet implemented */ 994215338Snetchild linux_msg(td, 995215338Snetchild "linux_sys_futex: " 996215666Snetchild "op FUTEX_WAIT_REQUEUE_PI not implemented\n"); 997246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, 998246290Sdchagin unimplemented_wait_requeue_pi); 999246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 1000215338Snetchild return (ENOSYS); 1001215338Snetchild 1002218118Sdchagin case LINUX_FUTEX_CMP_REQUEUE_PI: 1003218118Sdchagin /* not yet implemented */ 1004218118Sdchagin linux_msg(td, 1005218118Sdchagin "linux_sys_futex: " 1006218118Sdchagin "op LINUX_FUTEX_CMP_REQUEUE_PI not implemented\n"); 1007246290Sdchagin LIN_SDT_PROBE0(futex, linux_sys_futex, 1008246290Sdchagin unimplemented_cmp_requeue_pi); 1009246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 1010218118Sdchagin return (ENOSYS); 1011218118Sdchagin 1012191719Sdchagin default: 1013215338Snetchild linux_msg(td, 1014215338Snetchild "linux_sys_futex: unknown op %d\n", args->op); 1015246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, unknown_operation, 1016246290Sdchagin args->op); 1017246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 1018191719Sdchagin return (ENOSYS); 1019191719Sdchagin } 1020191719Sdchagin 1021246290Sdchagin LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 1022191719Sdchagin return (error); 1023191719Sdchagin} 1024191719Sdchagin 1025191719Sdchaginint 1026178976Srdivackylinux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args) 1027178976Srdivacky{ 1028178976Srdivacky struct linux_emuldata *em; 1029178976Srdivacky 1030246290Sdchagin LIN_SDT_PROBE2(futex, linux_set_robust_list, entry, td, args); 1031191719Sdchagin 1032246290Sdchagin if (args->len != sizeof(struct linux_robust_list_head)) { 1033246290Sdchagin LIN_SDT_PROBE0(futex, linux_set_robust_list, size_error); 1034246290Sdchagin LIN_SDT_PROBE1(futex, linux_set_robust_list, return, EINVAL); 1035178976Srdivacky return (EINVAL); 1036246290Sdchagin } 1037178976Srdivacky 1038178976Srdivacky em = em_find(td->td_proc, EMUL_DOLOCK); 1039178976Srdivacky em->robust_futexes = args->head; 1040178976Srdivacky EMUL_UNLOCK(&emul_lock); 1041178976Srdivacky 1042246290Sdchagin LIN_SDT_PROBE1(futex, linux_set_robust_list, return, 0); 1043215339Snetchild return (0); 1044178976Srdivacky} 1045178976Srdivacky 1046178976Srdivackyint 1047178976Srdivackylinux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args) 1048178976Srdivacky{ 1049178976Srdivacky struct linux_emuldata *em; 1050178976Srdivacky struct linux_robust_list_head *head; 1051178976Srdivacky l_size_t len = sizeof(struct linux_robust_list_head); 1052178976Srdivacky int error = 0; 1053178976Srdivacky 1054246290Sdchagin LIN_SDT_PROBE2(futex, linux_get_robust_list, entry, td, args); 1055178976Srdivacky 1056178976Srdivacky if (!args->pid) { 1057178976Srdivacky em = em_find(td->td_proc, EMUL_DONTLOCK); 1058215339Snetchild head = em->robust_futexes; 1059178976Srdivacky } else { 1060178976Srdivacky struct proc *p; 1061178976Srdivacky 1062178976Srdivacky p = pfind(args->pid); 1063246290Sdchagin if (p == NULL) { 1064246290Sdchagin LIN_SDT_PROBE1(futex, linux_get_robust_list, return, 1065246290Sdchagin ESRCH); 1066178976Srdivacky return (ESRCH); 1067246290Sdchagin } 1068178976Srdivacky 1069178976Srdivacky em = em_find(p, EMUL_DONTLOCK); 1070178976Srdivacky /* XXX: ptrace? */ 1071215339Snetchild if (priv_check(td, PRIV_CRED_SETUID) || 1072178976Srdivacky priv_check(td, PRIV_CRED_SETEUID) || 1073194203Sdchagin p_candebug(td, p)) { 1074194203Sdchagin PROC_UNLOCK(p); 1075246290Sdchagin 1076246290Sdchagin LIN_SDT_PROBE1(futex, linux_get_robust_list, return, 1077246290Sdchagin EPERM); 1078178976Srdivacky return (EPERM); 1079194203Sdchagin } 1080178976Srdivacky head = em->robust_futexes; 1081215339Snetchild 1082178976Srdivacky PROC_UNLOCK(p); 1083178976Srdivacky } 1084178976Srdivacky 1085178976Srdivacky error = copyout(&len, args->len, sizeof(l_size_t)); 1086246290Sdchagin if (error) { 1087246290Sdchagin LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error, 1088246290Sdchagin error); 1089246290Sdchagin LIN_SDT_PROBE1(futex, linux_get_robust_list, return, EFAULT); 1090178976Srdivacky return (EFAULT); 1091246290Sdchagin } 1092178976Srdivacky 1093293896Sglebius error = copyout(&head, args->head, sizeof(head)); 1094246290Sdchagin if (error) { 1095246290Sdchagin LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error, 1096246290Sdchagin error); 1097246290Sdchagin } 1098178976Srdivacky 1099246290Sdchagin LIN_SDT_PROBE1(futex, linux_get_robust_list, return, error); 1100178976Srdivacky return (error); 1101178976Srdivacky} 1102178976Srdivacky 1103178976Srdivackystatic int 1104191719Sdchaginhandle_futex_death(struct proc *p, uint32_t *uaddr, int pi) 1105178976Srdivacky{ 1106191719Sdchagin uint32_t uval, nval, mval; 1107178976Srdivacky struct futex *f; 1108191719Sdchagin int error; 1109178976Srdivacky 1110246290Sdchagin LIN_SDT_PROBE3(futex, handle_futex_death, entry, p, uaddr, pi); 1111246290Sdchagin 1112178976Srdivackyretry: 1113246290Sdchagin error = copyin(uaddr, &uval, 4); 1114246290Sdchagin if (error) { 1115246290Sdchagin LIN_SDT_PROBE1(futex, handle_futex_death, copyin_error, error); 1116246290Sdchagin LIN_SDT_PROBE1(futex, handle_futex_death, return, EFAULT); 1117178976Srdivacky return (EFAULT); 1118246290Sdchagin } 1119191719Sdchagin if ((uval & FUTEX_TID_MASK) == p->p_pid) { 1120178976Srdivacky mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED; 1121178976Srdivacky nval = casuword32(uaddr, uval, mval); 1122178976Srdivacky 1123246290Sdchagin if (nval == -1) { 1124246290Sdchagin LIN_SDT_PROBE1(futex, handle_futex_death, return, 1125246290Sdchagin EFAULT); 1126178976Srdivacky return (EFAULT); 1127246290Sdchagin } 1128178976Srdivacky 1129178976Srdivacky if (nval != uval) 1130178976Srdivacky goto retry; 1131178976Srdivacky 1132178976Srdivacky if (!pi && (uval & FUTEX_WAITERS)) { 1133191719Sdchagin error = futex_get(uaddr, NULL, &f, 1134218970Sjhb FUTEX_DONTCREATE | FUTEX_SHARED); 1135246290Sdchagin if (error) { 1136246290Sdchagin LIN_SDT_PROBE1(futex, handle_futex_death, 1137246290Sdchagin return, error); 1138191719Sdchagin return (error); 1139246290Sdchagin } 1140191719Sdchagin if (f != NULL) { 1141218117Sdchagin futex_wake(f, 1, FUTEX_BITSET_MATCH_ANY); 1142191719Sdchagin futex_put(f, NULL); 1143191719Sdchagin } 1144178976Srdivacky } 1145178976Srdivacky } 1146178976Srdivacky 1147246290Sdchagin LIN_SDT_PROBE1(futex, handle_futex_death, return, 0); 1148178976Srdivacky return (0); 1149178976Srdivacky} 1150178976Srdivacky 1151178976Srdivackystatic int 1152178976Srdivackyfetch_robust_entry(struct linux_robust_list **entry, 1153178976Srdivacky struct linux_robust_list **head, int *pi) 1154178976Srdivacky{ 1155178976Srdivacky l_ulong uentry; 1156246290Sdchagin int error; 1157178976Srdivacky 1158246290Sdchagin LIN_SDT_PROBE3(futex, fetch_robust_entry, entry, entry, head, pi); 1159246290Sdchagin 1160246290Sdchagin error = copyin((const void *)head, &uentry, sizeof(l_ulong)); 1161246290Sdchagin if (error) { 1162246290Sdchagin LIN_SDT_PROBE1(futex, fetch_robust_entry, copyin_error, error); 1163246290Sdchagin LIN_SDT_PROBE1(futex, fetch_robust_entry, return, EFAULT); 1164178976Srdivacky return (EFAULT); 1165246290Sdchagin } 1166178976Srdivacky 1167178976Srdivacky *entry = (void *)(uentry & ~1UL); 1168178976Srdivacky *pi = uentry & 1; 1169178976Srdivacky 1170246290Sdchagin LIN_SDT_PROBE1(futex, fetch_robust_entry, return, 0); 1171178976Srdivacky return (0); 1172178976Srdivacky} 1173178976Srdivacky 1174178976Srdivacky/* This walks the list of robust futexes releasing them. */ 1175178976Srdivackyvoid 1176178976Srdivackyrelease_futexes(struct proc *p) 1177178976Srdivacky{ 1178178976Srdivacky struct linux_robust_list_head *head = NULL; 1179178976Srdivacky struct linux_robust_list *entry, *next_entry, *pending; 1180178976Srdivacky unsigned int limit = 2048, pi, next_pi, pip; 1181178976Srdivacky struct linux_emuldata *em; 1182185002Skib l_long futex_offset; 1183246290Sdchagin int rc, error; 1184178976Srdivacky 1185246290Sdchagin LIN_SDT_PROBE1(futex, release_futexes, entry, p); 1186246290Sdchagin 1187178976Srdivacky em = em_find(p, EMUL_DONTLOCK); 1188178976Srdivacky head = em->robust_futexes; 1189178976Srdivacky 1190246290Sdchagin if (head == NULL) { 1191246290Sdchagin LIN_SDT_PROBE0(futex, release_futexes, return); 1192178976Srdivacky return; 1193246290Sdchagin } 1194178976Srdivacky 1195246290Sdchagin if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi)) { 1196246290Sdchagin LIN_SDT_PROBE0(futex, release_futexes, return); 1197178976Srdivacky return; 1198246290Sdchagin } 1199178976Srdivacky 1200246290Sdchagin error = copyin(&head->futex_offset, &futex_offset, 1201246290Sdchagin sizeof(futex_offset)); 1202246290Sdchagin if (error) { 1203246290Sdchagin LIN_SDT_PROBE1(futex, release_futexes, copyin_error, error); 1204246290Sdchagin LIN_SDT_PROBE0(futex, release_futexes, return); 1205178976Srdivacky return; 1206246290Sdchagin } 1207178976Srdivacky 1208246290Sdchagin if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip)) { 1209246290Sdchagin LIN_SDT_PROBE0(futex, release_futexes, return); 1210178976Srdivacky return; 1211246290Sdchagin } 1212178976Srdivacky 1213178976Srdivacky while (entry != &head->list) { 1214183871Skib rc = fetch_robust_entry(&next_entry, PTRIN(&entry->next), &next_pi); 1215178976Srdivacky 1216178976Srdivacky if (entry != pending) 1217246290Sdchagin if (handle_futex_death(p, 1218246292Sdchagin (uint32_t *)((caddr_t)entry + futex_offset), pi)) { 1219246290Sdchagin LIN_SDT_PROBE0(futex, release_futexes, return); 1220178976Srdivacky return; 1221246290Sdchagin } 1222246290Sdchagin if (rc) { 1223246290Sdchagin LIN_SDT_PROBE0(futex, release_futexes, return); 1224178976Srdivacky return; 1225246290Sdchagin } 1226178976Srdivacky 1227178976Srdivacky entry = next_entry; 1228178976Srdivacky pi = next_pi; 1229178976Srdivacky 1230178976Srdivacky if (!--limit) 1231178976Srdivacky break; 1232178976Srdivacky 1233178976Srdivacky sched_relinquish(curthread); 1234178976Srdivacky } 1235178976Srdivacky 1236178976Srdivacky if (pending) 1237246292Sdchagin handle_futex_death(p, (uint32_t *)((caddr_t)pending + futex_offset), pip); 1238246290Sdchagin 1239246290Sdchagin LIN_SDT_PROBE0(futex, release_futexes, return); 1240178976Srdivacky} 1241