linux_futex.c revision 301425
1/* $NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $ */ 2 3/*- 4 * Copyright (c) 2009-2016 Dmitry Chagin 5 * Copyright (c) 2005 Emmanuel Dreyfus 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Emmanuel Dreyfus 19 * 4. The name of the author may not be used to endorse or promote 20 * products derived from this software without specific prior written 21 * permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE THE AUTHOR AND CONTRIBUTORS ``AS IS'' 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 25 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36#include <sys/cdefs.h> 37__FBSDID("$FreeBSD: stable/10/sys/compat/linux/linux_futex.c 301425 2016-06-05 06:04:25Z dchagin $"); 38#if 0 39__KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $"); 40#endif 41 42#include "opt_compat.h" 43#include "opt_kdtrace.h" 44 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/imgact.h> 48#include <sys/kernel.h> 49#include <sys/ktr.h> 50#include <sys/lock.h> 51#include <sys/malloc.h> 52#include <sys/mutex.h> 53#include <sys/priv.h> 54#include <sys/proc.h> 55#include <sys/queue.h> 56#include <sys/sched.h> 57#include <sys/sdt.h> 58#include <sys/sx.h> 59#include <sys/umtx.h> 60 61#ifdef COMPAT_LINUX32 62#include <machine/../linux32/linux.h> 63#include <machine/../linux32/linux32_proto.h> 64#else 65#include <machine/../linux/linux.h> 66#include <machine/../linux/linux_proto.h> 67#endif 68#include <compat/linux/linux_dtrace.h> 69#include <compat/linux/linux_emul.h> 70#include <compat/linux/linux_futex.h> 71#include <compat/linux/linux_timer.h> 72#include <compat/linux/linux_util.h> 73 74/* DTrace init */ 75LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE); 76 77/** 78 * Futex part for the special DTrace module "locks". 79 */ 80LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, locked, "struct mtx *"); 81LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, unlock, "struct mtx *"); 82 83/** 84 * Per futex probes. 85 */ 86LIN_SDT_PROBE_DEFINE1(futex, futex, create, "struct sx *"); 87LIN_SDT_PROBE_DEFINE1(futex, futex, destroy, "struct sx *"); 88 89/** 90 * DTrace probes in this module. 91 */ 92LIN_SDT_PROBE_DEFINE2(futex, futex_put, entry, "struct futex *", 93 "struct waiting_proc *"); 94LIN_SDT_PROBE_DEFINE3(futex, futex_put, destroy, "uint32_t *", "uint32_t", 95 "int"); 96LIN_SDT_PROBE_DEFINE3(futex, futex_put, unlock, "uint32_t *", "uint32_t", 97 "int"); 98LIN_SDT_PROBE_DEFINE0(futex, futex_put, return); 99LIN_SDT_PROBE_DEFINE3(futex, futex_get0, entry, "uint32_t *", "struct futex **", 100 "uint32_t"); 101LIN_SDT_PROBE_DEFINE1(futex, futex_get0, umtx_key_get_error, "int"); 102LIN_SDT_PROBE_DEFINE3(futex, futex_get0, shared, "uint32_t *", "uint32_t", 103 "int"); 104LIN_SDT_PROBE_DEFINE1(futex, futex_get0, null, "uint32_t *"); 105LIN_SDT_PROBE_DEFINE3(futex, futex_get0, new, "uint32_t *", "uint32_t", "int"); 106LIN_SDT_PROBE_DEFINE1(futex, futex_get0, return, "int"); 107LIN_SDT_PROBE_DEFINE3(futex, futex_get, entry, "uint32_t *", 108 "struct waiting_proc **", "struct futex **"); 109LIN_SDT_PROBE_DEFINE0(futex, futex_get, error); 110LIN_SDT_PROBE_DEFINE1(futex, futex_get, return, "int"); 111LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, entry, "struct futex *", 112 "struct waiting_proc **", "int"); 113LIN_SDT_PROBE_DEFINE5(futex, futex_sleep, requeue_error, "int", "uint32_t *", 114 "struct waiting_proc *", "uint32_t *", "uint32_t"); 115LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, sleep_error, "int", "uint32_t *", 116 "struct waiting_proc *"); 117LIN_SDT_PROBE_DEFINE1(futex, futex_sleep, return, "int"); 118LIN_SDT_PROBE_DEFINE3(futex, futex_wake, entry, "struct futex *", "int", 119 "uint32_t"); 120LIN_SDT_PROBE_DEFINE3(futex, futex_wake, iterate, "uint32_t", 121 "struct waiting_proc *", "uint32_t"); 122LIN_SDT_PROBE_DEFINE1(futex, futex_wake, wakeup, "struct waiting_proc *"); 123LIN_SDT_PROBE_DEFINE1(futex, futex_wake, return, "int"); 124LIN_SDT_PROBE_DEFINE4(futex, futex_requeue, entry, "struct futex *", "int", 125 "struct futex *", "int"); 126LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, wakeup, "struct waiting_proc *"); 127LIN_SDT_PROBE_DEFINE3(futex, futex_requeue, requeue, "uint32_t *", 128 "struct waiting_proc *", "uint32_t"); 129LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, return, "int"); 130LIN_SDT_PROBE_DEFINE4(futex, futex_wait, entry, "struct futex *", 131 "struct waiting_proc **", "int", "uint32_t"); 132LIN_SDT_PROBE_DEFINE1(futex, futex_wait, sleep_error, "int"); 133LIN_SDT_PROBE_DEFINE1(futex, futex_wait, return, "int"); 134LIN_SDT_PROBE_DEFINE3(futex, futex_atomic_op, entry, "struct thread *", 135 "int", "uint32_t"); 136LIN_SDT_PROBE_DEFINE4(futex, futex_atomic_op, decoded_op, "int", "int", "int", 137 "int"); 138LIN_SDT_PROBE_DEFINE0(futex, futex_atomic_op, missing_access_check); 139LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_op, "int"); 140LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_cmp, "int"); 141LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, return, "int"); 142LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, entry, "struct thread *", 143 "struct linux_sys_futex_args *"); 144LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_clockswitch); 145LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, itimerfix_error, "int"); 146LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, copyin_error, "int"); 147LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, invalid_cmp_requeue_use); 148LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wait, "uint32_t *", 149 "uint32_t", "uint32_t"); 150LIN_SDT_PROBE_DEFINE4(futex, linux_sys_futex, debug_wait_value_neq, 151 "uint32_t *", "uint32_t", "int", "uint32_t"); 152LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wake, "uint32_t *", 153 "uint32_t", "uint32_t"); 154LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_cmp_requeue, "uint32_t *", 155 "uint32_t", "uint32_t", "uint32_t *", "struct l_timespec *"); 156LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, debug_cmp_requeue_value_neq, 157 "uint32_t", "int"); 158LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_wake_op, "uint32_t *", 159 "int", "uint32_t", "uint32_t *", "uint32_t"); 160LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unhandled_efault); 161LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_lock_pi); 162LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_unlock_pi); 163LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_trylock_pi); 164LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, deprecated_requeue); 165LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_wait_requeue_pi); 166LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_cmp_requeue_pi); 167LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, unknown_operation, "int"); 168LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, return, "int"); 169LIN_SDT_PROBE_DEFINE2(futex, linux_set_robust_list, entry, "struct thread *", 170 "struct linux_set_robust_list_args *"); 171LIN_SDT_PROBE_DEFINE0(futex, linux_set_robust_list, size_error); 172LIN_SDT_PROBE_DEFINE1(futex, linux_set_robust_list, return, "int"); 173LIN_SDT_PROBE_DEFINE2(futex, linux_get_robust_list, entry, "struct thread *", 174 "struct linux_get_robust_list_args *"); 175LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, copyout_error, "int"); 176LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, return, "int"); 177LIN_SDT_PROBE_DEFINE3(futex, handle_futex_death, entry, 178 "struct linux_emuldata *", "uint32_t *", "unsigned int"); 179LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, copyin_error, "int"); 180LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, return, "int"); 181LIN_SDT_PROBE_DEFINE3(futex, fetch_robust_entry, entry, 182 "struct linux_robust_list **", "struct linux_robust_list **", 183 "unsigned int *"); 184LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, copyin_error, "int"); 185LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, return, "int"); 186LIN_SDT_PROBE_DEFINE2(futex, release_futexes, entry, "struct thread *", 187 "struct linux_emuldata *"); 188LIN_SDT_PROBE_DEFINE1(futex, release_futexes, copyin_error, "int"); 189LIN_SDT_PROBE_DEFINE0(futex, release_futexes, return); 190 191struct futex; 192 193struct waiting_proc { 194 uint32_t wp_flags; 195 struct futex *wp_futex; 196 TAILQ_ENTRY(waiting_proc) wp_list; 197}; 198 199struct futex { 200 struct sx f_lck; 201 uint32_t *f_uaddr; /* user-supplied value, for debug */ 202 struct umtx_key f_key; 203 uint32_t f_refcount; 204 uint32_t f_bitset; 205 LIST_ENTRY(futex) f_list; 206 TAILQ_HEAD(lf_waiting_proc, waiting_proc) f_waiting_proc; 207}; 208 209struct futex_list futex_list; 210 211#define FUTEX_LOCK(f) sx_xlock(&(f)->f_lck) 212#define FUTEX_UNLOCK(f) sx_xunlock(&(f)->f_lck) 213#define FUTEX_INIT(f) do { \ 214 sx_init_flags(&(f)->f_lck, "ftlk", \ 215 SX_DUPOK); \ 216 LIN_SDT_PROBE1(futex, futex, create, \ 217 &(f)->f_lck); \ 218 } while (0) 219#define FUTEX_DESTROY(f) do { \ 220 LIN_SDT_PROBE1(futex, futex, destroy, \ 221 &(f)->f_lck); \ 222 sx_destroy(&(f)->f_lck); \ 223 } while (0) 224#define FUTEX_ASSERT_LOCKED(f) sx_assert(&(f)->f_lck, SA_XLOCKED) 225 226struct mtx futex_mtx; /* protects the futex list */ 227#define FUTEXES_LOCK do { \ 228 mtx_lock(&futex_mtx); \ 229 LIN_SDT_PROBE1(locks, futex_mtx, \ 230 locked, &futex_mtx); \ 231 } while (0) 232#define FUTEXES_UNLOCK do { \ 233 LIN_SDT_PROBE1(locks, futex_mtx, \ 234 unlock, &futex_mtx); \ 235 mtx_unlock(&futex_mtx); \ 236 } while (0) 237 238/* flags for futex_get() */ 239#define FUTEX_CREATE_WP 0x1 /* create waiting_proc */ 240#define FUTEX_DONTCREATE 0x2 /* don't create futex if not exists */ 241#define FUTEX_DONTEXISTS 0x4 /* return EINVAL if futex exists */ 242#define FUTEX_SHARED 0x8 /* shared futex */ 243 244/* wp_flags */ 245#define FUTEX_WP_REQUEUED 0x1 /* wp requeued - wp moved from wp_list 246 * of futex where thread sleep to wp_list 247 * of another futex. 248 */ 249#define FUTEX_WP_REMOVED 0x2 /* wp is woken up and removed from futex 250 * wp_list to prevent double wakeup. 251 */ 252 253static void futex_put(struct futex *, struct waiting_proc *); 254static int futex_get0(uint32_t *, struct futex **f, uint32_t); 255static int futex_get(uint32_t *, struct waiting_proc **, struct futex **, 256 uint32_t); 257static int futex_sleep(struct futex *, struct waiting_proc *, int); 258static int futex_wake(struct futex *, int, uint32_t); 259static int futex_requeue(struct futex *, int, struct futex *, int); 260static int futex_wait(struct futex *, struct waiting_proc *, int, 261 uint32_t); 262static int futex_atomic_op(struct thread *, int, uint32_t *); 263static int handle_futex_death(struct linux_emuldata *, uint32_t *, 264 unsigned int); 265static int fetch_robust_entry(struct linux_robust_list **, 266 struct linux_robust_list **, unsigned int *); 267 268/* support.s */ 269int futex_xchgl(int oparg, uint32_t *uaddr, int *oldval); 270int futex_addl(int oparg, uint32_t *uaddr, int *oldval); 271int futex_orl(int oparg, uint32_t *uaddr, int *oldval); 272int futex_andl(int oparg, uint32_t *uaddr, int *oldval); 273int futex_xorl(int oparg, uint32_t *uaddr, int *oldval); 274 275 276static void 277futex_put(struct futex *f, struct waiting_proc *wp) 278{ 279 LIN_SDT_PROBE2(futex, futex_put, entry, f, wp); 280 281 FUTEX_ASSERT_LOCKED(f); 282 if (wp != NULL) { 283 if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0) 284 TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 285 free(wp, M_FUTEX_WP); 286 } 287 288 FUTEXES_LOCK; 289 if (--f->f_refcount == 0) { 290 LIST_REMOVE(f, f_list); 291 FUTEXES_UNLOCK; 292 FUTEX_UNLOCK(f); 293 294 LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr, 295 f->f_refcount, f->f_key.shared); 296 LINUX_CTR3(sys_futex, "futex_put destroy uaddr %p ref %d " 297 "shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared); 298 umtx_key_release(&f->f_key); 299 FUTEX_DESTROY(f); 300 free(f, M_FUTEX); 301 302 LIN_SDT_PROBE0(futex, futex_put, return); 303 return; 304 } 305 306 LIN_SDT_PROBE3(futex, futex_put, unlock, f->f_uaddr, f->f_refcount, 307 f->f_key.shared); 308 LINUX_CTR3(sys_futex, "futex_put uaddr %p ref %d shared %d", 309 f->f_uaddr, f->f_refcount, f->f_key.shared); 310 FUTEXES_UNLOCK; 311 FUTEX_UNLOCK(f); 312 313 LIN_SDT_PROBE0(futex, futex_put, return); 314} 315 316static int 317futex_get0(uint32_t *uaddr, struct futex **newf, uint32_t flags) 318{ 319 struct futex *f, *tmpf; 320 struct umtx_key key; 321 int error; 322 323 LIN_SDT_PROBE3(futex, futex_get0, entry, uaddr, newf, flags); 324 325 *newf = tmpf = NULL; 326 327 error = umtx_key_get(uaddr, TYPE_FUTEX, (flags & FUTEX_SHARED) ? 328 AUTO_SHARE : THREAD_SHARE, &key); 329 if (error) { 330 LIN_SDT_PROBE1(futex, futex_get0, umtx_key_get_error, error); 331 LIN_SDT_PROBE1(futex, futex_get0, return, error); 332 return (error); 333 } 334retry: 335 FUTEXES_LOCK; 336 LIST_FOREACH(f, &futex_list, f_list) { 337 if (umtx_key_match(&f->f_key, &key)) { 338 if (tmpf != NULL) { 339 FUTEX_UNLOCK(tmpf); 340 FUTEX_DESTROY(tmpf); 341 free(tmpf, M_FUTEX); 342 } 343 if (flags & FUTEX_DONTEXISTS) { 344 FUTEXES_UNLOCK; 345 umtx_key_release(&key); 346 347 LIN_SDT_PROBE1(futex, futex_get0, return, 348 EINVAL); 349 return (EINVAL); 350 } 351 352 /* 353 * Increment refcount of the found futex to 354 * prevent it from deallocation before FUTEX_LOCK() 355 */ 356 ++f->f_refcount; 357 FUTEXES_UNLOCK; 358 umtx_key_release(&key); 359 360 FUTEX_LOCK(f); 361 *newf = f; 362 LIN_SDT_PROBE3(futex, futex_get0, shared, uaddr, 363 f->f_refcount, f->f_key.shared); 364 LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d", 365 uaddr, f->f_refcount, f->f_key.shared); 366 367 LIN_SDT_PROBE1(futex, futex_get0, return, 0); 368 return (0); 369 } 370 } 371 372 if (flags & FUTEX_DONTCREATE) { 373 FUTEXES_UNLOCK; 374 umtx_key_release(&key); 375 LIN_SDT_PROBE1(futex, futex_get0, null, uaddr); 376 LINUX_CTR1(sys_futex, "futex_get uaddr %p null", uaddr); 377 378 LIN_SDT_PROBE1(futex, futex_get0, return, 0); 379 return (0); 380 } 381 382 if (tmpf == NULL) { 383 FUTEXES_UNLOCK; 384 tmpf = malloc(sizeof(*tmpf), M_FUTEX, M_WAITOK | M_ZERO); 385 tmpf->f_uaddr = uaddr; 386 tmpf->f_key = key; 387 tmpf->f_refcount = 1; 388 tmpf->f_bitset = FUTEX_BITSET_MATCH_ANY; 389 FUTEX_INIT(tmpf); 390 TAILQ_INIT(&tmpf->f_waiting_proc); 391 392 /* 393 * Lock the new futex before an insert into the futex_list 394 * to prevent futex usage by other. 395 */ 396 FUTEX_LOCK(tmpf); 397 goto retry; 398 } 399 400 LIST_INSERT_HEAD(&futex_list, tmpf, f_list); 401 FUTEXES_UNLOCK; 402 403 LIN_SDT_PROBE3(futex, futex_get0, new, uaddr, tmpf->f_refcount, 404 tmpf->f_key.shared); 405 LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d new", 406 uaddr, tmpf->f_refcount, tmpf->f_key.shared); 407 *newf = tmpf; 408 409 LIN_SDT_PROBE1(futex, futex_get0, return, 0); 410 return (0); 411} 412 413static int 414futex_get(uint32_t *uaddr, struct waiting_proc **wp, struct futex **f, 415 uint32_t flags) 416{ 417 int error; 418 419 LIN_SDT_PROBE3(futex, futex_get, entry, uaddr, wp, f); 420 421 if (flags & FUTEX_CREATE_WP) { 422 *wp = malloc(sizeof(struct waiting_proc), M_FUTEX_WP, M_WAITOK); 423 (*wp)->wp_flags = 0; 424 } 425 error = futex_get0(uaddr, f, flags); 426 if (error) { 427 LIN_SDT_PROBE0(futex, futex_get, error); 428 429 if (flags & FUTEX_CREATE_WP) 430 free(*wp, M_FUTEX_WP); 431 432 LIN_SDT_PROBE1(futex, futex_get, return, error); 433 return (error); 434 } 435 if (flags & FUTEX_CREATE_WP) { 436 TAILQ_INSERT_HEAD(&(*f)->f_waiting_proc, *wp, wp_list); 437 (*wp)->wp_futex = *f; 438 } 439 440 LIN_SDT_PROBE1(futex, futex_get, return, error); 441 return (error); 442} 443 444static int 445futex_sleep(struct futex *f, struct waiting_proc *wp, int timeout) 446{ 447 int error; 448 449 FUTEX_ASSERT_LOCKED(f); 450 LIN_SDT_PROBE3(futex, futex_sleep, entry, f, wp, timeout); 451 LINUX_CTR4(sys_futex, "futex_sleep enter uaddr %p wp %p timo %d ref %d", 452 f->f_uaddr, wp, timeout, f->f_refcount); 453 error = sx_sleep(wp, &f->f_lck, PCATCH, "futex", timeout); 454 if (wp->wp_flags & FUTEX_WP_REQUEUED) { 455 KASSERT(f != wp->wp_futex, ("futex != wp_futex")); 456 457 if (error) { 458 LIN_SDT_PROBE5(futex, futex_sleep, requeue_error, error, 459 f->f_uaddr, wp, wp->wp_futex->f_uaddr, 460 wp->wp_futex->f_refcount); 461 } 462 463 LINUX_CTR5(sys_futex, "futex_sleep out error %d uaddr %p wp" 464 " %p requeued uaddr %p ref %d", 465 error, f->f_uaddr, wp, wp->wp_futex->f_uaddr, 466 wp->wp_futex->f_refcount); 467 futex_put(f, NULL); 468 f = wp->wp_futex; 469 FUTEX_LOCK(f); 470 } else { 471 if (error) { 472 LIN_SDT_PROBE3(futex, futex_sleep, sleep_error, error, 473 f->f_uaddr, wp); 474 } 475 LINUX_CTR3(sys_futex, "futex_sleep out error %d uaddr %p wp %p", 476 error, f->f_uaddr, wp); 477 } 478 479 futex_put(f, wp); 480 481 LIN_SDT_PROBE1(futex, futex_sleep, return, error); 482 return (error); 483} 484 485static int 486futex_wake(struct futex *f, int n, uint32_t bitset) 487{ 488 struct waiting_proc *wp, *wpt; 489 int count = 0; 490 491 LIN_SDT_PROBE3(futex, futex_wake, entry, f, n, bitset); 492 493 if (bitset == 0) { 494 LIN_SDT_PROBE1(futex, futex_wake, return, EINVAL); 495 return (EINVAL); 496 } 497 498 FUTEX_ASSERT_LOCKED(f); 499 TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) { 500 LIN_SDT_PROBE3(futex, futex_wake, iterate, f->f_uaddr, wp, 501 f->f_refcount); 502 LINUX_CTR3(sys_futex, "futex_wake uaddr %p wp %p ref %d", 503 f->f_uaddr, wp, f->f_refcount); 504 /* 505 * Unless we find a matching bit in 506 * the bitset, continue searching. 507 */ 508 if (!(wp->wp_futex->f_bitset & bitset)) 509 continue; 510 511 wp->wp_flags |= FUTEX_WP_REMOVED; 512 TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 513 LIN_SDT_PROBE1(futex, futex_wake, wakeup, wp); 514 wakeup_one(wp); 515 if (++count == n) 516 break; 517 } 518 519 LIN_SDT_PROBE1(futex, futex_wake, return, count); 520 return (count); 521} 522 523static int 524futex_requeue(struct futex *f, int n, struct futex *f2, int n2) 525{ 526 struct waiting_proc *wp, *wpt; 527 int count = 0; 528 529 LIN_SDT_PROBE4(futex, futex_requeue, entry, f, n, f2, n2); 530 531 FUTEX_ASSERT_LOCKED(f); 532 FUTEX_ASSERT_LOCKED(f2); 533 534 TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) { 535 if (++count <= n) { 536 LINUX_CTR2(sys_futex, "futex_req_wake uaddr %p wp %p", 537 f->f_uaddr, wp); 538 wp->wp_flags |= FUTEX_WP_REMOVED; 539 TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 540 LIN_SDT_PROBE1(futex, futex_requeue, wakeup, wp); 541 wakeup_one(wp); 542 } else { 543 LIN_SDT_PROBE3(futex, futex_requeue, requeue, 544 f->f_uaddr, wp, f2->f_uaddr); 545 LINUX_CTR3(sys_futex, "futex_requeue uaddr %p wp %p to %p", 546 f->f_uaddr, wp, f2->f_uaddr); 547 wp->wp_flags |= FUTEX_WP_REQUEUED; 548 /* Move wp to wp_list of f2 futex */ 549 TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); 550 TAILQ_INSERT_HEAD(&f2->f_waiting_proc, wp, wp_list); 551 552 /* 553 * Thread which sleeps on wp after waking should 554 * acquire f2 lock, so increment refcount of f2 to 555 * prevent it from premature deallocation. 556 */ 557 wp->wp_futex = f2; 558 FUTEXES_LOCK; 559 ++f2->f_refcount; 560 FUTEXES_UNLOCK; 561 if (count - n >= n2) 562 break; 563 } 564 } 565 566 LIN_SDT_PROBE1(futex, futex_requeue, return, count); 567 return (count); 568} 569 570static int 571futex_wait(struct futex *f, struct waiting_proc *wp, int timeout_hz, 572 uint32_t bitset) 573{ 574 int error; 575 576 LIN_SDT_PROBE4(futex, futex_wait, entry, f, wp, timeout_hz, bitset); 577 578 if (bitset == 0) { 579 LIN_SDT_PROBE1(futex, futex_wait, return, EINVAL); 580 return (EINVAL); 581 } 582 583 f->f_bitset = bitset; 584 error = futex_sleep(f, wp, timeout_hz); 585 if (error) 586 LIN_SDT_PROBE1(futex, futex_wait, sleep_error, error); 587 if (error == EWOULDBLOCK) 588 error = ETIMEDOUT; 589 590 LIN_SDT_PROBE1(futex, futex_wait, return, error); 591 return (error); 592} 593 594static int 595futex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr) 596{ 597 int op = (encoded_op >> 28) & 7; 598 int cmp = (encoded_op >> 24) & 15; 599 int oparg = (encoded_op << 8) >> 20; 600 int cmparg = (encoded_op << 20) >> 20; 601 int oldval = 0, ret; 602 603 LIN_SDT_PROBE3(futex, futex_atomic_op, entry, td, encoded_op, uaddr); 604 605 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) 606 oparg = 1 << oparg; 607 608 LIN_SDT_PROBE4(futex, futex_atomic_op, decoded_op, op, cmp, oparg, 609 cmparg); 610 611 /* XXX: Linux verifies access here and returns EFAULT */ 612 LIN_SDT_PROBE0(futex, futex_atomic_op, missing_access_check); 613 614 switch (op) { 615 case FUTEX_OP_SET: 616 ret = futex_xchgl(oparg, uaddr, &oldval); 617 break; 618 case FUTEX_OP_ADD: 619 ret = futex_addl(oparg, uaddr, &oldval); 620 break; 621 case FUTEX_OP_OR: 622 ret = futex_orl(oparg, uaddr, &oldval); 623 break; 624 case FUTEX_OP_ANDN: 625 ret = futex_andl(~oparg, uaddr, &oldval); 626 break; 627 case FUTEX_OP_XOR: 628 ret = futex_xorl(oparg, uaddr, &oldval); 629 break; 630 default: 631 LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_op, op); 632 ret = -ENOSYS; 633 break; 634 } 635 636 if (ret) { 637 LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret); 638 return (ret); 639 } 640 641 switch (cmp) { 642 case FUTEX_OP_CMP_EQ: 643 ret = (oldval == cmparg); 644 break; 645 case FUTEX_OP_CMP_NE: 646 ret = (oldval != cmparg); 647 break; 648 case FUTEX_OP_CMP_LT: 649 ret = (oldval < cmparg); 650 break; 651 case FUTEX_OP_CMP_GE: 652 ret = (oldval >= cmparg); 653 break; 654 case FUTEX_OP_CMP_LE: 655 ret = (oldval <= cmparg); 656 break; 657 case FUTEX_OP_CMP_GT: 658 ret = (oldval > cmparg); 659 break; 660 default: 661 LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_cmp, cmp); 662 ret = -ENOSYS; 663 } 664 665 LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret); 666 return (ret); 667} 668 669int 670linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args) 671{ 672 int clockrt, nrwake, op_ret, ret; 673 struct linux_pemuldata *pem; 674 struct waiting_proc *wp; 675 struct futex *f, *f2; 676 struct l_timespec ltimeout; 677 struct timespec timeout; 678 struct timeval utv, ctv; 679 int timeout_hz; 680 int error; 681 uint32_t flags, val; 682 683 LIN_SDT_PROBE2(futex, linux_sys_futex, entry, td, args); 684 685 if (args->op & LINUX_FUTEX_PRIVATE_FLAG) { 686 flags = 0; 687 args->op &= ~LINUX_FUTEX_PRIVATE_FLAG; 688 } else 689 flags = FUTEX_SHARED; 690 691 /* 692 * Currently support for switching between CLOCK_MONOTONIC and 693 * CLOCK_REALTIME is not present. However Linux forbids the use of 694 * FUTEX_CLOCK_REALTIME with any op except FUTEX_WAIT_BITSET and 695 * FUTEX_WAIT_REQUEUE_PI. 696 */ 697 clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME; 698 args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME; 699 if (clockrt && args->op != LINUX_FUTEX_WAIT_BITSET && 700 args->op != LINUX_FUTEX_WAIT_REQUEUE_PI) { 701 LIN_SDT_PROBE0(futex, linux_sys_futex, 702 unimplemented_clockswitch); 703 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 704 return (ENOSYS); 705 } 706 707 error = 0; 708 f = f2 = NULL; 709 710 switch (args->op) { 711 case LINUX_FUTEX_WAIT: 712 args->val3 = FUTEX_BITSET_MATCH_ANY; 713 /* FALLTHROUGH */ 714 715 case LINUX_FUTEX_WAIT_BITSET: 716 LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wait, args->uaddr, 717 args->val, args->val3); 718 LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x", 719 args->uaddr, args->val, args->val3); 720 721 if (args->timeout != NULL) { 722 error = copyin(args->timeout, <imeout, sizeof(ltimeout)); 723 if (error) { 724 LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, 725 error); 726 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 727 return (error); 728 } 729 error = linux_to_native_timespec(&timeout, <imeout); 730 if (error) 731 return (error); 732 TIMESPEC_TO_TIMEVAL(&utv, &timeout); 733 error = itimerfix(&utv); 734 if (error) { 735 LIN_SDT_PROBE1(futex, linux_sys_futex, itimerfix_error, 736 error); 737 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 738 return (error); 739 } 740 if (clockrt) { 741 microtime(&ctv); 742 timevalsub(&utv, &ctv); 743 } else if (args->op == LINUX_FUTEX_WAIT_BITSET) { 744 microuptime(&ctv); 745 timevalsub(&utv, &ctv); 746 } 747 if (utv.tv_sec < 0) 748 timevalclear(&utv); 749 timeout_hz = tvtohz(&utv); 750 } else 751 timeout_hz = 0; 752 753 error = futex_get(args->uaddr, &wp, &f, 754 flags | FUTEX_CREATE_WP); 755 if (error) { 756 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 757 return (error); 758 } 759 760 error = copyin(args->uaddr, &val, sizeof(val)); 761 if (error) { 762 LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, 763 error); 764 LINUX_CTR1(sys_futex, "WAIT copyin failed %d", 765 error); 766 futex_put(f, wp); 767 768 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 769 return (error); 770 } 771 if (val != args->val) { 772 LIN_SDT_PROBE4(futex, linux_sys_futex, 773 debug_wait_value_neq, args->uaddr, args->val, val, 774 args->val3); 775 LINUX_CTR3(sys_futex, 776 "WAIT uaddr %p val 0x%x != uval 0x%x", 777 args->uaddr, args->val, val); 778 futex_put(f, wp); 779 780 LIN_SDT_PROBE1(futex, linux_sys_futex, return, 781 EWOULDBLOCK); 782 return (EWOULDBLOCK); 783 } 784 785 error = futex_wait(f, wp, timeout_hz, args->val3); 786 break; 787 788 case LINUX_FUTEX_WAKE: 789 args->val3 = FUTEX_BITSET_MATCH_ANY; 790 /* FALLTHROUGH */ 791 792 case LINUX_FUTEX_WAKE_BITSET: 793 LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wake, args->uaddr, 794 args->val, args->val3); 795 LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x", 796 args->uaddr, args->val, args->val3); 797 798 error = futex_get(args->uaddr, NULL, &f, 799 flags | FUTEX_DONTCREATE); 800 if (error) { 801 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 802 return (error); 803 } 804 805 if (f == NULL) { 806 td->td_retval[0] = 0; 807 808 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 809 return (error); 810 } 811 td->td_retval[0] = futex_wake(f, args->val, args->val3); 812 futex_put(f, NULL); 813 break; 814 815 case LINUX_FUTEX_CMP_REQUEUE: 816 LIN_SDT_PROBE5(futex, linux_sys_futex, debug_cmp_requeue, 817 args->uaddr, args->val, args->val3, args->uaddr2, 818 args->timeout); 819 LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p " 820 "nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x", 821 args->uaddr, args->val, args->val3, args->uaddr2, 822 args->timeout); 823 824 /* 825 * Linux allows this, we would not, it is an incorrect 826 * usage of declared ABI, so return EINVAL. 827 */ 828 if (args->uaddr == args->uaddr2) { 829 LIN_SDT_PROBE0(futex, linux_sys_futex, 830 invalid_cmp_requeue_use); 831 LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL); 832 return (EINVAL); 833 } 834 835 error = futex_get(args->uaddr, NULL, &f, flags); 836 if (error) { 837 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 838 return (error); 839 } 840 841 /* 842 * To avoid deadlocks return EINVAL if second futex 843 * exists at this time. 844 * 845 * Glibc fall back to FUTEX_WAKE in case of any error 846 * returned by FUTEX_CMP_REQUEUE. 847 */ 848 error = futex_get(args->uaddr2, NULL, &f2, 849 flags | FUTEX_DONTEXISTS); 850 if (error) { 851 futex_put(f, NULL); 852 853 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 854 return (error); 855 } 856 error = copyin(args->uaddr, &val, sizeof(val)); 857 if (error) { 858 LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, 859 error); 860 LINUX_CTR1(sys_futex, "CMP_REQUEUE copyin failed %d", 861 error); 862 futex_put(f2, NULL); 863 futex_put(f, NULL); 864 865 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 866 return (error); 867 } 868 if (val != args->val3) { 869 LIN_SDT_PROBE2(futex, linux_sys_futex, 870 debug_cmp_requeue_value_neq, args->val, val); 871 LINUX_CTR2(sys_futex, "CMP_REQUEUE val 0x%x != uval 0x%x", 872 args->val, val); 873 futex_put(f2, NULL); 874 futex_put(f, NULL); 875 876 LIN_SDT_PROBE1(futex, linux_sys_futex, return, EAGAIN); 877 return (EAGAIN); 878 } 879 880 nrwake = (int)(unsigned long)args->timeout; 881 td->td_retval[0] = futex_requeue(f, args->val, f2, nrwake); 882 futex_put(f2, NULL); 883 futex_put(f, NULL); 884 break; 885 886 case LINUX_FUTEX_WAKE_OP: 887 LIN_SDT_PROBE5(futex, linux_sys_futex, debug_wake_op, 888 args->uaddr, args->op, args->val, args->uaddr2, args->val3); 889 LINUX_CTR5(sys_futex, "WAKE_OP " 890 "uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x", 891 args->uaddr, args->val, args->uaddr2, args->val3, 892 args->timeout); 893 894 error = futex_get(args->uaddr, NULL, &f, flags); 895 if (error) { 896 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 897 return (error); 898 } 899 900 if (args->uaddr != args->uaddr2) 901 error = futex_get(args->uaddr2, NULL, &f2, flags); 902 if (error) { 903 futex_put(f, NULL); 904 905 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 906 return (error); 907 } 908 909 /* 910 * This function returns positive number as results and 911 * negative as errors 912 */ 913 op_ret = futex_atomic_op(td, args->val3, args->uaddr2); 914 915 LINUX_CTR2(sys_futex, "WAKE_OP atomic_op uaddr %p ret 0x%x", 916 args->uaddr, op_ret); 917 918 if (op_ret < 0) { 919 /* XXX: We don't handle the EFAULT yet. */ 920 if (op_ret != -EFAULT) { 921 if (f2 != NULL) 922 futex_put(f2, NULL); 923 futex_put(f, NULL); 924 925 LIN_SDT_PROBE1(futex, linux_sys_futex, return, 926 -op_ret); 927 return (-op_ret); 928 } else { 929 LIN_SDT_PROBE0(futex, linux_sys_futex, 930 unhandled_efault); 931 } 932 if (f2 != NULL) 933 futex_put(f2, NULL); 934 futex_put(f, NULL); 935 936 LIN_SDT_PROBE1(futex, linux_sys_futex, return, EFAULT); 937 return (EFAULT); 938 } 939 940 ret = futex_wake(f, args->val, args->val3); 941 942 if (op_ret > 0) { 943 op_ret = 0; 944 nrwake = (int)(unsigned long)args->timeout; 945 946 if (f2 != NULL) 947 op_ret += futex_wake(f2, nrwake, args->val3); 948 else 949 op_ret += futex_wake(f, nrwake, args->val3); 950 ret += op_ret; 951 952 } 953 if (f2 != NULL) 954 futex_put(f2, NULL); 955 futex_put(f, NULL); 956 td->td_retval[0] = ret; 957 break; 958 959 case LINUX_FUTEX_LOCK_PI: 960 /* not yet implemented */ 961 pem = pem_find(td->td_proc); 962 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) { 963 linux_msg(td, 964 "linux_sys_futex: " 965 "unsupported futex_pi op\n"); 966 pem->flags |= LINUX_XUNSUP_FUTEXPIOP; 967 LIN_SDT_PROBE0(futex, linux_sys_futex, 968 unimplemented_lock_pi); 969 } 970 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 971 return (ENOSYS); 972 973 case LINUX_FUTEX_UNLOCK_PI: 974 /* not yet implemented */ 975 pem = pem_find(td->td_proc); 976 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) { 977 linux_msg(td, 978 "linux_sys_futex: " 979 "unsupported futex_pi op\n"); 980 pem->flags |= LINUX_XUNSUP_FUTEXPIOP; 981 LIN_SDT_PROBE0(futex, linux_sys_futex, 982 unimplemented_unlock_pi); 983 } 984 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 985 return (ENOSYS); 986 987 case LINUX_FUTEX_TRYLOCK_PI: 988 /* not yet implemented */ 989 pem = pem_find(td->td_proc); 990 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) { 991 linux_msg(td, 992 "linux_sys_futex: " 993 "unsupported futex_pi op\n"); 994 pem->flags |= LINUX_XUNSUP_FUTEXPIOP; 995 LIN_SDT_PROBE0(futex, linux_sys_futex, 996 unimplemented_trylock_pi); 997 } 998 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 999 return (ENOSYS); 1000 1001 case LINUX_FUTEX_REQUEUE: 1002 /* 1003 * Glibc does not use this operation since version 2.3.3, 1004 * as it is racy and replaced by FUTEX_CMP_REQUEUE operation. 1005 * Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when 1006 * FUTEX_REQUEUE returned EINVAL. 1007 */ 1008 pem = pem_find(td->td_proc); 1009 if ((pem->flags & LINUX_XDEPR_REQUEUEOP) == 0) { 1010 linux_msg(td, 1011 "linux_sys_futex: " 1012 "unsupported futex_requeue op\n"); 1013 pem->flags |= LINUX_XDEPR_REQUEUEOP; 1014 LIN_SDT_PROBE0(futex, linux_sys_futex, 1015 deprecated_requeue); 1016 } 1017 1018 LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL); 1019 return (EINVAL); 1020 1021 case LINUX_FUTEX_WAIT_REQUEUE_PI: 1022 /* not yet implemented */ 1023 pem = pem_find(td->td_proc); 1024 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) { 1025 linux_msg(td, 1026 "linux_sys_futex: " 1027 "unsupported futex_pi op\n"); 1028 pem->flags |= LINUX_XUNSUP_FUTEXPIOP; 1029 LIN_SDT_PROBE0(futex, linux_sys_futex, 1030 unimplemented_wait_requeue_pi); 1031 } 1032 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 1033 return (ENOSYS); 1034 1035 case LINUX_FUTEX_CMP_REQUEUE_PI: 1036 /* not yet implemented */ 1037 pem = pem_find(td->td_proc); 1038 if ((pem->flags & LINUX_XUNSUP_FUTEXPIOP) == 0) { 1039 linux_msg(td, 1040 "linux_sys_futex: " 1041 "unsupported futex_pi op\n"); 1042 pem->flags |= LINUX_XUNSUP_FUTEXPIOP; 1043 LIN_SDT_PROBE0(futex, linux_sys_futex, 1044 unimplemented_cmp_requeue_pi); 1045 } 1046 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 1047 return (ENOSYS); 1048 1049 default: 1050 linux_msg(td, 1051 "linux_sys_futex: unknown op %d\n", args->op); 1052 LIN_SDT_PROBE1(futex, linux_sys_futex, unknown_operation, 1053 args->op); 1054 LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS); 1055 return (ENOSYS); 1056 } 1057 1058 LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); 1059 return (error); 1060} 1061 1062int 1063linux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args) 1064{ 1065 struct linux_emuldata *em; 1066 1067 LIN_SDT_PROBE2(futex, linux_set_robust_list, entry, td, args); 1068 1069 if (args->len != sizeof(struct linux_robust_list_head)) { 1070 LIN_SDT_PROBE0(futex, linux_set_robust_list, size_error); 1071 LIN_SDT_PROBE1(futex, linux_set_robust_list, return, EINVAL); 1072 return (EINVAL); 1073 } 1074 1075 em = em_find(td); 1076 em->robust_futexes = args->head; 1077 1078 LIN_SDT_PROBE1(futex, linux_set_robust_list, return, 0); 1079 return (0); 1080} 1081 1082int 1083linux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args) 1084{ 1085 struct linux_emuldata *em; 1086 struct linux_robust_list_head *head; 1087 l_size_t len = sizeof(struct linux_robust_list_head); 1088 struct thread *td2; 1089 int error = 0; 1090 1091 LIN_SDT_PROBE2(futex, linux_get_robust_list, entry, td, args); 1092 1093 if (!args->pid) { 1094 em = em_find(td); 1095 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n")); 1096 head = em->robust_futexes; 1097 } else { 1098 td2 = tdfind(args->pid, -1); 1099 if (td2 == NULL) { 1100 LIN_SDT_PROBE1(futex, linux_get_robust_list, return, 1101 ESRCH); 1102 return (ESRCH); 1103 } 1104 if (SV_PROC_ABI(td2->td_proc) != SV_ABI_LINUX) { 1105 LIN_SDT_PROBE1(futex, linux_get_robust_list, return, 1106 EPERM); 1107 PROC_UNLOCK(td2->td_proc); 1108 return (EPERM); 1109 } 1110 1111 em = em_find(td2); 1112 KASSERT(em != NULL, ("get_robust_list: emuldata notfound.\n")); 1113 /* XXX: ptrace? */ 1114 if (priv_check(td, PRIV_CRED_SETUID) || 1115 priv_check(td, PRIV_CRED_SETEUID) || 1116 p_candebug(td, td2->td_proc)) { 1117 PROC_UNLOCK(td2->td_proc); 1118 1119 LIN_SDT_PROBE1(futex, linux_get_robust_list, return, 1120 EPERM); 1121 return (EPERM); 1122 } 1123 head = em->robust_futexes; 1124 1125 PROC_UNLOCK(td2->td_proc); 1126 } 1127 1128 error = copyout(&len, args->len, sizeof(l_size_t)); 1129 if (error) { 1130 LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error, 1131 error); 1132 LIN_SDT_PROBE1(futex, linux_get_robust_list, return, EFAULT); 1133 return (EFAULT); 1134 } 1135 1136 error = copyout(&head, args->head, sizeof(head)); 1137 if (error) { 1138 LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error, 1139 error); 1140 } 1141 1142 LIN_SDT_PROBE1(futex, linux_get_robust_list, return, error); 1143 return (error); 1144} 1145 1146static int 1147handle_futex_death(struct linux_emuldata *em, uint32_t *uaddr, 1148 unsigned int pi) 1149{ 1150 uint32_t uval, nval, mval; 1151 struct futex *f; 1152 int error; 1153 1154 LIN_SDT_PROBE3(futex, handle_futex_death, entry, em, uaddr, pi); 1155 1156retry: 1157 error = copyin(uaddr, &uval, 4); 1158 if (error) { 1159 LIN_SDT_PROBE1(futex, handle_futex_death, copyin_error, error); 1160 LIN_SDT_PROBE1(futex, handle_futex_death, return, EFAULT); 1161 return (EFAULT); 1162 } 1163 if ((uval & FUTEX_TID_MASK) == em->em_tid) { 1164 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED; 1165 nval = casuword32(uaddr, uval, mval); 1166 1167 if (nval == -1) { 1168 LIN_SDT_PROBE1(futex, handle_futex_death, return, 1169 EFAULT); 1170 return (EFAULT); 1171 } 1172 1173 if (nval != uval) 1174 goto retry; 1175 1176 if (!pi && (uval & FUTEX_WAITERS)) { 1177 error = futex_get(uaddr, NULL, &f, 1178 FUTEX_DONTCREATE | FUTEX_SHARED); 1179 if (error) { 1180 LIN_SDT_PROBE1(futex, handle_futex_death, 1181 return, error); 1182 return (error); 1183 } 1184 if (f != NULL) { 1185 futex_wake(f, 1, FUTEX_BITSET_MATCH_ANY); 1186 futex_put(f, NULL); 1187 } 1188 } 1189 } 1190 1191 LIN_SDT_PROBE1(futex, handle_futex_death, return, 0); 1192 return (0); 1193} 1194 1195static int 1196fetch_robust_entry(struct linux_robust_list **entry, 1197 struct linux_robust_list **head, unsigned int *pi) 1198{ 1199 l_ulong uentry; 1200 int error; 1201 1202 LIN_SDT_PROBE3(futex, fetch_robust_entry, entry, entry, head, pi); 1203 1204 error = copyin((const void *)head, &uentry, sizeof(l_ulong)); 1205 if (error) { 1206 LIN_SDT_PROBE1(futex, fetch_robust_entry, copyin_error, error); 1207 LIN_SDT_PROBE1(futex, fetch_robust_entry, return, EFAULT); 1208 return (EFAULT); 1209 } 1210 1211 *entry = (void *)(uentry & ~1UL); 1212 *pi = uentry & 1; 1213 1214 LIN_SDT_PROBE1(futex, fetch_robust_entry, return, 0); 1215 return (0); 1216} 1217 1218/* This walks the list of robust futexes releasing them. */ 1219void 1220release_futexes(struct thread *td, struct linux_emuldata *em) 1221{ 1222 struct linux_robust_list_head *head = NULL; 1223 struct linux_robust_list *entry, *next_entry, *pending; 1224 unsigned int limit = 2048, pi, next_pi, pip; 1225 l_long futex_offset; 1226 int rc, error; 1227 1228 LIN_SDT_PROBE2(futex, release_futexes, entry, td, em); 1229 1230 head = em->robust_futexes; 1231 1232 if (head == NULL) { 1233 LIN_SDT_PROBE0(futex, release_futexes, return); 1234 return; 1235 } 1236 1237 if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi)) { 1238 LIN_SDT_PROBE0(futex, release_futexes, return); 1239 return; 1240 } 1241 1242 error = copyin(&head->futex_offset, &futex_offset, 1243 sizeof(futex_offset)); 1244 if (error) { 1245 LIN_SDT_PROBE1(futex, release_futexes, copyin_error, error); 1246 LIN_SDT_PROBE0(futex, release_futexes, return); 1247 return; 1248 } 1249 1250 if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip)) { 1251 LIN_SDT_PROBE0(futex, release_futexes, return); 1252 return; 1253 } 1254 1255 while (entry != &head->list) { 1256 rc = fetch_robust_entry(&next_entry, PTRIN(&entry->next), &next_pi); 1257 1258 if (entry != pending) 1259 if (handle_futex_death(em, 1260 (uint32_t *)((caddr_t)entry + futex_offset), pi)) { 1261 LIN_SDT_PROBE0(futex, release_futexes, return); 1262 return; 1263 } 1264 if (rc) { 1265 LIN_SDT_PROBE0(futex, release_futexes, return); 1266 return; 1267 } 1268 1269 entry = next_entry; 1270 pi = next_pi; 1271 1272 if (!--limit) 1273 break; 1274 1275 sched_relinquish(curthread); 1276 } 1277 1278 if (pending) 1279 handle_futex_death(em, (uint32_t *)((caddr_t)pending + futex_offset), pip); 1280 1281 LIN_SDT_PROBE0(futex, release_futexes, return); 1282} 1283