kern_umtx.c revision 151692
1/*- 2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice unmodified, this list of conditions, and the following 11 * disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/kern/kern_umtx.c 151692 2005-10-26 06:55:46Z davidxu $"); 30 31#include <sys/param.h> 32#include <sys/kernel.h> 33#include <sys/limits.h> 34#include <sys/lock.h> 35#include <sys/malloc.h> 36#include <sys/mutex.h> 37#include <sys/proc.h> 38#include <sys/sysent.h> 39#include <sys/systm.h> 40#include <sys/sysproto.h> 41#include <sys/eventhandler.h> 42#include <sys/thr.h> 43#include <sys/umtx.h> 44 45#include <vm/vm.h> 46#include <vm/vm_param.h> 47#include <vm/pmap.h> 48#include <vm/vm_map.h> 49#include <vm/vm_object.h> 50 51#define UMTX_PRIVATE 0 52#define UMTX_SHARED 1 53 54#define UMTX_STATIC_SHARED 55 56struct umtx_key { 57 int type; 58 union { 59 struct { 60 vm_object_t object; 61 long offset; 62 } shared; 63 struct { 64 struct umtx *umtx; 65 long pid; 66 } private; 67 struct { 68 void *ptr; 69 long word; 70 } both; 71 } info; 72}; 73 74struct umtx_q { 75 LIST_ENTRY(umtx_q) uq_next; /* Linked list for the hash. */ 76 struct umtx_key uq_key; /* Umtx key. */ 77 struct thread *uq_thread; /* The thread waits on. */ 78 LIST_ENTRY(umtx_q) uq_rqnext; /* Linked list for requeuing. */ 79 vm_offset_t uq_addr; /* Umtx's virtual address. */ 80}; 81 82LIST_HEAD(umtx_head, umtx_q); 83struct umtxq_chain { 84 struct mtx uc_lock; /* Lock for this chain. */ 85 struct umtx_head uc_queue; /* List of sleep queues. */ 86#define UCF_BUSY 0x01 87#define UCF_WANT 0x02 88 int uc_flags; 89}; 90 91#define GOLDEN_RATIO_PRIME 2654404609U 92#define UMTX_CHAINS 128 93#define UMTX_SHIFTS (__WORD_BIT - 7) 94 95static struct umtxq_chain umtxq_chains[UMTX_CHAINS]; 96static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); 97 98static void umtxq_init_chains(void *); 99static int umtxq_hash(struct umtx_key *key); 100static struct mtx *umtxq_mtx(int chain); 101static void umtxq_lock(struct umtx_key *key); 102static void umtxq_unlock(struct umtx_key *key); 103static void umtxq_busy(struct umtx_key *key); 104static void umtxq_unbusy(struct umtx_key *key); 105static void umtxq_insert(struct umtx_q *uq); 106static void umtxq_remove(struct umtx_q *uq); 107static int umtxq_sleep(struct thread *td, struct umtx_key *key, 108 int prio, const char *wmesg, int timo); 109static int umtxq_count(struct umtx_key *key); 110static int umtxq_signal(struct umtx_key *key, int nr_wakeup); 111#ifdef UMTX_DYNAMIC_SHARED 112static void fork_handler(void *arg, struct proc *p1, struct proc *p2, 113 int flags); 114#endif 115static int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2); 116static int umtx_key_get(struct thread *td, struct umtx *umtx, 117 struct umtx_key *key); 118static void umtx_key_release(struct umtx_key *key); 119 120SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_init_chains, NULL); 121 122struct umtx_q * 123umtxq_alloc(void) 124{ 125 return (malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK)); 126} 127 128void 129umtxq_free(struct umtx_q *uq) 130{ 131 free(uq, M_UMTX); 132} 133 134static void 135umtxq_init_chains(void *arg __unused) 136{ 137 int i; 138 139 for (i = 0; i < UMTX_CHAINS; ++i) { 140 mtx_init(&umtxq_chains[i].uc_lock, "umtxq_lock", NULL, 141 MTX_DEF | MTX_DUPOK); 142 LIST_INIT(&umtxq_chains[i].uc_queue); 143 umtxq_chains[i].uc_flags = 0; 144 } 145#ifdef UMTX_DYNAMIC_SHARED 146 EVENTHANDLER_REGISTER(process_fork, fork_handler, 0, 10000); 147#endif 148} 149 150static inline int 151umtxq_hash(struct umtx_key *key) 152{ 153 unsigned n = (uintptr_t)key->info.both.ptr + key->info.both.word; 154 return (((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS); 155} 156 157static inline int 158umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) 159{ 160 return (k1->type == k2->type && 161 k1->info.both.ptr == k2->info.both.ptr && 162 k1->info.both.word == k2->info.both.word); 163} 164 165static inline struct mtx * 166umtxq_mtx(int chain) 167{ 168 return (&umtxq_chains[chain].uc_lock); 169} 170 171static inline void 172umtxq_busy(struct umtx_key *key) 173{ 174 int chain = umtxq_hash(key); 175 176 mtx_assert(umtxq_mtx(chain), MA_OWNED); 177 while (umtxq_chains[chain].uc_flags & UCF_BUSY) { 178 umtxq_chains[chain].uc_flags |= UCF_WANT; 179 msleep(&umtxq_chains[chain], umtxq_mtx(chain), 180 curthread->td_priority, "umtxq_busy", 0); 181 } 182 umtxq_chains[chain].uc_flags |= UCF_BUSY; 183} 184 185static inline void 186umtxq_unbusy(struct umtx_key *key) 187{ 188 int chain = umtxq_hash(key); 189 190 mtx_assert(umtxq_mtx(chain), MA_OWNED); 191 KASSERT(umtxq_chains[chain].uc_flags & UCF_BUSY, ("not busy")); 192 umtxq_chains[chain].uc_flags &= ~UCF_BUSY; 193 if (umtxq_chains[chain].uc_flags & UCF_WANT) { 194 umtxq_chains[chain].uc_flags &= ~UCF_WANT; 195 wakeup(&umtxq_chains[chain]); 196 } 197} 198 199static inline void 200umtxq_lock(struct umtx_key *key) 201{ 202 int chain = umtxq_hash(key); 203 mtx_lock(umtxq_mtx(chain)); 204} 205 206static inline void 207umtxq_unlock(struct umtx_key *key) 208{ 209 int chain = umtxq_hash(key); 210 mtx_unlock(umtxq_mtx(chain)); 211} 212 213/* 214 * Insert a thread onto the umtx queue. 215 */ 216static inline void 217umtxq_insert(struct umtx_q *uq) 218{ 219 struct umtx_head *head; 220 int chain = umtxq_hash(&uq->uq_key); 221 222 mtx_assert(umtxq_mtx(chain), MA_OWNED); 223 head = &umtxq_chains[chain].uc_queue; 224 LIST_INSERT_HEAD(head, uq, uq_next); 225 mtx_lock_spin(&sched_lock); 226 uq->uq_thread->td_flags |= TDF_UMTXQ; 227 mtx_unlock_spin(&sched_lock); 228} 229 230/* 231 * Remove thread from the umtx queue. 232 */ 233static inline void 234umtxq_remove(struct umtx_q *uq) 235{ 236 mtx_assert(umtxq_mtx(umtxq_hash(&uq->uq_key)), MA_OWNED); 237 if (uq->uq_thread->td_flags & TDF_UMTXQ) { 238 LIST_REMOVE(uq, uq_next); 239 /* turning off TDF_UMTXQ should be the last thing. */ 240 mtx_lock_spin(&sched_lock); 241 uq->uq_thread->td_flags &= ~TDF_UMTXQ; 242 mtx_unlock_spin(&sched_lock); 243 } 244} 245 246static int 247umtxq_count(struct umtx_key *key) 248{ 249 struct umtx_q *uq; 250 struct umtx_head *head; 251 int chain, count = 0; 252 253 chain = umtxq_hash(key); 254 mtx_assert(umtxq_mtx(chain), MA_OWNED); 255 head = &umtxq_chains[chain].uc_queue; 256 LIST_FOREACH(uq, head, uq_next) { 257 if (umtx_key_match(&uq->uq_key, key)) { 258 if (++count > 1) 259 break; 260 } 261 } 262 return (count); 263} 264 265static int 266umtxq_signal(struct umtx_key *key, int n_wake) 267{ 268 struct umtx_q *uq, *next; 269 struct umtx_head *head; 270 struct thread *blocked = NULL; 271 int chain, ret; 272 273 ret = 0; 274 chain = umtxq_hash(key); 275 mtx_assert(umtxq_mtx(chain), MA_OWNED); 276 head = &umtxq_chains[chain].uc_queue; 277 for (uq = LIST_FIRST(head); uq; uq = next) { 278 next = LIST_NEXT(uq, uq_next); 279 if (umtx_key_match(&uq->uq_key, key)) { 280 blocked = uq->uq_thread; 281 umtxq_remove(uq); 282 wakeup(blocked); 283 if (++ret >= n_wake) 284 break; 285 } 286 } 287 return (ret); 288} 289 290static inline int 291umtxq_sleep(struct thread *td, struct umtx_key *key, int priority, 292 const char *wmesg, int timo) 293{ 294 int chain = umtxq_hash(key); 295 int error = msleep(td, umtxq_mtx(chain), priority, wmesg, timo); 296 if (error == EWOULDBLOCK) 297 error = ETIMEDOUT; 298 return (error); 299} 300 301static int 302umtx_key_get(struct thread *td, struct umtx *umtx, struct umtx_key *key) 303{ 304#if defined(UMTX_DYNAMIC_SHARED) || defined(UMTX_STATIC_SHARED) 305 vm_map_t map; 306 vm_map_entry_t entry; 307 vm_pindex_t pindex; 308 vm_prot_t prot; 309 boolean_t wired; 310 311 map = &td->td_proc->p_vmspace->vm_map; 312 if (vm_map_lookup(&map, (vm_offset_t)umtx, VM_PROT_WRITE, 313 &entry, &key->info.shared.object, &pindex, &prot, 314 &wired) != KERN_SUCCESS) { 315 return EFAULT; 316 } 317#endif 318 319#if defined(UMTX_DYNAMIC_SHARED) 320 key->type = UMTX_SHARED; 321 key->info.shared.offset = entry->offset + entry->start - 322 (vm_offset_t)umtx; 323 /* 324 * Add object reference, if we don't do this, a buggy application 325 * deallocates the object, the object will be reused by other 326 * applications, then unlock will wake wrong thread. 327 */ 328 vm_object_reference(key->info.shared.object); 329 vm_map_lookup_done(map, entry); 330#elif defined(UMTX_STATIC_SHARED) 331 if (VM_INHERIT_SHARE == entry->inheritance) { 332 key->type = UMTX_SHARED; 333 key->info.shared.offset = entry->offset + entry->start - 334 (vm_offset_t)umtx; 335 vm_object_reference(key->info.shared.object); 336 } else { 337 key->type = UMTX_PRIVATE; 338 key->info.private.umtx = umtx; 339 key->info.private.pid = td->td_proc->p_pid; 340 } 341 vm_map_lookup_done(map, entry); 342#else 343 key->type = UMTX_PRIVATE; 344 key->info.private.umtx = umtx; 345 key->info.private.pid = td->td_proc->p_pid; 346#endif 347 return (0); 348} 349 350static inline void 351umtx_key_release(struct umtx_key *key) 352{ 353 if (key->type == UMTX_SHARED) 354 vm_object_deallocate(key->info.shared.object); 355} 356 357static inline int 358umtxq_queue_me(struct thread *td, struct umtx *umtx, struct umtx_q *uq) 359{ 360 int error; 361 362 if ((error = umtx_key_get(td, umtx, &uq->uq_key)) != 0) 363 return (error); 364 365 uq->uq_addr = (vm_offset_t)umtx; 366 uq->uq_thread = td; 367 umtxq_lock(&uq->uq_key); 368 /* hmm, for condition variable, we don't need busy flag. */ 369 umtxq_busy(&uq->uq_key); 370 umtxq_insert(uq); 371 umtxq_unbusy(&uq->uq_key); 372 umtxq_unlock(&uq->uq_key); 373 return (0); 374} 375 376#if defined(UMTX_DYNAMIC_SHARED) 377static void 378fork_handler(void *arg, struct proc *p1, struct proc *p2, int flags) 379{ 380 vm_map_t map; 381 vm_map_entry_t entry; 382 vm_object_t object; 383 vm_pindex_t pindex; 384 vm_prot_t prot; 385 boolean_t wired; 386 struct umtx_key key; 387 LIST_HEAD(, umtx_q) workq; 388 struct umtx_q *uq; 389 struct thread *td; 390 int onq; 391 392 LIST_INIT(&workq); 393 394 /* Collect threads waiting on umtxq */ 395 PROC_LOCK(p1); 396 FOREACH_THREAD_IN_PROC(p1, td) { 397 if (td->td_flags & TDF_UMTXQ) { 398 uq = td->td_umtxq; 399 if (uq) 400 LIST_INSERT_HEAD(&workq, uq, uq_rqnext); 401 } 402 } 403 PROC_UNLOCK(p1); 404 405 LIST_FOREACH(uq, &workq, uq_rqnext) { 406 map = &p1->p_vmspace->vm_map; 407 if (vm_map_lookup(&map, uq->uq_addr, VM_PROT_WRITE, 408 &entry, &object, &pindex, &prot, &wired) != KERN_SUCCESS) { 409 continue; 410 } 411 key.type = UMTX_SHARED; 412 key.info.shared.object = object; 413 key.info.shared.offset = entry->offset + entry->start - 414 uq->uq_addr; 415 if (umtx_key_match(&key, &uq->uq_key)) { 416 vm_map_lookup_done(map, entry); 417 continue; 418 } 419 420 umtxq_lock(&uq->uq_key); 421 umtxq_busy(&uq->uq_key); 422 if (uq->uq_thread->td_flags & TDF_UMTXQ) { 423 umtxq_remove(uq); 424 onq = 1; 425 } else 426 onq = 0; 427 umtxq_unbusy(&uq->uq_key); 428 umtxq_unlock(&uq->uq_key); 429 if (onq) { 430 vm_object_deallocate(uq->uq_key.info.shared.object); 431 uq->uq_key = key; 432 umtxq_lock(&uq->uq_key); 433 umtxq_busy(&uq->uq_key); 434 umtxq_insert(uq); 435 umtxq_unbusy(&uq->uq_key); 436 umtxq_unlock(&uq->uq_key); 437 vm_object_reference(uq->uq_key.info.shared.object); 438 } 439 vm_map_lookup_done(map, entry); 440 } 441} 442#endif 443 444static int 445_do_lock(struct thread *td, struct umtx *umtx, long id, int timo) 446{ 447 struct umtx_q *uq; 448 intptr_t owner; 449 intptr_t old; 450 int error = 0; 451 452 uq = td->td_umtxq; 453 /* 454 * Care must be exercised when dealing with umtx structure. It 455 * can fault on any access. 456 */ 457 458 for (;;) { 459 /* 460 * Try the uncontested case. This should be done in userland. 461 */ 462 owner = casuptr((intptr_t *)&umtx->u_owner, 463 UMTX_UNOWNED, id); 464 465 /* The acquire succeeded. */ 466 if (owner == UMTX_UNOWNED) 467 return (0); 468 469 /* The address was invalid. */ 470 if (owner == -1) 471 return (EFAULT); 472 473 /* If no one owns it but it is contested try to acquire it. */ 474 if (owner == UMTX_CONTESTED) { 475 owner = casuptr((intptr_t *)&umtx->u_owner, 476 UMTX_CONTESTED, id | UMTX_CONTESTED); 477 478 if (owner == UMTX_CONTESTED) 479 return (0); 480 481 /* The address was invalid. */ 482 if (owner == -1) 483 return (EFAULT); 484 485 /* If this failed the lock has changed, restart. */ 486 continue; 487 } 488 489 /* 490 * If we caught a signal, we have retried and now 491 * exit immediately. 492 */ 493 if (error || (error = umtxq_queue_me(td, umtx, uq)) != 0) 494 return (error); 495 496 /* 497 * Set the contested bit so that a release in user space 498 * knows to use the system call for unlock. If this fails 499 * either some one else has acquired the lock or it has been 500 * released. 501 */ 502 old = casuptr((intptr_t *)&umtx->u_owner, owner, 503 owner | UMTX_CONTESTED); 504 505 /* The address was invalid. */ 506 if (old == -1) { 507 umtxq_lock(&uq->uq_key); 508 umtxq_busy(&uq->uq_key); 509 umtxq_remove(uq); 510 umtxq_unbusy(&uq->uq_key); 511 umtxq_unlock(&uq->uq_key); 512 umtx_key_release(&uq->uq_key); 513 return (EFAULT); 514 } 515 516 /* 517 * We set the contested bit, sleep. Otherwise the lock changed 518 * and we need to retry or we lost a race to the thread 519 * unlocking the umtx. 520 */ 521 umtxq_lock(&uq->uq_key); 522 if (old == owner && (td->td_flags & TDF_UMTXQ)) { 523 error = umtxq_sleep(td, &uq->uq_key, 524 td->td_priority | PCATCH, 525 "umtx", timo); 526 } 527 umtxq_busy(&uq->uq_key); 528 umtxq_remove(uq); 529 umtxq_unbusy(&uq->uq_key); 530 umtxq_unlock(&uq->uq_key); 531 umtx_key_release(&uq->uq_key); 532 } 533 534 return (0); 535} 536 537static int 538do_lock(struct thread *td, struct umtx *umtx, long id, 539 struct timespec *timeout) 540{ 541 struct timespec ts, ts2, ts3; 542 struct timeval tv; 543 int error; 544 545 if (timeout == NULL) { 546 error = _do_lock(td, umtx, id, 0); 547 } else { 548 getnanouptime(&ts); 549 timespecadd(&ts, timeout); 550 TIMESPEC_TO_TIMEVAL(&tv, timeout); 551 for (;;) { 552 error = _do_lock(td, umtx, id, tvtohz(&tv)); 553 if (error != ETIMEDOUT) 554 break; 555 getnanouptime(&ts2); 556 if (timespeccmp(&ts2, &ts, >=)) { 557 error = ETIMEDOUT; 558 break; 559 } 560 ts3 = ts; 561 timespecsub(&ts3, &ts2); 562 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 563 } 564 } 565 /* 566 * This lets userland back off critical region if needed. 567 */ 568 if (error == ERESTART) 569 error = EINTR; 570 return (error); 571} 572 573static int 574do_unlock(struct thread *td, struct umtx *umtx, long id) 575{ 576 struct umtx_key key; 577 intptr_t owner; 578 intptr_t old; 579 int error; 580 int count; 581 582 /* 583 * Make sure we own this mtx. 584 * 585 * XXX Need a {fu,su}ptr this is not correct on arch where 586 * sizeof(intptr_t) != sizeof(long). 587 */ 588 if ((owner = fuword(&umtx->u_owner)) == -1) 589 return (EFAULT); 590 591 if ((owner & ~UMTX_CONTESTED) != id) 592 return (EPERM); 593 594 /* We should only ever be in here for contested locks */ 595 if ((owner & UMTX_CONTESTED) == 0) 596 return (EINVAL); 597 598 if ((error = umtx_key_get(td, umtx, &key)) != 0) 599 return (error); 600 601 umtxq_lock(&key); 602 umtxq_busy(&key); 603 count = umtxq_count(&key); 604 umtxq_unlock(&key); 605 606 /* 607 * When unlocking the umtx, it must be marked as unowned if 608 * there is zero or one thread only waiting for it. 609 * Otherwise, it must be marked as contested. 610 */ 611 old = casuptr((intptr_t *)&umtx->u_owner, owner, 612 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED); 613 umtxq_lock(&key); 614 umtxq_signal(&key, 0); 615 umtxq_unbusy(&key); 616 umtxq_unlock(&key); 617 umtx_key_release(&key); 618 if (old == -1) 619 return (EFAULT); 620 if (old != owner) 621 return (EINVAL); 622 return (0); 623} 624 625static int 626do_wait(struct thread *td, struct umtx *umtx, long id, struct timespec *timeout) 627{ 628 struct umtx_q *uq; 629 struct timespec ts, ts2, ts3; 630 struct timeval tv; 631 long tmp; 632 int error = 0; 633 634 uq = td->td_umtxq; 635 if ((error = umtxq_queue_me(td, umtx, uq)) != 0) 636 return (error); 637 tmp = fuword(&umtx->u_owner); 638 if (tmp != id) { 639 umtxq_lock(&uq->uq_key); 640 umtxq_remove(uq); 641 umtxq_unlock(&uq->uq_key); 642 } else if (timeout == NULL) { 643 umtxq_lock(&uq->uq_key); 644 if (td->td_flags & TDF_UMTXQ) 645 error = umtxq_sleep(td, &uq->uq_key, 646 td->td_priority | PCATCH, "ucond", 0); 647 if (!(td->td_flags & TDF_UMTXQ)) 648 error = 0; 649 else 650 umtxq_remove(uq); 651 umtxq_unlock(&uq->uq_key); 652 } else { 653 getnanouptime(&ts); 654 timespecadd(&ts, timeout); 655 TIMESPEC_TO_TIMEVAL(&tv, timeout); 656 for (;;) { 657 umtxq_lock(&uq->uq_key); 658 if (td->td_flags & TDF_UMTXQ) { 659 error = umtxq_sleep(td, &uq->uq_key, 660 td->td_priority | PCATCH, 661 "ucond", tvtohz(&tv)); 662 } 663 if (!(td->td_flags & TDF_UMTXQ)) { 664 umtxq_unlock(&uq->uq_key); 665 goto out; 666 } 667 umtxq_unlock(&uq->uq_key); 668 if (error != ETIMEDOUT) 669 break; 670 getnanouptime(&ts2); 671 if (timespeccmp(&ts2, &ts, >=)) { 672 error = ETIMEDOUT; 673 break; 674 } 675 ts3 = ts; 676 timespecsub(&ts3, &ts2); 677 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 678 } 679 umtxq_lock(&uq->uq_key); 680 umtxq_remove(uq); 681 umtxq_unlock(&uq->uq_key); 682 } 683out: 684 umtx_key_release(&uq->uq_key); 685 if (error == ERESTART) 686 error = EINTR; 687 return (error); 688} 689 690int 691kern_umtx_wake(struct thread *td, void *uaddr, int n_wake) 692{ 693 struct umtx_key key; 694 int ret; 695 696 if ((ret = umtx_key_get(td, uaddr, &key)) != 0) 697 return (ret); 698 umtxq_lock(&key); 699 ret = umtxq_signal(&key, n_wake); 700 umtxq_unlock(&key); 701 umtx_key_release(&key); 702 return (0); 703} 704 705int 706_umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 707 /* struct umtx *umtx */ 708{ 709 return _do_lock(td, uap->umtx, td->td_tid, 0); 710} 711 712int 713_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 714 /* struct umtx *umtx */ 715{ 716 return do_unlock(td, uap->umtx, td->td_tid); 717} 718 719int 720_umtx_op(struct thread *td, struct _umtx_op_args *uap) 721{ 722 struct timespec timeout; 723 struct timespec *ts; 724 int error; 725 726 switch(uap->op) { 727 case UMTX_OP_LOCK: 728 /* Allow a null timespec (wait forever). */ 729 if (uap->uaddr2 == NULL) 730 ts = NULL; 731 else { 732 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 733 if (error != 0) 734 break; 735 if (timeout.tv_nsec >= 1000000000 || 736 timeout.tv_nsec < 0) { 737 error = EINVAL; 738 break; 739 } 740 ts = &timeout; 741 } 742 error = do_lock(td, uap->umtx, uap->id, ts); 743 break; 744 case UMTX_OP_UNLOCK: 745 error = do_unlock(td, uap->umtx, uap->id); 746 break; 747 case UMTX_OP_WAIT: 748 /* Allow a null timespec (wait forever). */ 749 if (uap->uaddr2 == NULL) 750 ts = NULL; 751 else { 752 error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 753 if (error != 0) 754 break; 755 if (timeout.tv_nsec >= 1000000000 || 756 timeout.tv_nsec < 0) { 757 error = EINVAL; 758 break; 759 } 760 ts = &timeout; 761 } 762 error = do_wait(td, uap->umtx, uap->id, ts); 763 break; 764 case UMTX_OP_WAKE: 765 error = kern_umtx_wake(td, uap->umtx, uap->id); 766 break; 767 default: 768 error = EINVAL; 769 break; 770 } 771 return (error); 772} 773