1/* $NetBSD: kern_lwp.c,v 1.166 2012/02/11 23:16:17 martin Exp $ */ 2 3/*- 4 * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams, and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Overview 34 * 35 * Lightweight processes (LWPs) are the basic unit or thread of 36 * execution within the kernel. The core state of an LWP is described 37 * by "struct lwp", also known as lwp_t. 38 * 39 * Each LWP is contained within a process (described by "struct proc"), 40 * Every process contains at least one LWP, but may contain more. The 41 * process describes attributes shared among all of its LWPs such as a 42 * private address space, global execution state (stopped, active, 43 * zombie, ...), signal disposition and so on. On a multiprocessor 44 * machine, multiple LWPs be executing concurrently in the kernel. 45 * 46 * Execution states 47 * 48 * At any given time, an LWP has overall state that is described by 49 * lwp::l_stat. The states are broken into two sets below. The first 50 * set is guaranteed to represent the absolute, current state of the 51 * LWP: 52 * 53 * LSONPROC 54 * 55 * On processor: the LWP is executing on a CPU, either in the 56 * kernel or in user space. 57 * 58 * LSRUN 59 * 60 * Runnable: the LWP is parked on a run queue, and may soon be 61 * chosen to run by an idle processor, or by a processor that 62 * has been asked to preempt a currently runnning but lower 63 * priority LWP. 64 * 65 * LSIDL 66 * 67 * Idle: the LWP has been created but has not yet executed, 68 * or it has ceased executing a unit of work and is waiting 69 * to be started again. 70 * 71 * LSSUSPENDED: 72 * 73 * Suspended: the LWP has had its execution suspended by 74 * another LWP in the same process using the _lwp_suspend() 75 * system call. User-level LWPs also enter the suspended 76 * state when the system is shutting down. 77 * 78 * The second set represent a "statement of intent" on behalf of the 79 * LWP. The LWP may in fact be executing on a processor, may be 80 * sleeping or idle. It is expected to take the necessary action to 81 * stop executing or become "running" again within a short timeframe. 82 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running. 83 * Importantly, it indicates that its state is tied to a CPU. 84 * 85 * LSZOMB: 86 * 87 * Dead or dying: the LWP has released most of its resources 88 * and is about to switch away into oblivion, or has already 89 * switched away. When it switches away, its few remaining 90 * resources can be collected. 91 * 92 * LSSLEEP: 93 * 94 * Sleeping: the LWP has entered itself onto a sleep queue, and 95 * has switched away or will switch away shortly to allow other 96 * LWPs to run on the CPU. 97 * 98 * LSSTOP: 99 * 100 * Stopped: the LWP has been stopped as a result of a job 101 * control signal, or as a result of the ptrace() interface. 102 * 103 * Stopped LWPs may run briefly within the kernel to handle 104 * signals that they receive, but will not return to user space 105 * until their process' state is changed away from stopped. 106 * 107 * Single LWPs within a process can not be set stopped 108 * selectively: all actions that can stop or continue LWPs 109 * occur at the process level. 110 * 111 * State transitions 112 * 113 * Note that the LSSTOP state may only be set when returning to 114 * user space in userret(), or when sleeping interruptably. The 115 * LSSUSPENDED state may only be set in userret(). Before setting 116 * those states, we try to ensure that the LWPs will release all 117 * locks that they hold, and at a minimum try to ensure that the 118 * LWP can be set runnable again by a signal. 119 * 120 * LWPs may transition states in the following ways: 121 * 122 * RUN -------> ONPROC ONPROC -----> RUN 123 * > SLEEP 124 * > STOPPED 125 * > SUSPENDED 126 * > ZOMB 127 * > IDL (special cases) 128 * 129 * STOPPED ---> RUN SUSPENDED --> RUN 130 * > SLEEP 131 * 132 * SLEEP -----> ONPROC IDL --------> RUN 133 * > RUN > SUSPENDED 134 * > STOPPED > STOPPED 135 * > ONPROC (special cases) 136 * 137 * Some state transitions are only possible with kernel threads (eg 138 * ONPROC -> IDL) and happen under tightly controlled circumstances 139 * free of unwanted side effects. 140 * 141 * Migration 142 * 143 * Migration of threads from one CPU to another could be performed 144 * internally by the scheduler via sched_takecpu() or sched_catchlwp() 145 * functions. The universal lwp_migrate() function should be used for 146 * any other cases. Subsystems in the kernel must be aware that CPU 147 * of LWP may change, while it is not locked. 148 * 149 * Locking 150 * 151 * The majority of fields in 'struct lwp' are covered by a single, 152 * general spin lock pointed to by lwp::l_mutex. The locks covering 153 * each field are documented in sys/lwp.h. 154 * 155 * State transitions must be made with the LWP's general lock held, 156 * and may cause the LWP's lock pointer to change. Manipulation of 157 * the general lock is not performed directly, but through calls to 158 * lwp_lock(), lwp_unlock() and others. It should be noted that the 159 * adaptive locks are not allowed to be released while the LWP's lock 160 * is being held (unlike for other spin-locks). 161 * 162 * States and their associated locks: 163 * 164 * LSONPROC, LSZOMB: 165 * 166 * Always covered by spc_lwplock, which protects running LWPs. 167 * This is a per-CPU lock and matches lwp::l_cpu. 168 * 169 * LSIDL, LSRUN: 170 * 171 * Always covered by spc_mutex, which protects the run queues. 172 * This is a per-CPU lock and matches lwp::l_cpu. 173 * 174 * LSSLEEP: 175 * 176 * Covered by a lock associated with the sleep queue that the 177 * LWP resides on. Matches lwp::l_sleepq::sq_mutex. 178 * 179 * LSSTOP, LSSUSPENDED: 180 * 181 * If the LWP was previously sleeping (l_wchan != NULL), then 182 * l_mutex references the sleep queue lock. If the LWP was 183 * runnable or on the CPU when halted, or has been removed from 184 * the sleep queue since halted, then the lock is spc_lwplock. 185 * 186 * The lock order is as follows: 187 * 188 * spc::spc_lwplock -> 189 * sleeptab::st_mutex -> 190 * tschain_t::tc_mutex -> 191 * spc::spc_mutex 192 * 193 * Each process has an scheduler state lock (proc::p_lock), and a 194 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and 195 * so on. When an LWP is to be entered into or removed from one of the 196 * following states, p_lock must be held and the process wide counters 197 * adjusted: 198 * 199 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED 200 * 201 * (But not always for kernel threads. There are some special cases 202 * as mentioned above. See kern_softint.c.) 203 * 204 * Note that an LWP is considered running or likely to run soon if in 205 * one of the following states. This affects the value of p_nrlwps: 206 * 207 * LSRUN, LSONPROC, LSSLEEP 208 * 209 * p_lock does not need to be held when transitioning among these 210 * three states, hence p_lock is rarely taken for state transitions. 211 */ 212 213#include <sys/cdefs.h> 214__KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.166 2012/02/11 23:16:17 martin Exp $"); 215 216#include "opt_ddb.h" 217#include "opt_lockdebug.h" 218#include "opt_sa.h" 219#include "opt_dtrace.h" 220 221#define _LWP_API_PRIVATE 222 223#include <sys/param.h> 224#include <sys/systm.h> 225#include <sys/cpu.h> 226#include <sys/pool.h> 227#include <sys/proc.h> 228#include <sys/sa.h> 229#include <sys/savar.h> 230#include <sys/syscallargs.h> 231#include <sys/syscall_stats.h> 232#include <sys/kauth.h> 233#include <sys/pserialize.h> 234#include <sys/sleepq.h> 235#include <sys/lockdebug.h> 236#include <sys/kmem.h> 237#include <sys/pset.h> 238#include <sys/intr.h> 239#include <sys/lwpctl.h> 240#include <sys/atomic.h> 241#include <sys/filedesc.h> 242#include <sys/dtrace_bsd.h> 243#include <sys/sdt.h> 244#include <sys/xcall.h> 245 246#include <uvm/uvm_extern.h> 247#include <uvm/uvm_object.h> 248 249static pool_cache_t lwp_cache __read_mostly; 250struct lwplist alllwp __cacheline_aligned; 251 252static void lwp_dtor(void *, void *); 253 254/* DTrace proc provider probes */ 255SDT_PROBE_DEFINE(proc,,,lwp_create, 256 "struct lwp *", NULL, 257 NULL, NULL, NULL, NULL, 258 NULL, NULL, NULL, NULL); 259SDT_PROBE_DEFINE(proc,,,lwp_start, 260 "struct lwp *", NULL, 261 NULL, NULL, NULL, NULL, 262 NULL, NULL, NULL, NULL); 263SDT_PROBE_DEFINE(proc,,,lwp_exit, 264 "struct lwp *", NULL, 265 NULL, NULL, NULL, NULL, 266 NULL, NULL, NULL, NULL); 267 268struct turnstile turnstile0; 269struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = { 270#ifdef LWP0_CPU_INFO 271 .l_cpu = LWP0_CPU_INFO, 272#endif 273#ifdef LWP0_MD_INITIALIZER 274 .l_md = LWP0_MD_INITIALIZER, 275#endif 276 .l_proc = &proc0, 277 .l_lid = 1, 278 .l_flag = LW_SYSTEM, 279 .l_stat = LSONPROC, 280 .l_ts = &turnstile0, 281 .l_syncobj = &sched_syncobj, 282 .l_refcnt = 1, 283 .l_priority = PRI_USER + NPRI_USER - 1, 284 .l_inheritedprio = -1, 285 .l_class = SCHED_OTHER, 286 .l_psid = PS_NONE, 287 .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders), 288 .l_name = __UNCONST("swapper"), 289 .l_fd = &filedesc0, 290}; 291 292void 293lwpinit(void) 294{ 295 296 LIST_INIT(&alllwp); 297 lwpinit_specificdata(); 298 lwp_sys_init(); 299 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0, 300 "lwppl", NULL, IPL_NONE, NULL, lwp_dtor, NULL); 301} 302 303void 304lwp0_init(void) 305{ 306 struct lwp *l = &lwp0; 307 308 KASSERT((void *)uvm_lwp_getuarea(l) != NULL); 309 KASSERT(l->l_lid == proc0.p_nlwpid); 310 311 LIST_INSERT_HEAD(&alllwp, l, l_list); 312 313 callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE); 314 callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l); 315 cv_init(&l->l_sigcv, "sigwait"); 316 317 kauth_cred_hold(proc0.p_cred); 318 l->l_cred = proc0.p_cred; 319 320 kdtrace_thread_ctor(NULL, l); 321 lwp_initspecific(l); 322 323 SYSCALL_TIME_LWP_INIT(l); 324} 325 326static void 327lwp_dtor(void *arg, void *obj) 328{ 329 lwp_t *l = obj; 330 uint64_t where; 331 (void)l; 332 333 /* 334 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu() 335 * calls will exit before memory of LWP is returned to the pool, where 336 * KVA of LWP structure might be freed and re-used for other purposes. 337 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu() 338 * callers, therefore cross-call to all CPUs will do the job. Also, 339 * the value of l->l_cpu must be still valid at this point. 340 */ 341 KASSERT(l->l_cpu != NULL); 342 where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL); 343 xc_wait(where); 344} 345 346/* 347 * Set an suspended. 348 * 349 * Must be called with p_lock held, and the LWP locked. Will unlock the 350 * LWP before return. 351 */ 352int 353lwp_suspend(struct lwp *curl, struct lwp *t) 354{ 355 int error; 356 357 KASSERT(mutex_owned(t->l_proc->p_lock)); 358 KASSERT(lwp_locked(t, NULL)); 359 360 KASSERT(curl != t || curl->l_stat == LSONPROC); 361 362 /* 363 * If the current LWP has been told to exit, we must not suspend anyone 364 * else or deadlock could occur. We won't return to userspace. 365 */ 366 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) { 367 lwp_unlock(t); 368 return (EDEADLK); 369 } 370 371 error = 0; 372 373 switch (t->l_stat) { 374 case LSRUN: 375 case LSONPROC: 376 t->l_flag |= LW_WSUSPEND; 377 lwp_need_userret(t); 378 lwp_unlock(t); 379 break; 380 381 case LSSLEEP: 382 t->l_flag |= LW_WSUSPEND; 383 384 /* 385 * Kick the LWP and try to get it to the kernel boundary 386 * so that it will release any locks that it holds. 387 * setrunnable() will release the lock. 388 */ 389 if ((t->l_flag & LW_SINTR) != 0) 390 setrunnable(t); 391 else 392 lwp_unlock(t); 393 break; 394 395 case LSSUSPENDED: 396 lwp_unlock(t); 397 break; 398 399 case LSSTOP: 400 t->l_flag |= LW_WSUSPEND; 401 setrunnable(t); 402 break; 403 404 case LSIDL: 405 case LSZOMB: 406 error = EINTR; /* It's what Solaris does..... */ 407 lwp_unlock(t); 408 break; 409 } 410 411 return (error); 412} 413 414/* 415 * Restart a suspended LWP. 416 * 417 * Must be called with p_lock held, and the LWP locked. Will unlock the 418 * LWP before return. 419 */ 420void 421lwp_continue(struct lwp *l) 422{ 423 424 KASSERT(mutex_owned(l->l_proc->p_lock)); 425 KASSERT(lwp_locked(l, NULL)); 426 427 /* If rebooting or not suspended, then just bail out. */ 428 if ((l->l_flag & LW_WREBOOT) != 0) { 429 lwp_unlock(l); 430 return; 431 } 432 433 l->l_flag &= ~LW_WSUSPEND; 434 435 if (l->l_stat != LSSUSPENDED) { 436 lwp_unlock(l); 437 return; 438 } 439 440 /* setrunnable() will release the lock. */ 441 setrunnable(l); 442} 443 444/* 445 * Restart a stopped LWP. 446 * 447 * Must be called with p_lock held, and the LWP NOT locked. Will unlock the 448 * LWP before return. 449 */ 450void 451lwp_unstop(struct lwp *l) 452{ 453 struct proc *p = l->l_proc; 454 455 KASSERT(mutex_owned(proc_lock)); 456 KASSERT(mutex_owned(p->p_lock)); 457 458 lwp_lock(l); 459 460 /* If not stopped, then just bail out. */ 461 if (l->l_stat != LSSTOP) { 462 lwp_unlock(l); 463 return; 464 } 465 466 p->p_stat = SACTIVE; 467 p->p_sflag &= ~PS_STOPPING; 468 469 if (!p->p_waited) 470 p->p_pptr->p_nstopchild--; 471 472 if (l->l_wchan == NULL) { 473 /* setrunnable() will release the lock. */ 474 setrunnable(l); 475 } else if (p->p_xstat && (l->l_flag & LW_SINTR) != 0) { 476 /* setrunnable() so we can receive the signal */ 477 setrunnable(l); 478 } else { 479 l->l_stat = LSSLEEP; 480 p->p_nrlwps++; 481 lwp_unlock(l); 482 } 483} 484 485/* 486 * Wait for an LWP within the current process to exit. If 'lid' is 487 * non-zero, we are waiting for a specific LWP. 488 * 489 * Must be called with p->p_lock held. 490 */ 491int 492lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting) 493{ 494 const lwpid_t curlid = l->l_lid; 495 proc_t *p = l->l_proc; 496 lwp_t *l2; 497 int error; 498 499 KASSERT(mutex_owned(p->p_lock)); 500 501 p->p_nlwpwait++; 502 l->l_waitingfor = lid; 503 504 for (;;) { 505 int nfound; 506 507 /* 508 * Avoid a race between exit1() and sigexit(): if the 509 * process is dumping core, then we need to bail out: call 510 * into lwp_userret() where we will be suspended until the 511 * deed is done. 512 */ 513 if ((p->p_sflag & PS_WCORE) != 0) { 514 mutex_exit(p->p_lock); 515 lwp_userret(l); 516 KASSERT(false); 517 } 518 519 /* 520 * First off, drain any detached LWP that is waiting to be 521 * reaped. 522 */ 523 while ((l2 = p->p_zomblwp) != NULL) { 524 p->p_zomblwp = NULL; 525 lwp_free(l2, false, false);/* releases proc mutex */ 526 mutex_enter(p->p_lock); 527 } 528 529 /* 530 * Now look for an LWP to collect. If the whole process is 531 * exiting, count detached LWPs as eligible to be collected, 532 * but don't drain them here. 533 */ 534 nfound = 0; 535 error = 0; 536 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 537 /* 538 * If a specific wait and the target is waiting on 539 * us, then avoid deadlock. This also traps LWPs 540 * that try to wait on themselves. 541 * 542 * Note that this does not handle more complicated 543 * cycles, like: t1 -> t2 -> t3 -> t1. The process 544 * can still be killed so it is not a major problem. 545 */ 546 if (l2->l_lid == lid && l2->l_waitingfor == curlid) { 547 error = EDEADLK; 548 break; 549 } 550 if (l2 == l) 551 continue; 552 if ((l2->l_prflag & LPR_DETACHED) != 0) { 553 nfound += exiting; 554 continue; 555 } 556 if (lid != 0) { 557 if (l2->l_lid != lid) 558 continue; 559 /* 560 * Mark this LWP as the first waiter, if there 561 * is no other. 562 */ 563 if (l2->l_waiter == 0) 564 l2->l_waiter = curlid; 565 } else if (l2->l_waiter != 0) { 566 /* 567 * It already has a waiter - so don't 568 * collect it. If the waiter doesn't 569 * grab it we'll get another chance 570 * later. 571 */ 572 nfound++; 573 continue; 574 } 575 nfound++; 576 577 /* No need to lock the LWP in order to see LSZOMB. */ 578 if (l2->l_stat != LSZOMB) 579 continue; 580 581 /* 582 * We're no longer waiting. Reset the "first waiter" 583 * pointer on the target, in case it was us. 584 */ 585 l->l_waitingfor = 0; 586 l2->l_waiter = 0; 587 p->p_nlwpwait--; 588 if (departed) 589 *departed = l2->l_lid; 590 sched_lwp_collect(l2); 591 592 /* lwp_free() releases the proc lock. */ 593 lwp_free(l2, false, false); 594 mutex_enter(p->p_lock); 595 return 0; 596 } 597 598 if (error != 0) 599 break; 600 if (nfound == 0) { 601 error = ESRCH; 602 break; 603 } 604 605 /* 606 * Note: since the lock will be dropped, need to restart on 607 * wakeup to run all LWPs again, e.g. there may be new LWPs. 608 */ 609 if (exiting) { 610 KASSERT(p->p_nlwps > 1); 611 cv_wait(&p->p_lwpcv, p->p_lock); 612 error = EAGAIN; 613 break; 614 } 615 616 /* 617 * If all other LWPs are waiting for exits or suspends 618 * and the supply of zombies and potential zombies is 619 * exhausted, then we are about to deadlock. 620 * 621 * If the process is exiting (and this LWP is not the one 622 * that is coordinating the exit) then bail out now. 623 */ 624 if ((p->p_sflag & PS_WEXIT) != 0 || 625 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) { 626 error = EDEADLK; 627 break; 628 } 629 630 /* 631 * Sit around and wait for something to happen. We'll be 632 * awoken if any of the conditions examined change: if an 633 * LWP exits, is collected, or is detached. 634 */ 635 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0) 636 break; 637 } 638 639 /* 640 * We didn't find any LWPs to collect, we may have received a 641 * signal, or some other condition has caused us to bail out. 642 * 643 * If waiting on a specific LWP, clear the waiters marker: some 644 * other LWP may want it. Then, kick all the remaining waiters 645 * so that they can re-check for zombies and for deadlock. 646 */ 647 if (lid != 0) { 648 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 649 if (l2->l_lid == lid) { 650 if (l2->l_waiter == curlid) 651 l2->l_waiter = 0; 652 break; 653 } 654 } 655 } 656 p->p_nlwpwait--; 657 l->l_waitingfor = 0; 658 cv_broadcast(&p->p_lwpcv); 659 660 return error; 661} 662 663/* 664 * Create a new LWP within process 'p2', using LWP 'l1' as a template. 665 * The new LWP is created in state LSIDL and must be set running, 666 * suspended, or stopped by the caller. 667 */ 668int 669lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags, 670 void *stack, size_t stacksize, void (*func)(void *), void *arg, 671 lwp_t **rnewlwpp, int sclass) 672{ 673 struct lwp *l2, *isfree; 674 turnstile_t *ts; 675 lwpid_t lid; 676 677 KASSERT(l1 == curlwp || l1->l_proc == &proc0); 678 679 /* 680 * First off, reap any detached LWP waiting to be collected. 681 * We can re-use its LWP structure and turnstile. 682 */ 683 isfree = NULL; 684 if (p2->p_zomblwp != NULL) { 685 mutex_enter(p2->p_lock); 686 if ((isfree = p2->p_zomblwp) != NULL) { 687 p2->p_zomblwp = NULL; 688 lwp_free(isfree, true, false);/* releases proc mutex */ 689 } else 690 mutex_exit(p2->p_lock); 691 } 692 if (isfree == NULL) { 693 l2 = pool_cache_get(lwp_cache, PR_WAITOK); 694 memset(l2, 0, sizeof(*l2)); 695 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK); 696 SLIST_INIT(&l2->l_pi_lenders); 697 } else { 698 l2 = isfree; 699 ts = l2->l_ts; 700 KASSERT(l2->l_inheritedprio == -1); 701 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders)); 702 memset(l2, 0, sizeof(*l2)); 703 l2->l_ts = ts; 704 } 705 706 l2->l_stat = LSIDL; 707 l2->l_proc = p2; 708 l2->l_refcnt = 1; 709 l2->l_class = sclass; 710 711 /* 712 * If vfork(), we want the LWP to run fast and on the same CPU 713 * as its parent, so that it can reuse the VM context and cache 714 * footprint on the local CPU. 715 */ 716 l2->l_kpriority = ((flags & LWP_VFORK) ? true : false); 717 l2->l_kpribase = PRI_KERNEL; 718 l2->l_priority = l1->l_priority; 719 l2->l_inheritedprio = -1; 720 l2->l_flag = 0; 721 l2->l_pflag = LP_MPSAFE; 722 TAILQ_INIT(&l2->l_ld_locks); 723 724 /* 725 * For vfork, borrow parent's lwpctl context if it exists. 726 * This also causes us to return via lwp_userret. 727 */ 728 if (flags & LWP_VFORK && l1->l_lwpctl) { 729 l2->l_lwpctl = l1->l_lwpctl; 730 l2->l_flag |= LW_LWPCTL; 731 } 732 733 /* 734 * If not the first LWP in the process, grab a reference to the 735 * descriptor table. 736 */ 737 l2->l_fd = p2->p_fd; 738 if (p2->p_nlwps != 0) { 739 KASSERT(l1->l_proc == p2); 740 fd_hold(l2); 741 } else { 742 KASSERT(l1->l_proc != p2); 743 } 744 745 if (p2->p_flag & PK_SYSTEM) { 746 /* Mark it as a system LWP. */ 747 l2->l_flag |= LW_SYSTEM; 748 } 749 750 kpreempt_disable(); 751 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex; 752 l2->l_cpu = l1->l_cpu; 753 kpreempt_enable(); 754 755 kdtrace_thread_ctor(NULL, l2); 756 lwp_initspecific(l2); 757 sched_lwp_fork(l1, l2); 758 lwp_update_creds(l2); 759 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE); 760 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2); 761 cv_init(&l2->l_sigcv, "sigwait"); 762 l2->l_syncobj = &sched_syncobj; 763 764 if (rnewlwpp != NULL) 765 *rnewlwpp = l2; 766 767 /* 768 * PCU state needs to be saved before calling uvm_lwp_fork() so that 769 * the MD cpu_lwp_fork() can copy the saved state to the new LWP. 770 */ 771 pcu_save_all(l1); 772 773 uvm_lwp_setuarea(l2, uaddr); 774 uvm_lwp_fork(l1, l2, stack, stacksize, func, 775 (arg != NULL) ? arg : l2); 776 777 if ((flags & LWP_PIDLID) != 0) { 778 lid = proc_alloc_pid(p2); 779 l2->l_pflag |= LP_PIDLID; 780 } else { 781 lid = 0; 782 } 783 784 mutex_enter(p2->p_lock); 785 786 if ((flags & LWP_DETACHED) != 0) { 787 l2->l_prflag = LPR_DETACHED; 788 p2->p_ndlwps++; 789 } else 790 l2->l_prflag = 0; 791 792 l2->l_sigstk = l1->l_sigstk; 793 l2->l_sigmask = l1->l_sigmask; 794 CIRCLEQ_INIT(&l2->l_sigpend.sp_info); 795 sigemptyset(&l2->l_sigpend.sp_set); 796 797 if (lid == 0) { 798 p2->p_nlwpid++; 799 if (p2->p_nlwpid == 0) 800 p2->p_nlwpid++; 801 lid = p2->p_nlwpid; 802 } 803 l2->l_lid = lid; 804 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling); 805 p2->p_nlwps++; 806 p2->p_nrlwps++; 807 808 KASSERT(l2->l_affinity == NULL); 809 810 if ((p2->p_flag & PK_SYSTEM) == 0) { 811 /* Inherit the affinity mask. */ 812 if (l1->l_affinity) { 813 /* 814 * Note that we hold the state lock while inheriting 815 * the affinity to avoid race with sched_setaffinity(). 816 */ 817 lwp_lock(l1); 818 if (l1->l_affinity) { 819 kcpuset_use(l1->l_affinity); 820 l2->l_affinity = l1->l_affinity; 821 } 822 lwp_unlock(l1); 823 } 824 lwp_lock(l2); 825 /* Inherit a processor-set */ 826 l2->l_psid = l1->l_psid; 827 /* Look for a CPU to start */ 828 l2->l_cpu = sched_takecpu(l2); 829 lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex); 830 } 831 mutex_exit(p2->p_lock); 832 833 SDT_PROBE(proc,,,lwp_create, l2, 0,0,0,0); 834 835 mutex_enter(proc_lock); 836 LIST_INSERT_HEAD(&alllwp, l2, l_list); 837 mutex_exit(proc_lock); 838 839 SYSCALL_TIME_LWP_INIT(l2); 840 841 if (p2->p_emul->e_lwp_fork) 842 (*p2->p_emul->e_lwp_fork)(l1, l2); 843 844 return (0); 845} 846 847/* 848 * Called by MD code when a new LWP begins execution. Must be called 849 * with the previous LWP locked (so at splsched), or if there is no 850 * previous LWP, at splsched. 851 */ 852void 853lwp_startup(struct lwp *prev, struct lwp *new) 854{ 855 856 SDT_PROBE(proc,,,lwp_start, new, 0,0,0,0); 857 858 KASSERT(kpreempt_disabled()); 859 if (prev != NULL) { 860 /* 861 * Normalize the count of the spin-mutexes, it was 862 * increased in mi_switch(). Unmark the state of 863 * context switch - it is finished for previous LWP. 864 */ 865 curcpu()->ci_mtx_count++; 866 membar_exit(); 867 prev->l_ctxswtch = 0; 868 } 869 KPREEMPT_DISABLE(new); 870 spl0(); 871 if (__predict_true(new->l_proc->p_vmspace)) 872 pmap_activate(new); 873 874 /* Note trip through cpu_switchto(). */ 875 pserialize_switchpoint(); 876 877 LOCKDEBUG_BARRIER(NULL, 0); 878 KPREEMPT_ENABLE(new); 879 if ((new->l_pflag & LP_MPSAFE) == 0) { 880 KERNEL_LOCK(1, new); 881 } 882} 883 884/* 885 * Exit an LWP. 886 */ 887void 888lwp_exit(struct lwp *l) 889{ 890 struct proc *p = l->l_proc; 891 struct lwp *l2; 892 bool current; 893 894 current = (l == curlwp); 895 896 KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL)); 897 KASSERT(p == curproc); 898 899 SDT_PROBE(proc,,,lwp_exit, l, 0,0,0,0); 900 901 /* 902 * Verify that we hold no locks other than the kernel lock. 903 */ 904 LOCKDEBUG_BARRIER(&kernel_lock, 0); 905 906 /* 907 * If we are the last live LWP in a process, we need to exit the 908 * entire process. We do so with an exit status of zero, because 909 * it's a "controlled" exit, and because that's what Solaris does. 910 * 911 * We are not quite a zombie yet, but for accounting purposes we 912 * must increment the count of zombies here. 913 * 914 * Note: the last LWP's specificdata will be deleted here. 915 */ 916 mutex_enter(p->p_lock); 917 if (p->p_nlwps - p->p_nzlwps == 1) { 918 KASSERT(current == true); 919 /* XXXSMP kernel_lock not held */ 920 exit1(l, 0); 921 /* NOTREACHED */ 922 } 923 p->p_nzlwps++; 924 mutex_exit(p->p_lock); 925 926 if (p->p_emul->e_lwp_exit) 927 (*p->p_emul->e_lwp_exit)(l); 928 929 /* Drop filedesc reference. */ 930 fd_free(); 931 932 /* Delete the specificdata while it's still safe to sleep. */ 933 lwp_finispecific(l); 934 935 /* 936 * Release our cached credentials. 937 */ 938 kauth_cred_free(l->l_cred); 939 callout_destroy(&l->l_timeout_ch); 940 941 /* 942 * Remove the LWP from the global list. 943 * Free its LID from the PID namespace if needed. 944 */ 945 mutex_enter(proc_lock); 946 LIST_REMOVE(l, l_list); 947 if ((l->l_pflag & LP_PIDLID) != 0 && l->l_lid != p->p_pid) { 948 proc_free_pid(l->l_lid); 949 } 950 mutex_exit(proc_lock); 951 952 /* 953 * Get rid of all references to the LWP that others (e.g. procfs) 954 * may have, and mark the LWP as a zombie. If the LWP is detached, 955 * mark it waiting for collection in the proc structure. Note that 956 * before we can do that, we need to free any other dead, deatched 957 * LWP waiting to meet its maker. 958 */ 959 mutex_enter(p->p_lock); 960 lwp_drainrefs(l); 961 962 if ((l->l_prflag & LPR_DETACHED) != 0) { 963 while ((l2 = p->p_zomblwp) != NULL) { 964 p->p_zomblwp = NULL; 965 lwp_free(l2, false, false);/* releases proc mutex */ 966 mutex_enter(p->p_lock); 967 l->l_refcnt++; 968 lwp_drainrefs(l); 969 } 970 p->p_zomblwp = l; 971 } 972 973 /* 974 * If we find a pending signal for the process and we have been 975 * asked to check for signals, then we lose: arrange to have 976 * all other LWPs in the process check for signals. 977 */ 978 if ((l->l_flag & LW_PENDSIG) != 0 && 979 firstsig(&p->p_sigpend.sp_set) != 0) { 980 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 981 lwp_lock(l2); 982 l2->l_flag |= LW_PENDSIG; 983 lwp_unlock(l2); 984 } 985 } 986 987 /* 988 * Release any PCU resources before becoming a zombie. 989 */ 990 pcu_discard_all(l); 991 992 lwp_lock(l); 993 l->l_stat = LSZOMB; 994 if (l->l_name != NULL) { 995 strcpy(l->l_name, "(zombie)"); 996 } 997 lwp_unlock(l); 998 p->p_nrlwps--; 999 cv_broadcast(&p->p_lwpcv); 1000 if (l->l_lwpctl != NULL) 1001 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 1002 mutex_exit(p->p_lock); 1003 1004 /* 1005 * We can no longer block. At this point, lwp_free() may already 1006 * be gunning for us. On a multi-CPU system, we may be off p_lwps. 1007 * 1008 * Free MD LWP resources. 1009 */ 1010 cpu_lwp_free(l, 0); 1011 1012 if (current) { 1013 pmap_deactivate(l); 1014 1015 /* 1016 * Release the kernel lock, and switch away into 1017 * oblivion. 1018 */ 1019#ifdef notyet 1020 /* XXXSMP hold in lwp_userret() */ 1021 KERNEL_UNLOCK_LAST(l); 1022#else 1023 KERNEL_UNLOCK_ALL(l, NULL); 1024#endif 1025 lwp_exit_switchaway(l); 1026 } 1027} 1028 1029/* 1030 * Free a dead LWP's remaining resources. 1031 * 1032 * XXXLWP limits. 1033 */ 1034void 1035lwp_free(struct lwp *l, bool recycle, bool last) 1036{ 1037 struct proc *p = l->l_proc; 1038 struct rusage *ru; 1039 ksiginfoq_t kq; 1040 1041 KASSERT(l != curlwp); 1042 KASSERT(last || mutex_owned(p->p_lock)); 1043 1044 /* 1045 * If this was not the last LWP in the process, then adjust 1046 * counters and unlock. 1047 */ 1048 if (!last) { 1049 /* 1050 * Add the LWP's run time to the process' base value. 1051 * This needs to co-incide with coming off p_lwps. 1052 */ 1053 bintime_add(&p->p_rtime, &l->l_rtime); 1054 p->p_pctcpu += l->l_pctcpu; 1055 ru = &p->p_stats->p_ru; 1056 ruadd(ru, &l->l_ru); 1057 ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw); 1058 ru->ru_nivcsw += l->l_nivcsw; 1059 LIST_REMOVE(l, l_sibling); 1060 p->p_nlwps--; 1061 p->p_nzlwps--; 1062 if ((l->l_prflag & LPR_DETACHED) != 0) 1063 p->p_ndlwps--; 1064 1065 /* 1066 * Have any LWPs sleeping in lwp_wait() recheck for 1067 * deadlock. 1068 */ 1069 cv_broadcast(&p->p_lwpcv); 1070 mutex_exit(p->p_lock); 1071 } 1072 1073#ifdef MULTIPROCESSOR 1074 /* 1075 * In the unlikely event that the LWP is still on the CPU, 1076 * then spin until it has switched away. We need to release 1077 * all locks to avoid deadlock against interrupt handlers on 1078 * the target CPU. 1079 */ 1080 if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) { 1081 int count; 1082 (void)count; /* XXXgcc */ 1083 KERNEL_UNLOCK_ALL(curlwp, &count); 1084 while ((l->l_pflag & LP_RUNNING) != 0 || 1085 l->l_cpu->ci_curlwp == l) 1086 SPINLOCK_BACKOFF_HOOK; 1087 KERNEL_LOCK(count, curlwp); 1088 } 1089#endif 1090 1091 /* 1092 * Destroy the LWP's remaining signal information. 1093 */ 1094 ksiginfo_queue_init(&kq); 1095 sigclear(&l->l_sigpend, NULL, &kq); 1096 ksiginfo_queue_drain(&kq); 1097 cv_destroy(&l->l_sigcv); 1098 1099 /* 1100 * Free lwpctl structure and affinity. 1101 */ 1102 if (l->l_lwpctl) { 1103 lwp_ctl_free(l); 1104 } 1105 if (l->l_affinity) { 1106 kcpuset_unuse(l->l_affinity, NULL); 1107 l->l_affinity = NULL; 1108 } 1109 1110 /* 1111 * Free the LWP's turnstile and the LWP structure itself unless the 1112 * caller wants to recycle them. Also, free the scheduler specific 1113 * data. 1114 * 1115 * We can't return turnstile0 to the pool (it didn't come from it), 1116 * so if it comes up just drop it quietly and move on. 1117 * 1118 * We don't recycle the VM resources at this time. 1119 */ 1120 1121 if (!recycle && l->l_ts != &turnstile0) 1122 pool_cache_put(turnstile_cache, l->l_ts); 1123 if (l->l_name != NULL) 1124 kmem_free(l->l_name, MAXCOMLEN); 1125 1126 cpu_lwp_free2(l); 1127 uvm_lwp_exit(l); 1128 1129 KASSERT(SLIST_EMPTY(&l->l_pi_lenders)); 1130 KASSERT(l->l_inheritedprio == -1); 1131 KASSERT(l->l_blcnt == 0); 1132 kdtrace_thread_dtor(NULL, l); 1133 if (!recycle) 1134 pool_cache_put(lwp_cache, l); 1135} 1136 1137/* 1138 * Migrate the LWP to the another CPU. Unlocks the LWP. 1139 */ 1140void 1141lwp_migrate(lwp_t *l, struct cpu_info *tci) 1142{ 1143 struct schedstate_percpu *tspc; 1144 int lstat = l->l_stat; 1145 1146 KASSERT(lwp_locked(l, NULL)); 1147 KASSERT(tci != NULL); 1148 1149 /* If LWP is still on the CPU, it must be handled like LSONPROC */ 1150 if ((l->l_pflag & LP_RUNNING) != 0) { 1151 lstat = LSONPROC; 1152 } 1153 1154 /* 1155 * The destination CPU could be changed while previous migration 1156 * was not finished. 1157 */ 1158 if (l->l_target_cpu != NULL) { 1159 l->l_target_cpu = tci; 1160 lwp_unlock(l); 1161 return; 1162 } 1163 1164 /* Nothing to do if trying to migrate to the same CPU */ 1165 if (l->l_cpu == tci) { 1166 lwp_unlock(l); 1167 return; 1168 } 1169 1170 KASSERT(l->l_target_cpu == NULL); 1171 tspc = &tci->ci_schedstate; 1172 switch (lstat) { 1173 case LSRUN: 1174 l->l_target_cpu = tci; 1175 break; 1176 case LSIDL: 1177 l->l_cpu = tci; 1178 lwp_unlock_to(l, tspc->spc_mutex); 1179 return; 1180 case LSSLEEP: 1181 l->l_cpu = tci; 1182 break; 1183 case LSSTOP: 1184 case LSSUSPENDED: 1185 l->l_cpu = tci; 1186 if (l->l_wchan == NULL) { 1187 lwp_unlock_to(l, tspc->spc_lwplock); 1188 return; 1189 } 1190 break; 1191 case LSONPROC: 1192 l->l_target_cpu = tci; 1193 spc_lock(l->l_cpu); 1194 cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT); 1195 spc_unlock(l->l_cpu); 1196 break; 1197 } 1198 lwp_unlock(l); 1199} 1200 1201/* 1202 * Find the LWP in the process. Arguments may be zero, in such case, 1203 * the calling process and first LWP in the list will be used. 1204 * On success - returns proc locked. 1205 */ 1206struct lwp * 1207lwp_find2(pid_t pid, lwpid_t lid) 1208{ 1209 proc_t *p; 1210 lwp_t *l; 1211 1212 /* Find the process. */ 1213 if (pid != 0) { 1214 mutex_enter(proc_lock); 1215 p = proc_find(pid); 1216 if (p == NULL) { 1217 mutex_exit(proc_lock); 1218 return NULL; 1219 } 1220 mutex_enter(p->p_lock); 1221 mutex_exit(proc_lock); 1222 } else { 1223 p = curlwp->l_proc; 1224 mutex_enter(p->p_lock); 1225 } 1226 /* Find the thread. */ 1227 if (lid != 0) { 1228 l = lwp_find(p, lid); 1229 } else { 1230 l = LIST_FIRST(&p->p_lwps); 1231 } 1232 if (l == NULL) { 1233 mutex_exit(p->p_lock); 1234 } 1235 return l; 1236} 1237 1238/* 1239 * Look up a live LWP within the specified process, and return it locked. 1240 * 1241 * Must be called with p->p_lock held. 1242 */ 1243struct lwp * 1244lwp_find(struct proc *p, lwpid_t id) 1245{ 1246 struct lwp *l; 1247 1248 KASSERT(mutex_owned(p->p_lock)); 1249 1250 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1251 if (l->l_lid == id) 1252 break; 1253 } 1254 1255 /* 1256 * No need to lock - all of these conditions will 1257 * be visible with the process level mutex held. 1258 */ 1259 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB)) 1260 l = NULL; 1261 1262 return l; 1263} 1264 1265/* 1266 * Update an LWP's cached credentials to mirror the process' master copy. 1267 * 1268 * This happens early in the syscall path, on user trap, and on LWP 1269 * creation. A long-running LWP can also voluntarily choose to update 1270 * it's credentials by calling this routine. This may be called from 1271 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand. 1272 */ 1273void 1274lwp_update_creds(struct lwp *l) 1275{ 1276 kauth_cred_t oc; 1277 struct proc *p; 1278 1279 p = l->l_proc; 1280 oc = l->l_cred; 1281 1282 mutex_enter(p->p_lock); 1283 kauth_cred_hold(p->p_cred); 1284 l->l_cred = p->p_cred; 1285 l->l_prflag &= ~LPR_CRMOD; 1286 mutex_exit(p->p_lock); 1287 if (oc != NULL) 1288 kauth_cred_free(oc); 1289} 1290 1291/* 1292 * Verify that an LWP is locked, and optionally verify that the lock matches 1293 * one we specify. 1294 */ 1295int 1296lwp_locked(struct lwp *l, kmutex_t *mtx) 1297{ 1298 kmutex_t *cur = l->l_mutex; 1299 1300 return mutex_owned(cur) && (mtx == cur || mtx == NULL); 1301} 1302 1303/* 1304 * Lend a new mutex to an LWP. The old mutex must be held. 1305 */ 1306void 1307lwp_setlock(struct lwp *l, kmutex_t *new) 1308{ 1309 1310 KASSERT(mutex_owned(l->l_mutex)); 1311 1312 membar_exit(); 1313 l->l_mutex = new; 1314} 1315 1316/* 1317 * Lend a new mutex to an LWP, and release the old mutex. The old mutex 1318 * must be held. 1319 */ 1320void 1321lwp_unlock_to(struct lwp *l, kmutex_t *new) 1322{ 1323 kmutex_t *old; 1324 1325 KASSERT(lwp_locked(l, NULL)); 1326 1327 old = l->l_mutex; 1328 membar_exit(); 1329 l->l_mutex = new; 1330 mutex_spin_exit(old); 1331} 1332 1333int 1334lwp_trylock(struct lwp *l) 1335{ 1336 kmutex_t *old; 1337 1338 for (;;) { 1339 if (!mutex_tryenter(old = l->l_mutex)) 1340 return 0; 1341 if (__predict_true(l->l_mutex == old)) 1342 return 1; 1343 mutex_spin_exit(old); 1344 } 1345} 1346 1347void 1348lwp_unsleep(lwp_t *l, bool cleanup) 1349{ 1350 1351 KASSERT(mutex_owned(l->l_mutex)); 1352 (*l->l_syncobj->sobj_unsleep)(l, cleanup); 1353} 1354 1355/* 1356 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is 1357 * set. 1358 */ 1359void 1360lwp_userret(struct lwp *l) 1361{ 1362 struct proc *p; 1363 int sig; 1364 1365 KASSERT(l == curlwp); 1366 KASSERT(l->l_stat == LSONPROC); 1367 p = l->l_proc; 1368 1369#ifndef __HAVE_FAST_SOFTINTS 1370 /* Run pending soft interrupts. */ 1371 if (l->l_cpu->ci_data.cpu_softints != 0) 1372 softint_overlay(); 1373#endif 1374 1375#ifdef KERN_SA 1376 /* Generate UNBLOCKED upcall if needed */ 1377 if (l->l_flag & LW_SA_BLOCKING) { 1378 sa_unblock_userret(l); 1379 /* NOTREACHED */ 1380 } 1381#endif 1382 1383 /* 1384 * It should be safe to do this read unlocked on a multiprocessor 1385 * system.. 1386 * 1387 * LW_SA_UPCALL will be handled after the while() loop, so don't 1388 * consider it now. 1389 */ 1390 while ((l->l_flag & (LW_USERRET & ~(LW_SA_UPCALL))) != 0) { 1391 /* 1392 * Process pending signals first, unless the process 1393 * is dumping core or exiting, where we will instead 1394 * enter the LW_WSUSPEND case below. 1395 */ 1396 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == 1397 LW_PENDSIG) { 1398 mutex_enter(p->p_lock); 1399 while ((sig = issignal(l)) != 0) 1400 postsig(sig); 1401 mutex_exit(p->p_lock); 1402 } 1403 1404 /* 1405 * Core-dump or suspend pending. 1406 * 1407 * In case of core dump, suspend ourselves, so that the kernel 1408 * stack and therefore the userland registers saved in the 1409 * trapframe are around for coredump() to write them out. 1410 * We also need to save any PCU resources that we have so that 1411 * they accessible for coredump(). We issue a wakeup on 1412 * p->p_lwpcv so that sigexit() will write the core file out 1413 * once all other LWPs are suspended. 1414 */ 1415 if ((l->l_flag & LW_WSUSPEND) != 0) { 1416 pcu_save_all(l); 1417 mutex_enter(p->p_lock); 1418 p->p_nrlwps--; 1419 cv_broadcast(&p->p_lwpcv); 1420 lwp_lock(l); 1421 l->l_stat = LSSUSPENDED; 1422 lwp_unlock(l); 1423 mutex_exit(p->p_lock); 1424 lwp_lock(l); 1425 mi_switch(l); 1426 } 1427 1428 /* Process is exiting. */ 1429 if ((l->l_flag & LW_WEXIT) != 0) { 1430 lwp_exit(l); 1431 KASSERT(0); 1432 /* NOTREACHED */ 1433 } 1434 1435 /* update lwpctl processor (for vfork child_return) */ 1436 if (l->l_flag & LW_LWPCTL) { 1437 lwp_lock(l); 1438 KASSERT(kpreempt_disabled()); 1439 l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu); 1440 l->l_lwpctl->lc_pctr++; 1441 l->l_flag &= ~LW_LWPCTL; 1442 lwp_unlock(l); 1443 } 1444 } 1445 1446#ifdef KERN_SA 1447 /* 1448 * Timer events are handled specially. We only try once to deliver 1449 * pending timer upcalls; if if fails, we can try again on the next 1450 * loop around. If we need to re-enter lwp_userret(), MD code will 1451 * bounce us back here through the trap path after we return. 1452 */ 1453 if (p->p_timerpend) 1454 timerupcall(l); 1455 if (l->l_flag & LW_SA_UPCALL) 1456 sa_upcall_userret(l); 1457#endif /* KERN_SA */ 1458} 1459 1460/* 1461 * Force an LWP to enter the kernel, to take a trip through lwp_userret(). 1462 */ 1463void 1464lwp_need_userret(struct lwp *l) 1465{ 1466 KASSERT(lwp_locked(l, NULL)); 1467 1468 /* 1469 * Since the tests in lwp_userret() are done unlocked, make sure 1470 * that the condition will be seen before forcing the LWP to enter 1471 * kernel mode. 1472 */ 1473 membar_producer(); 1474 cpu_signotify(l); 1475} 1476 1477/* 1478 * Add one reference to an LWP. This will prevent the LWP from 1479 * exiting, thus keep the lwp structure and PCB around to inspect. 1480 */ 1481void 1482lwp_addref(struct lwp *l) 1483{ 1484 1485 KASSERT(mutex_owned(l->l_proc->p_lock)); 1486 KASSERT(l->l_stat != LSZOMB); 1487 KASSERT(l->l_refcnt != 0); 1488 1489 l->l_refcnt++; 1490} 1491 1492/* 1493 * Remove one reference to an LWP. If this is the last reference, 1494 * then we must finalize the LWP's death. 1495 */ 1496void 1497lwp_delref(struct lwp *l) 1498{ 1499 struct proc *p = l->l_proc; 1500 1501 mutex_enter(p->p_lock); 1502 lwp_delref2(l); 1503 mutex_exit(p->p_lock); 1504} 1505 1506/* 1507 * Remove one reference to an LWP. If this is the last reference, 1508 * then we must finalize the LWP's death. The proc mutex is held 1509 * on entry. 1510 */ 1511void 1512lwp_delref2(struct lwp *l) 1513{ 1514 struct proc *p = l->l_proc; 1515 1516 KASSERT(mutex_owned(p->p_lock)); 1517 KASSERT(l->l_stat != LSZOMB); 1518 KASSERT(l->l_refcnt > 0); 1519 if (--l->l_refcnt == 0) 1520 cv_broadcast(&p->p_lwpcv); 1521} 1522 1523/* 1524 * Drain all references to the current LWP. 1525 */ 1526void 1527lwp_drainrefs(struct lwp *l) 1528{ 1529 struct proc *p = l->l_proc; 1530 1531 KASSERT(mutex_owned(p->p_lock)); 1532 KASSERT(l->l_refcnt != 0); 1533 1534 l->l_refcnt--; 1535 while (l->l_refcnt != 0) 1536 cv_wait(&p->p_lwpcv, p->p_lock); 1537} 1538 1539/* 1540 * Return true if the specified LWP is 'alive'. Only p->p_lock need 1541 * be held. 1542 */ 1543bool 1544lwp_alive(lwp_t *l) 1545{ 1546 1547 KASSERT(mutex_owned(l->l_proc->p_lock)); 1548 1549 switch (l->l_stat) { 1550 case LSSLEEP: 1551 case LSRUN: 1552 case LSONPROC: 1553 case LSSTOP: 1554 case LSSUSPENDED: 1555 return true; 1556 default: 1557 return false; 1558 } 1559} 1560 1561/* 1562 * Return first live LWP in the process. 1563 */ 1564lwp_t * 1565lwp_find_first(proc_t *p) 1566{ 1567 lwp_t *l; 1568 1569 KASSERT(mutex_owned(p->p_lock)); 1570 1571 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1572 if (lwp_alive(l)) { 1573 return l; 1574 } 1575 } 1576 1577 return NULL; 1578} 1579 1580/* 1581 * Allocate a new lwpctl structure for a user LWP. 1582 */ 1583int 1584lwp_ctl_alloc(vaddr_t *uaddr) 1585{ 1586 lcproc_t *lp; 1587 u_int bit, i, offset; 1588 struct uvm_object *uao; 1589 int error; 1590 lcpage_t *lcp; 1591 proc_t *p; 1592 lwp_t *l; 1593 1594 l = curlwp; 1595 p = l->l_proc; 1596 1597 /* don't allow a vforked process to create lwp ctls */ 1598 if (p->p_lflag & PL_PPWAIT) 1599 return EBUSY; 1600 1601 if (l->l_lcpage != NULL) { 1602 lcp = l->l_lcpage; 1603 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr; 1604 return 0; 1605 } 1606 1607 /* First time around, allocate header structure for the process. */ 1608 if ((lp = p->p_lwpctl) == NULL) { 1609 lp = kmem_alloc(sizeof(*lp), KM_SLEEP); 1610 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE); 1611 lp->lp_uao = NULL; 1612 TAILQ_INIT(&lp->lp_pages); 1613 mutex_enter(p->p_lock); 1614 if (p->p_lwpctl == NULL) { 1615 p->p_lwpctl = lp; 1616 mutex_exit(p->p_lock); 1617 } else { 1618 mutex_exit(p->p_lock); 1619 mutex_destroy(&lp->lp_lock); 1620 kmem_free(lp, sizeof(*lp)); 1621 lp = p->p_lwpctl; 1622 } 1623 } 1624 1625 /* 1626 * Set up an anonymous memory region to hold the shared pages. 1627 * Map them into the process' address space. The user vmspace 1628 * gets the first reference on the UAO. 1629 */ 1630 mutex_enter(&lp->lp_lock); 1631 if (lp->lp_uao == NULL) { 1632 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0); 1633 lp->lp_cur = 0; 1634 lp->lp_max = LWPCTL_UAREA_SZ; 1635 lp->lp_uva = p->p_emul->e_vm_default_addr(p, 1636 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ); 1637 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva, 1638 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, 1639 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0)); 1640 if (error != 0) { 1641 uao_detach(lp->lp_uao); 1642 lp->lp_uao = NULL; 1643 mutex_exit(&lp->lp_lock); 1644 return error; 1645 } 1646 } 1647 1648 /* Get a free block and allocate for this LWP. */ 1649 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) { 1650 if (lcp->lcp_nfree != 0) 1651 break; 1652 } 1653 if (lcp == NULL) { 1654 /* Nothing available - try to set up a free page. */ 1655 if (lp->lp_cur == lp->lp_max) { 1656 mutex_exit(&lp->lp_lock); 1657 return ENOMEM; 1658 } 1659 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP); 1660 if (lcp == NULL) { 1661 mutex_exit(&lp->lp_lock); 1662 return ENOMEM; 1663 } 1664 /* 1665 * Wire the next page down in kernel space. Since this 1666 * is a new mapping, we must add a reference. 1667 */ 1668 uao = lp->lp_uao; 1669 (*uao->pgops->pgo_reference)(uao); 1670 lcp->lcp_kaddr = vm_map_min(kernel_map); 1671 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE, 1672 uao, lp->lp_cur, PAGE_SIZE, 1673 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, 1674 UVM_INH_NONE, UVM_ADV_RANDOM, 0)); 1675 if (error != 0) { 1676 mutex_exit(&lp->lp_lock); 1677 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1678 (*uao->pgops->pgo_detach)(uao); 1679 return error; 1680 } 1681 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr, 1682 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0); 1683 if (error != 0) { 1684 mutex_exit(&lp->lp_lock); 1685 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1686 lcp->lcp_kaddr + PAGE_SIZE); 1687 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1688 return error; 1689 } 1690 /* Prepare the page descriptor and link into the list. */ 1691 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur; 1692 lp->lp_cur += PAGE_SIZE; 1693 lcp->lcp_nfree = LWPCTL_PER_PAGE; 1694 lcp->lcp_rotor = 0; 1695 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ); 1696 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1697 } 1698 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) { 1699 if (++i >= LWPCTL_BITMAP_ENTRIES) 1700 i = 0; 1701 } 1702 bit = ffs(lcp->lcp_bitmap[i]) - 1; 1703 lcp->lcp_bitmap[i] ^= (1 << bit); 1704 lcp->lcp_rotor = i; 1705 lcp->lcp_nfree--; 1706 l->l_lcpage = lcp; 1707 offset = (i << 5) + bit; 1708 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset; 1709 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t); 1710 mutex_exit(&lp->lp_lock); 1711 1712 KPREEMPT_DISABLE(l); 1713 l->l_lwpctl->lc_curcpu = (int)curcpu()->ci_data.cpu_index; 1714 KPREEMPT_ENABLE(l); 1715 1716 return 0; 1717} 1718 1719/* 1720 * Free an lwpctl structure back to the per-process list. 1721 */ 1722void 1723lwp_ctl_free(lwp_t *l) 1724{ 1725 struct proc *p = l->l_proc; 1726 lcproc_t *lp; 1727 lcpage_t *lcp; 1728 u_int map, offset; 1729 1730 /* don't free a lwp context we borrowed for vfork */ 1731 if (p->p_lflag & PL_PPWAIT) { 1732 l->l_lwpctl = NULL; 1733 return; 1734 } 1735 1736 lp = p->p_lwpctl; 1737 KASSERT(lp != NULL); 1738 1739 lcp = l->l_lcpage; 1740 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr); 1741 KASSERT(offset < LWPCTL_PER_PAGE); 1742 1743 mutex_enter(&lp->lp_lock); 1744 lcp->lcp_nfree++; 1745 map = offset >> 5; 1746 lcp->lcp_bitmap[map] |= (1 << (offset & 31)); 1747 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0) 1748 lcp->lcp_rotor = map; 1749 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) { 1750 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain); 1751 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1752 } 1753 mutex_exit(&lp->lp_lock); 1754} 1755 1756/* 1757 * Process is exiting; tear down lwpctl state. This can only be safely 1758 * called by the last LWP in the process. 1759 */ 1760void 1761lwp_ctl_exit(void) 1762{ 1763 lcpage_t *lcp, *next; 1764 lcproc_t *lp; 1765 proc_t *p; 1766 lwp_t *l; 1767 1768 l = curlwp; 1769 l->l_lwpctl = NULL; 1770 l->l_lcpage = NULL; 1771 p = l->l_proc; 1772 lp = p->p_lwpctl; 1773 1774 KASSERT(lp != NULL); 1775 KASSERT(p->p_nlwps == 1); 1776 1777 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) { 1778 next = TAILQ_NEXT(lcp, lcp_chain); 1779 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1780 lcp->lcp_kaddr + PAGE_SIZE); 1781 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1782 } 1783 1784 if (lp->lp_uao != NULL) { 1785 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva, 1786 lp->lp_uva + LWPCTL_UAREA_SZ); 1787 } 1788 1789 mutex_destroy(&lp->lp_lock); 1790 kmem_free(lp, sizeof(*lp)); 1791 p->p_lwpctl = NULL; 1792} 1793 1794/* 1795 * Return the current LWP's "preemption counter". Used to detect 1796 * preemption across operations that can tolerate preemption without 1797 * crashing, but which may generate incorrect results if preempted. 1798 */ 1799uint64_t 1800lwp_pctr(void) 1801{ 1802 1803 return curlwp->l_ncsw; 1804} 1805 1806/* 1807 * Set an LWP's private data pointer. 1808 */ 1809int 1810lwp_setprivate(struct lwp *l, void *ptr) 1811{ 1812 int error = 0; 1813 1814 l->l_private = ptr; 1815#ifdef __HAVE_CPU_LWP_SETPRIVATE 1816 error = cpu_lwp_setprivate(l, ptr); 1817#endif 1818 return error; 1819} 1820 1821#if defined(DDB) 1822#include <machine/pcb.h> 1823 1824void 1825lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 1826{ 1827 lwp_t *l; 1828 1829 LIST_FOREACH(l, &alllwp, l_list) { 1830 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l); 1831 1832 if (addr < stack || stack + KSTACK_SIZE <= addr) { 1833 continue; 1834 } 1835 (*pr)("%p is %p+%zu, LWP %p's stack\n", 1836 (void *)addr, (void *)stack, 1837 (size_t)(addr - stack), l); 1838 } 1839} 1840#endif /* defined(DDB) */ 1841