kern_thread.c revision 218972
1/*- 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29#include "opt_witness.h" 30#include "opt_hwpmc_hooks.h" 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/kern/kern_thread.c 218972 2011-02-23 13:50:24Z kib $"); 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/lock.h> 39#include <sys/mutex.h> 40#include <sys/proc.h> 41#include <sys/resourcevar.h> 42#include <sys/smp.h> 43#include <sys/sysctl.h> 44#include <sys/sched.h> 45#include <sys/sleepqueue.h> 46#include <sys/selinfo.h> 47#include <sys/turnstile.h> 48#include <sys/ktr.h> 49#include <sys/rwlock.h> 50#include <sys/umtx.h> 51#include <sys/cpuset.h> 52#ifdef HWPMC_HOOKS 53#include <sys/pmckern.h> 54#endif 55 56#include <security/audit/audit.h> 57 58#include <vm/vm.h> 59#include <vm/vm_extern.h> 60#include <vm/uma.h> 61#include <sys/eventhandler.h> 62 63/* 64 * thread related storage. 65 */ 66static uma_zone_t thread_zone; 67 68TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 69static struct mtx zombie_lock; 70MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); 71 72static void thread_zombie(struct thread *); 73 74#define TID_BUFFER_SIZE 1024 75 76struct mtx tid_lock; 77static struct unrhdr *tid_unrhdr; 78static lwpid_t tid_buffer[TID_BUFFER_SIZE]; 79static int tid_head, tid_tail; 80static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash"); 81 82struct tidhashhead *tidhashtbl; 83u_long tidhash; 84struct rwlock tidhash_lock; 85 86static lwpid_t 87tid_alloc(void) 88{ 89 lwpid_t tid; 90 91 tid = alloc_unr(tid_unrhdr); 92 if (tid != -1) 93 return (tid); 94 mtx_lock(&tid_lock); 95 if (tid_head == tid_tail) { 96 mtx_unlock(&tid_lock); 97 return (-1); 98 } 99 tid = tid_buffer[tid_head++]; 100 tid_head %= TID_BUFFER_SIZE; 101 mtx_unlock(&tid_lock); 102 return (tid); 103} 104 105static void 106tid_free(lwpid_t tid) 107{ 108 lwpid_t tmp_tid = -1; 109 110 mtx_lock(&tid_lock); 111 if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) { 112 tmp_tid = tid_buffer[tid_head++]; 113 tid_head = (tid_head + 1) % TID_BUFFER_SIZE; 114 } 115 tid_buffer[tid_tail++] = tid; 116 tid_tail %= TID_BUFFER_SIZE; 117 mtx_unlock(&tid_lock); 118 if (tmp_tid != -1) 119 free_unr(tid_unrhdr, tmp_tid); 120} 121 122/* 123 * Prepare a thread for use. 124 */ 125static int 126thread_ctor(void *mem, int size, void *arg, int flags) 127{ 128 struct thread *td; 129 130 td = (struct thread *)mem; 131 td->td_state = TDS_INACTIVE; 132 td->td_oncpu = NOCPU; 133 134 td->td_tid = tid_alloc(); 135 136 /* 137 * Note that td_critnest begins life as 1 because the thread is not 138 * running and is thereby implicitly waiting to be on the receiving 139 * end of a context switch. 140 */ 141 td->td_critnest = 1; 142 td->td_lend_user_pri = PRI_MAX; 143 EVENTHANDLER_INVOKE(thread_ctor, td); 144#ifdef AUDIT 145 audit_thread_alloc(td); 146#endif 147 umtx_thread_alloc(td); 148 return (0); 149} 150 151/* 152 * Reclaim a thread after use. 153 */ 154static void 155thread_dtor(void *mem, int size, void *arg) 156{ 157 struct thread *td; 158 159 td = (struct thread *)mem; 160 161#ifdef INVARIANTS 162 /* Verify that this thread is in a safe state to free. */ 163 switch (td->td_state) { 164 case TDS_INHIBITED: 165 case TDS_RUNNING: 166 case TDS_CAN_RUN: 167 case TDS_RUNQ: 168 /* 169 * We must never unlink a thread that is in one of 170 * these states, because it is currently active. 171 */ 172 panic("bad state for thread unlinking"); 173 /* NOTREACHED */ 174 case TDS_INACTIVE: 175 break; 176 default: 177 panic("bad thread state"); 178 /* NOTREACHED */ 179 } 180#endif 181#ifdef AUDIT 182 audit_thread_free(td); 183#endif 184 /* Free all OSD associated to this thread. */ 185 osd_thread_exit(td); 186 187 EVENTHANDLER_INVOKE(thread_dtor, td); 188 tid_free(td->td_tid); 189} 190 191/* 192 * Initialize type-stable parts of a thread (when newly created). 193 */ 194static int 195thread_init(void *mem, int size, int flags) 196{ 197 struct thread *td; 198 199 td = (struct thread *)mem; 200 201 td->td_sleepqueue = sleepq_alloc(); 202 td->td_turnstile = turnstile_alloc(); 203 EVENTHANDLER_INVOKE(thread_init, td); 204 td->td_sched = (struct td_sched *)&td[1]; 205 umtx_thread_init(td); 206 td->td_kstack = 0; 207 return (0); 208} 209 210/* 211 * Tear down type-stable parts of a thread (just before being discarded). 212 */ 213static void 214thread_fini(void *mem, int size) 215{ 216 struct thread *td; 217 218 td = (struct thread *)mem; 219 EVENTHANDLER_INVOKE(thread_fini, td); 220 turnstile_free(td->td_turnstile); 221 sleepq_free(td->td_sleepqueue); 222 umtx_thread_fini(td); 223 seltdfini(td); 224} 225 226/* 227 * For a newly created process, 228 * link up all the structures and its initial threads etc. 229 * called from: 230 * {arch}/{arch}/machdep.c ia64_init(), init386() etc. 231 * proc_dtor() (should go away) 232 * proc_init() 233 */ 234void 235proc_linkup0(struct proc *p, struct thread *td) 236{ 237 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 238 proc_linkup(p, td); 239} 240 241void 242proc_linkup(struct proc *p, struct thread *td) 243{ 244 245 sigqueue_init(&p->p_sigqueue, p); 246 p->p_ksi = ksiginfo_alloc(1); 247 if (p->p_ksi != NULL) { 248 /* XXX p_ksi may be null if ksiginfo zone is not ready */ 249 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 250 } 251 LIST_INIT(&p->p_mqnotifier); 252 p->p_numthreads = 0; 253 thread_link(td, p); 254} 255 256/* 257 * Initialize global thread allocation resources. 258 */ 259void 260threadinit(void) 261{ 262 263 mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 264 /* leave one number for thread0 */ 265 tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock); 266 267 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 268 thread_ctor, thread_dtor, thread_init, thread_fini, 269 16 - 1, 0); 270 tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash); 271 rw_init(&tidhash_lock, "tidhash"); 272} 273 274/* 275 * Place an unused thread on the zombie list. 276 * Use the slpq as that must be unused by now. 277 */ 278void 279thread_zombie(struct thread *td) 280{ 281 mtx_lock_spin(&zombie_lock); 282 TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); 283 mtx_unlock_spin(&zombie_lock); 284} 285 286/* 287 * Release a thread that has exited after cpu_throw(). 288 */ 289void 290thread_stash(struct thread *td) 291{ 292 atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); 293 thread_zombie(td); 294} 295 296/* 297 * Reap zombie resources. 298 */ 299void 300thread_reap(void) 301{ 302 struct thread *td_first, *td_next; 303 304 /* 305 * Don't even bother to lock if none at this instant, 306 * we really don't care about the next instant.. 307 */ 308 if (!TAILQ_EMPTY(&zombie_threads)) { 309 mtx_lock_spin(&zombie_lock); 310 td_first = TAILQ_FIRST(&zombie_threads); 311 if (td_first) 312 TAILQ_INIT(&zombie_threads); 313 mtx_unlock_spin(&zombie_lock); 314 while (td_first) { 315 td_next = TAILQ_NEXT(td_first, td_slpq); 316 if (td_first->td_ucred) 317 crfree(td_first->td_ucred); 318 thread_free(td_first); 319 td_first = td_next; 320 } 321 } 322} 323 324/* 325 * Allocate a thread. 326 */ 327struct thread * 328thread_alloc(int pages) 329{ 330 struct thread *td; 331 332 thread_reap(); /* check if any zombies to get */ 333 334 td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK); 335 KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); 336 if (!vm_thread_new(td, pages)) { 337 uma_zfree(thread_zone, td); 338 return (NULL); 339 } 340 cpu_thread_alloc(td); 341 return (td); 342} 343 344int 345thread_alloc_stack(struct thread *td, int pages) 346{ 347 348 KASSERT(td->td_kstack == 0, 349 ("thread_alloc_stack called on a thread with kstack")); 350 if (!vm_thread_new(td, pages)) 351 return (0); 352 cpu_thread_alloc(td); 353 return (1); 354} 355 356/* 357 * Deallocate a thread. 358 */ 359void 360thread_free(struct thread *td) 361{ 362 363 lock_profile_thread_exit(td); 364 if (td->td_cpuset) 365 cpuset_rel(td->td_cpuset); 366 td->td_cpuset = NULL; 367 cpu_thread_free(td); 368 if (td->td_kstack != 0) 369 vm_thread_dispose(td); 370 uma_zfree(thread_zone, td); 371} 372 373/* 374 * Discard the current thread and exit from its context. 375 * Always called with scheduler locked. 376 * 377 * Because we can't free a thread while we're operating under its context, 378 * push the current thread into our CPU's deadthread holder. This means 379 * we needn't worry about someone else grabbing our context before we 380 * do a cpu_throw(). 381 */ 382void 383thread_exit(void) 384{ 385 uint64_t new_switchtime; 386 struct thread *td; 387 struct thread *td2; 388 struct proc *p; 389 int wakeup_swapper; 390 391 td = curthread; 392 p = td->td_proc; 393 394 PROC_SLOCK_ASSERT(p, MA_OWNED); 395 mtx_assert(&Giant, MA_NOTOWNED); 396 397 PROC_LOCK_ASSERT(p, MA_OWNED); 398 KASSERT(p != NULL, ("thread exiting without a process")); 399 CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 400 (long)p->p_pid, td->td_name); 401 KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 402 403#ifdef AUDIT 404 AUDIT_SYSCALL_EXIT(0, td); 405#endif 406 umtx_thread_exit(td); 407 /* 408 * drop FPU & debug register state storage, or any other 409 * architecture specific resources that 410 * would not be on a new untouched process. 411 */ 412 cpu_thread_exit(td); /* XXXSMP */ 413 414 /* Do the same timestamp bookkeeping that mi_switch() would do. */ 415 new_switchtime = cpu_ticks(); 416 p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime)); 417 PCPU_SET(switchtime, new_switchtime); 418 PCPU_SET(switchticks, ticks); 419 PCPU_INC(cnt.v_swtch); 420 /* Save our resource usage in our process. */ 421 td->td_ru.ru_nvcsw++; 422 rucollect(&p->p_ru, &td->td_ru); 423 /* 424 * The last thread is left attached to the process 425 * So that the whole bundle gets recycled. Skip 426 * all this stuff if we never had threads. 427 * EXIT clears all sign of other threads when 428 * it goes to single threading, so the last thread always 429 * takes the short path. 430 */ 431 if (p->p_flag & P_HADTHREADS) { 432 if (p->p_numthreads > 1) { 433 thread_unlink(td); 434 td2 = FIRST_THREAD_IN_PROC(p); 435 sched_exit_thread(td2, td); 436 437 /* 438 * The test below is NOT true if we are the 439 * sole exiting thread. P_STOPPED_SINGLE is unset 440 * in exit1() after it is the only survivor. 441 */ 442 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 443 if (p->p_numthreads == p->p_suspcount) { 444 thread_lock(p->p_singlethread); 445 wakeup_swapper = thread_unsuspend_one( 446 p->p_singlethread); 447 thread_unlock(p->p_singlethread); 448 if (wakeup_swapper) 449 kick_proc0(); 450 } 451 } 452 453 atomic_add_int(&td->td_proc->p_exitthreads, 1); 454 PCPU_SET(deadthread, td); 455 } else { 456 /* 457 * The last thread is exiting.. but not through exit() 458 */ 459 panic ("thread_exit: Last thread exiting on its own"); 460 } 461 } 462#ifdef HWPMC_HOOKS 463 /* 464 * If this thread is part of a process that is being tracked by hwpmc(4), 465 * inform the module of the thread's impending exit. 466 */ 467 if (PMC_PROC_IS_USING_PMCS(td->td_proc)) 468 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); 469#endif 470 PROC_UNLOCK(p); 471 ruxagg(p, td); 472 thread_lock(td); 473 PROC_SUNLOCK(p); 474 td->td_state = TDS_INACTIVE; 475#ifdef WITNESS 476 witness_thread_exit(td); 477#endif 478 CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 479 sched_throw(td); 480 panic("I'm a teapot!"); 481 /* NOTREACHED */ 482} 483 484/* 485 * Do any thread specific cleanups that may be needed in wait() 486 * called with Giant, proc and schedlock not held. 487 */ 488void 489thread_wait(struct proc *p) 490{ 491 struct thread *td; 492 493 mtx_assert(&Giant, MA_NOTOWNED); 494 KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 495 td = FIRST_THREAD_IN_PROC(p); 496 /* Lock the last thread so we spin until it exits cpu_throw(). */ 497 thread_lock(td); 498 thread_unlock(td); 499 /* Wait for any remaining threads to exit cpu_throw(). */ 500 while (p->p_exitthreads) 501 sched_relinquish(curthread); 502 lock_profile_thread_exit(td); 503 cpuset_rel(td->td_cpuset); 504 td->td_cpuset = NULL; 505 cpu_thread_clean(td); 506 crfree(td->td_ucred); 507 thread_reap(); /* check for zombie threads etc. */ 508} 509 510/* 511 * Link a thread to a process. 512 * set up anything that needs to be initialized for it to 513 * be used by the process. 514 */ 515void 516thread_link(struct thread *td, struct proc *p) 517{ 518 519 /* 520 * XXX This can't be enabled because it's called for proc0 before 521 * its lock has been created. 522 * PROC_LOCK_ASSERT(p, MA_OWNED); 523 */ 524 td->td_state = TDS_INACTIVE; 525 td->td_proc = p; 526 td->td_flags = TDF_INMEM; 527 528 LIST_INIT(&td->td_contested); 529 LIST_INIT(&td->td_lprof[0]); 530 LIST_INIT(&td->td_lprof[1]); 531 sigqueue_init(&td->td_sigqueue, p); 532 callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 533 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 534 p->p_numthreads++; 535} 536 537/* 538 * Convert a process with one thread to an unthreaded process. 539 */ 540void 541thread_unthread(struct thread *td) 542{ 543 struct proc *p = td->td_proc; 544 545 KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); 546 p->p_flag &= ~P_HADTHREADS; 547} 548 549/* 550 * Called from: 551 * thread_exit() 552 */ 553void 554thread_unlink(struct thread *td) 555{ 556 struct proc *p = td->td_proc; 557 558 PROC_LOCK_ASSERT(p, MA_OWNED); 559 TAILQ_REMOVE(&p->p_threads, td, td_plist); 560 p->p_numthreads--; 561 /* could clear a few other things here */ 562 /* Must NOT clear links to proc! */ 563} 564 565static int 566calc_remaining(struct proc *p, int mode) 567{ 568 int remaining; 569 570 if (mode == SINGLE_EXIT) 571 remaining = p->p_numthreads; 572 else if (mode == SINGLE_BOUNDARY) 573 remaining = p->p_numthreads - p->p_boundary_count; 574 else if (mode == SINGLE_NO_EXIT) 575 remaining = p->p_numthreads - p->p_suspcount; 576 else 577 panic("calc_remaining: wrong mode %d", mode); 578 return (remaining); 579} 580 581/* 582 * Enforce single-threading. 583 * 584 * Returns 1 if the caller must abort (another thread is waiting to 585 * exit the process or similar). Process is locked! 586 * Returns 0 when you are successfully the only thread running. 587 * A process has successfully single threaded in the suspend mode when 588 * There are no threads in user mode. Threads in the kernel must be 589 * allowed to continue until they get to the user boundary. They may even 590 * copy out their return values and data before suspending. They may however be 591 * accelerated in reaching the user boundary as we will wake up 592 * any sleeping threads that are interruptable. (PCATCH). 593 */ 594int 595thread_single(int mode) 596{ 597 struct thread *td; 598 struct thread *td2; 599 struct proc *p; 600 int remaining, wakeup_swapper; 601 602 td = curthread; 603 p = td->td_proc; 604 mtx_assert(&Giant, MA_NOTOWNED); 605 PROC_LOCK_ASSERT(p, MA_OWNED); 606 KASSERT((td != NULL), ("curthread is NULL")); 607 608 if ((p->p_flag & P_HADTHREADS) == 0) 609 return (0); 610 611 /* Is someone already single threading? */ 612 if (p->p_singlethread != NULL && p->p_singlethread != td) 613 return (1); 614 615 if (mode == SINGLE_EXIT) { 616 p->p_flag |= P_SINGLE_EXIT; 617 p->p_flag &= ~P_SINGLE_BOUNDARY; 618 } else { 619 p->p_flag &= ~P_SINGLE_EXIT; 620 if (mode == SINGLE_BOUNDARY) 621 p->p_flag |= P_SINGLE_BOUNDARY; 622 else 623 p->p_flag &= ~P_SINGLE_BOUNDARY; 624 } 625 p->p_flag |= P_STOPPED_SINGLE; 626 PROC_SLOCK(p); 627 p->p_singlethread = td; 628 remaining = calc_remaining(p, mode); 629 while (remaining != 1) { 630 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 631 goto stopme; 632 wakeup_swapper = 0; 633 FOREACH_THREAD_IN_PROC(p, td2) { 634 if (td2 == td) 635 continue; 636 thread_lock(td2); 637 td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK; 638 if (TD_IS_INHIBITED(td2)) { 639 switch (mode) { 640 case SINGLE_EXIT: 641 if (TD_IS_SUSPENDED(td2)) 642 wakeup_swapper |= 643 thread_unsuspend_one(td2); 644 if (TD_ON_SLEEPQ(td2) && 645 (td2->td_flags & TDF_SINTR)) 646 wakeup_swapper |= 647 sleepq_abort(td2, EINTR); 648 break; 649 case SINGLE_BOUNDARY: 650 if (TD_IS_SUSPENDED(td2) && 651 !(td2->td_flags & TDF_BOUNDARY)) 652 wakeup_swapper |= 653 thread_unsuspend_one(td2); 654 if (TD_ON_SLEEPQ(td2) && 655 (td2->td_flags & TDF_SINTR)) 656 wakeup_swapper |= 657 sleepq_abort(td2, ERESTART); 658 break; 659 case SINGLE_NO_EXIT: 660 if (TD_IS_SUSPENDED(td2) && 661 !(td2->td_flags & TDF_BOUNDARY)) 662 wakeup_swapper |= 663 thread_unsuspend_one(td2); 664 if (TD_ON_SLEEPQ(td2) && 665 (td2->td_flags & TDF_SINTR)) 666 wakeup_swapper |= 667 sleepq_abort(td2, ERESTART); 668 break; 669 default: 670 break; 671 } 672 } 673#ifdef SMP 674 else if (TD_IS_RUNNING(td2) && td != td2) { 675 forward_signal(td2); 676 } 677#endif 678 thread_unlock(td2); 679 } 680 if (wakeup_swapper) 681 kick_proc0(); 682 remaining = calc_remaining(p, mode); 683 684 /* 685 * Maybe we suspended some threads.. was it enough? 686 */ 687 if (remaining == 1) 688 break; 689 690stopme: 691 /* 692 * Wake us up when everyone else has suspended. 693 * In the mean time we suspend as well. 694 */ 695 thread_suspend_switch(td); 696 remaining = calc_remaining(p, mode); 697 } 698 if (mode == SINGLE_EXIT) { 699 /* 700 * We have gotten rid of all the other threads and we 701 * are about to either exit or exec. In either case, 702 * we try our utmost to revert to being a non-threaded 703 * process. 704 */ 705 p->p_singlethread = NULL; 706 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 707 thread_unthread(td); 708 } 709 PROC_SUNLOCK(p); 710 return (0); 711} 712 713/* 714 * Called in from locations that can safely check to see 715 * whether we have to suspend or at least throttle for a 716 * single-thread event (e.g. fork). 717 * 718 * Such locations include userret(). 719 * If the "return_instead" argument is non zero, the thread must be able to 720 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 721 * 722 * The 'return_instead' argument tells the function if it may do a 723 * thread_exit() or suspend, or whether the caller must abort and back 724 * out instead. 725 * 726 * If the thread that set the single_threading request has set the 727 * P_SINGLE_EXIT bit in the process flags then this call will never return 728 * if 'return_instead' is false, but will exit. 729 * 730 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 731 *---------------+--------------------+--------------------- 732 * 0 | returns 0 | returns 0 or 1 733 * | when ST ends | immediatly 734 *---------------+--------------------+--------------------- 735 * 1 | thread exits | returns 1 736 * | | immediatly 737 * 0 = thread_exit() or suspension ok, 738 * other = return error instead of stopping the thread. 739 * 740 * While a full suspension is under effect, even a single threading 741 * thread would be suspended if it made this call (but it shouldn't). 742 * This call should only be made from places where 743 * thread_exit() would be safe as that may be the outcome unless 744 * return_instead is set. 745 */ 746int 747thread_suspend_check(int return_instead) 748{ 749 struct thread *td; 750 struct proc *p; 751 int wakeup_swapper; 752 753 td = curthread; 754 p = td->td_proc; 755 mtx_assert(&Giant, MA_NOTOWNED); 756 PROC_LOCK_ASSERT(p, MA_OWNED); 757 while (P_SHOULDSTOP(p) || 758 ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) { 759 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 760 KASSERT(p->p_singlethread != NULL, 761 ("singlethread not set")); 762 /* 763 * The only suspension in action is a 764 * single-threading. Single threader need not stop. 765 * XXX Should be safe to access unlocked 766 * as it can only be set to be true by us. 767 */ 768 if (p->p_singlethread == td) 769 return (0); /* Exempt from stopping. */ 770 } 771 if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 772 return (EINTR); 773 774 /* Should we goto user boundary if we didn't come from there? */ 775 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 776 (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 777 return (ERESTART); 778 779 /* 780 * If the process is waiting for us to exit, 781 * this thread should just suicide. 782 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 783 */ 784 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 785 PROC_UNLOCK(p); 786 tidhash_remove(td); 787 PROC_LOCK(p); 788 tdsigcleanup(td); 789 PROC_SLOCK(p); 790 thread_stopped(p); 791 thread_exit(); 792 } 793 794 PROC_SLOCK(p); 795 thread_stopped(p); 796 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 797 if (p->p_numthreads == p->p_suspcount + 1) { 798 thread_lock(p->p_singlethread); 799 wakeup_swapper = 800 thread_unsuspend_one(p->p_singlethread); 801 thread_unlock(p->p_singlethread); 802 if (wakeup_swapper) 803 kick_proc0(); 804 } 805 } 806 PROC_UNLOCK(p); 807 thread_lock(td); 808 /* 809 * When a thread suspends, it just 810 * gets taken off all queues. 811 */ 812 thread_suspend_one(td); 813 if (return_instead == 0) { 814 p->p_boundary_count++; 815 td->td_flags |= TDF_BOUNDARY; 816 } 817 PROC_SUNLOCK(p); 818 mi_switch(SW_INVOL | SWT_SUSPEND, NULL); 819 if (return_instead == 0) 820 td->td_flags &= ~TDF_BOUNDARY; 821 thread_unlock(td); 822 PROC_LOCK(p); 823 if (return_instead == 0) 824 p->p_boundary_count--; 825 } 826 return (0); 827} 828 829void 830thread_suspend_switch(struct thread *td) 831{ 832 struct proc *p; 833 834 p = td->td_proc; 835 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 836 PROC_LOCK_ASSERT(p, MA_OWNED); 837 PROC_SLOCK_ASSERT(p, MA_OWNED); 838 /* 839 * We implement thread_suspend_one in stages here to avoid 840 * dropping the proc lock while the thread lock is owned. 841 */ 842 thread_stopped(p); 843 p->p_suspcount++; 844 PROC_UNLOCK(p); 845 thread_lock(td); 846 td->td_flags &= ~TDF_NEEDSUSPCHK; 847 TD_SET_SUSPENDED(td); 848 sched_sleep(td, 0); 849 PROC_SUNLOCK(p); 850 DROP_GIANT(); 851 mi_switch(SW_VOL | SWT_SUSPEND, NULL); 852 thread_unlock(td); 853 PICKUP_GIANT(); 854 PROC_LOCK(p); 855 PROC_SLOCK(p); 856} 857 858void 859thread_suspend_one(struct thread *td) 860{ 861 struct proc *p = td->td_proc; 862 863 PROC_SLOCK_ASSERT(p, MA_OWNED); 864 THREAD_LOCK_ASSERT(td, MA_OWNED); 865 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 866 p->p_suspcount++; 867 td->td_flags &= ~TDF_NEEDSUSPCHK; 868 TD_SET_SUSPENDED(td); 869 sched_sleep(td, 0); 870} 871 872int 873thread_unsuspend_one(struct thread *td) 874{ 875 struct proc *p = td->td_proc; 876 877 PROC_SLOCK_ASSERT(p, MA_OWNED); 878 THREAD_LOCK_ASSERT(td, MA_OWNED); 879 KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 880 TD_CLR_SUSPENDED(td); 881 p->p_suspcount--; 882 return (setrunnable(td)); 883} 884 885/* 886 * Allow all threads blocked by single threading to continue running. 887 */ 888void 889thread_unsuspend(struct proc *p) 890{ 891 struct thread *td; 892 int wakeup_swapper; 893 894 PROC_LOCK_ASSERT(p, MA_OWNED); 895 PROC_SLOCK_ASSERT(p, MA_OWNED); 896 wakeup_swapper = 0; 897 if (!P_SHOULDSTOP(p)) { 898 FOREACH_THREAD_IN_PROC(p, td) { 899 thread_lock(td); 900 if (TD_IS_SUSPENDED(td)) { 901 wakeup_swapper |= thread_unsuspend_one(td); 902 } 903 thread_unlock(td); 904 } 905 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 906 (p->p_numthreads == p->p_suspcount)) { 907 /* 908 * Stopping everything also did the job for the single 909 * threading request. Now we've downgraded to single-threaded, 910 * let it continue. 911 */ 912 thread_lock(p->p_singlethread); 913 wakeup_swapper = thread_unsuspend_one(p->p_singlethread); 914 thread_unlock(p->p_singlethread); 915 } 916 if (wakeup_swapper) 917 kick_proc0(); 918} 919 920/* 921 * End the single threading mode.. 922 */ 923void 924thread_single_end(void) 925{ 926 struct thread *td; 927 struct proc *p; 928 int wakeup_swapper; 929 930 td = curthread; 931 p = td->td_proc; 932 PROC_LOCK_ASSERT(p, MA_OWNED); 933 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); 934 PROC_SLOCK(p); 935 p->p_singlethread = NULL; 936 wakeup_swapper = 0; 937 /* 938 * If there are other threads they may now run, 939 * unless of course there is a blanket 'stop order' 940 * on the process. The single threader must be allowed 941 * to continue however as this is a bad place to stop. 942 */ 943 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 944 FOREACH_THREAD_IN_PROC(p, td) { 945 thread_lock(td); 946 if (TD_IS_SUSPENDED(td)) { 947 wakeup_swapper |= thread_unsuspend_one(td); 948 } 949 thread_unlock(td); 950 } 951 } 952 PROC_SUNLOCK(p); 953 if (wakeup_swapper) 954 kick_proc0(); 955} 956 957struct thread * 958thread_find(struct proc *p, lwpid_t tid) 959{ 960 struct thread *td; 961 962 PROC_LOCK_ASSERT(p, MA_OWNED); 963 FOREACH_THREAD_IN_PROC(p, td) { 964 if (td->td_tid == tid) 965 break; 966 } 967 return (td); 968} 969 970/* Locate a thread by number; return with proc lock held. */ 971struct thread * 972tdfind(lwpid_t tid, pid_t pid) 973{ 974#define RUN_THRESH 16 975 struct thread *td; 976 int run = 0; 977 978 rw_rlock(&tidhash_lock); 979 LIST_FOREACH(td, TIDHASH(tid), td_hash) { 980 if (td->td_tid == tid) { 981 if (pid != -1 && td->td_proc->p_pid != pid) { 982 td = NULL; 983 break; 984 } 985 if (td->td_proc->p_state == PRS_NEW) { 986 td = NULL; 987 break; 988 } 989 if (run > RUN_THRESH) { 990 if (rw_try_upgrade(&tidhash_lock)) { 991 LIST_REMOVE(td, td_hash); 992 LIST_INSERT_HEAD(TIDHASH(td->td_tid), 993 td, td_hash); 994 PROC_LOCK(td->td_proc); 995 rw_wunlock(&tidhash_lock); 996 return (td); 997 } 998 } 999 PROC_LOCK(td->td_proc); 1000 break; 1001 } 1002 run++; 1003 } 1004 rw_runlock(&tidhash_lock); 1005 return (td); 1006} 1007 1008void 1009tidhash_add(struct thread *td) 1010{ 1011 rw_wlock(&tidhash_lock); 1012 LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); 1013 rw_wunlock(&tidhash_lock); 1014} 1015 1016void 1017tidhash_remove(struct thread *td) 1018{ 1019 rw_wlock(&tidhash_lock); 1020 LIST_REMOVE(td, td_hash); 1021 rw_wunlock(&tidhash_lock); 1022} 1023