kern_thread.c revision 181695
1139804Simp/*- 299026Sjulian * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 399026Sjulian * All rights reserved. 499026Sjulian * 599026Sjulian * Redistribution and use in source and binary forms, with or without 699026Sjulian * modification, are permitted provided that the following conditions 799026Sjulian * are met: 899026Sjulian * 1. Redistributions of source code must retain the above copyright 999026Sjulian * notice(s), this list of conditions and the following disclaimer as 10124350Sschweikh * the first lines of this file unmodified other than the possible 1199026Sjulian * addition of one or more copyright notices. 1299026Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1399026Sjulian * notice(s), this list of conditions and the following disclaimer in the 1499026Sjulian * documentation and/or other materials provided with the distribution. 1599026Sjulian * 1699026Sjulian * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1799026Sjulian * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1899026Sjulian * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1999026Sjulian * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2099026Sjulian * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2199026Sjulian * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2299026Sjulian * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2399026Sjulian * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2499026Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2599026Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2699026Sjulian * DAMAGE. 2799026Sjulian */ 2899026Sjulian 29181695Sattilio#include "opt_witness.h" 30181695Sattilio 31116182Sobrien#include <sys/cdefs.h> 32116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_thread.c 181695 2008-08-13 18:24:22Z attilio $"); 33116182Sobrien 3499026Sjulian#include <sys/param.h> 3599026Sjulian#include <sys/systm.h> 3699026Sjulian#include <sys/kernel.h> 3799026Sjulian#include <sys/lock.h> 3899026Sjulian#include <sys/mutex.h> 3999026Sjulian#include <sys/proc.h> 40156705Sdavidxu#include <sys/resourcevar.h> 41130355Sjulian#include <sys/smp.h> 4299026Sjulian#include <sys/sysctl.h> 43107126Sjeff#include <sys/sched.h> 44126326Sjhb#include <sys/sleepqueue.h> 45174647Sjeff#include <sys/selinfo.h> 46122514Sjhb#include <sys/turnstile.h> 4799026Sjulian#include <sys/ktr.h> 48143149Sdavidxu#include <sys/umtx.h> 49176730Sjeff#include <sys/cpuset.h> 5099026Sjulian 51155195Srwatson#include <security/audit/audit.h> 52155195Srwatson 5399026Sjulian#include <vm/vm.h> 54116355Salc#include <vm/vm_extern.h> 5599026Sjulian#include <vm/uma.h> 56173631Srrs#include <sys/eventhandler.h> 5799026Sjulian 5899026Sjulian/* 59163709Sjb * thread related storage. 60163709Sjb */ 6199026Sjulianstatic uma_zone_t thread_zone; 6299026Sjulian 6399026SjulianSYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 6499026Sjulian 65130199Sjulianint max_threads_per_proc = 1500; 66107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 67103367Sjulian &max_threads_per_proc, 0, "Limit on threads per proc"); 68103367Sjulian 69130199Sjulianint max_threads_hits; 70111115SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 71111115Sdavidxu &max_threads_hits, 0, ""); 72111115Sdavidxu 73111028SjeffTAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 74172256Sattiliostatic struct mtx zombie_lock; 75170296SjeffMTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); 7699026Sjulian 77170598Sjeffstatic void thread_zombie(struct thread *); 78170598Sjeff 79127794Smarcelstruct mtx tid_lock; 80143802Sphkstatic struct unrhdr *tid_unrhdr; 81127794Smarcel 82127794Smarcel/* 83107719Sjulian * Prepare a thread for use. 8499026Sjulian */ 85132987Sgreenstatic int 86132987Sgreenthread_ctor(void *mem, int size, void *arg, int flags) 8799026Sjulian{ 8899026Sjulian struct thread *td; 8999026Sjulian 9099026Sjulian td = (struct thread *)mem; 91103216Sjulian td->td_state = TDS_INACTIVE; 92135573Sjhb td->td_oncpu = NOCPU; 93130269Sjmallett 94143840Sphk td->td_tid = alloc_unr(tid_unrhdr); 95167352Smohans td->td_syscalls = 0; 96143840Sphk 97130269Sjmallett /* 98130269Sjmallett * Note that td_critnest begins life as 1 because the thread is not 99130269Sjmallett * running and is thereby implicitly waiting to be on the receiving 100170296Sjeff * end of a context switch. 101130269Sjmallett */ 102118442Sjhb td->td_critnest = 1; 103173631Srrs EVENTHANDLER_INVOKE(thread_ctor, td); 104155195Srwatson#ifdef AUDIT 105155195Srwatson audit_thread_alloc(td); 106155195Srwatson#endif 107161678Sdavidxu umtx_thread_alloc(td); 108132987Sgreen return (0); 10999026Sjulian} 11099026Sjulian 11199026Sjulian/* 11299026Sjulian * Reclaim a thread after use. 11399026Sjulian */ 11499026Sjulianstatic void 11599026Sjulianthread_dtor(void *mem, int size, void *arg) 11699026Sjulian{ 117127794Smarcel struct thread *td; 11899026Sjulian 11999026Sjulian td = (struct thread *)mem; 12099026Sjulian 12199026Sjulian#ifdef INVARIANTS 12299026Sjulian /* Verify that this thread is in a safe state to free. */ 12399026Sjulian switch (td->td_state) { 124103216Sjulian case TDS_INHIBITED: 125103216Sjulian case TDS_RUNNING: 126103216Sjulian case TDS_CAN_RUN: 12799026Sjulian case TDS_RUNQ: 12899026Sjulian /* 12999026Sjulian * We must never unlink a thread that is in one of 13099026Sjulian * these states, because it is currently active. 13199026Sjulian */ 13299026Sjulian panic("bad state for thread unlinking"); 13399026Sjulian /* NOTREACHED */ 134103216Sjulian case TDS_INACTIVE: 13599026Sjulian break; 13699026Sjulian default: 13799026Sjulian panic("bad thread state"); 13899026Sjulian /* NOTREACHED */ 13999026Sjulian } 14099026Sjulian#endif 141155353Srwatson#ifdef AUDIT 142155353Srwatson audit_thread_free(td); 143155353Srwatson#endif 144173631Srrs EVENTHANDLER_INVOKE(thread_dtor, td); 145143840Sphk free_unr(tid_unrhdr, td->td_tid); 14699026Sjulian} 14799026Sjulian 14899026Sjulian/* 14999026Sjulian * Initialize type-stable parts of a thread (when newly created). 15099026Sjulian */ 151132987Sgreenstatic int 152132987Sgreenthread_init(void *mem, int size, int flags) 15399026Sjulian{ 154131149Smarcel struct thread *td; 15599026Sjulian 15699026Sjulian td = (struct thread *)mem; 157131149Smarcel 158126326Sjhb td->td_sleepqueue = sleepq_alloc(); 159122514Sjhb td->td_turnstile = turnstile_alloc(); 160173631Srrs EVENTHANDLER_INVOKE(thread_init, td); 161107126Sjeff td->td_sched = (struct td_sched *)&td[1]; 162161678Sdavidxu umtx_thread_init(td); 163173361Skib td->td_kstack = 0; 164132987Sgreen return (0); 16599026Sjulian} 16699026Sjulian 16799026Sjulian/* 16899026Sjulian * Tear down type-stable parts of a thread (just before being discarded). 16999026Sjulian */ 17099026Sjulianstatic void 17199026Sjulianthread_fini(void *mem, int size) 17299026Sjulian{ 173131149Smarcel struct thread *td; 17499026Sjulian 17599026Sjulian td = (struct thread *)mem; 176173631Srrs EVENTHANDLER_INVOKE(thread_fini, td); 177122514Sjhb turnstile_free(td->td_turnstile); 178126326Sjhb sleepq_free(td->td_sleepqueue); 179161678Sdavidxu umtx_thread_fini(td); 180174647Sjeff seltdfini(td); 18199026Sjulian} 182111028Sjeff 183107126Sjeff/* 184111028Sjeff * For a newly created process, 185111028Sjeff * link up all the structures and its initial threads etc. 186134791Sjulian * called from: 187134791Sjulian * {arch}/{arch}/machdep.c ia64_init(), init386() etc. 188134791Sjulian * proc_dtor() (should go away) 189134791Sjulian * proc_init() 190105854Sjulian */ 191105854Sjulianvoid 192173361Skibproc_linkup0(struct proc *p, struct thread *td) 193173361Skib{ 194173361Skib TAILQ_INIT(&p->p_threads); /* all threads in proc */ 195173361Skib proc_linkup(p, td); 196173361Skib} 197173361Skib 198173361Skibvoid 199163709Sjbproc_linkup(struct proc *p, struct thread *td) 200105854Sjulian{ 201170296Sjeff 202151316Sdavidxu sigqueue_init(&p->p_sigqueue, p); 203153253Sdavidxu p->p_ksi = ksiginfo_alloc(1); 204153253Sdavidxu if (p->p_ksi != NULL) { 205153253Sdavidxu /* XXX p_ksi may be null if ksiginfo zone is not ready */ 206153253Sdavidxu p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 207152185Sdavidxu } 208152948Sdavidxu LIST_INIT(&p->p_mqnotifier); 209105854Sjulian p->p_numthreads = 0; 210163709Sjb thread_link(td, p); 211105854Sjulian} 212105854Sjulian 213111028Sjeff/* 21499026Sjulian * Initialize global thread allocation resources. 21599026Sjulian */ 21699026Sjulianvoid 21799026Sjulianthreadinit(void) 21899026Sjulian{ 21999026Sjulian 220143802Sphk mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 221174848Sjulian /* leave one number for thread0 */ 222174848Sjulian tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock); 223143802Sphk 224107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 22599026Sjulian thread_ctor, thread_dtor, thread_init, thread_fini, 226167944Sjhb 16 - 1, 0); 22799026Sjulian} 22899026Sjulian 22999026Sjulian/* 230170598Sjeff * Place an unused thread on the zombie list. 231164936Sjulian * Use the slpq as that must be unused by now. 23299026Sjulian */ 23399026Sjulianvoid 234170598Sjeffthread_zombie(struct thread *td) 23599026Sjulian{ 236170296Sjeff mtx_lock_spin(&zombie_lock); 237164936Sjulian TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); 238170296Sjeff mtx_unlock_spin(&zombie_lock); 23999026Sjulian} 24099026Sjulian 241103410Smini/* 242170598Sjeff * Release a thread that has exited after cpu_throw(). 243170598Sjeff */ 244170598Sjeffvoid 245170598Sjeffthread_stash(struct thread *td) 246170598Sjeff{ 247170598Sjeff atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); 248170598Sjeff thread_zombie(td); 249170598Sjeff} 250170598Sjeff 251170598Sjeff/* 252177091Sjeff * Reap zombie resources. 25399026Sjulian */ 25499026Sjulianvoid 25599026Sjulianthread_reap(void) 25699026Sjulian{ 257105854Sjulian struct thread *td_first, *td_next; 25899026Sjulian 25999026Sjulian /* 260111028Sjeff * Don't even bother to lock if none at this instant, 261111028Sjeff * we really don't care about the next instant.. 26299026Sjulian */ 263163709Sjb if (!TAILQ_EMPTY(&zombie_threads)) { 264170296Sjeff mtx_lock_spin(&zombie_lock); 265105854Sjulian td_first = TAILQ_FIRST(&zombie_threads); 266105854Sjulian if (td_first) 267105854Sjulian TAILQ_INIT(&zombie_threads); 268170296Sjeff mtx_unlock_spin(&zombie_lock); 269105854Sjulian while (td_first) { 270164936Sjulian td_next = TAILQ_NEXT(td_first, td_slpq); 271111028Sjeff if (td_first->td_ucred) 272111028Sjeff crfree(td_first->td_ucred); 273105854Sjulian thread_free(td_first); 274105854Sjulian td_first = td_next; 27599026Sjulian } 27699026Sjulian } 27799026Sjulian} 27899026Sjulian 27999026Sjulian/* 28099026Sjulian * Allocate a thread. 28199026Sjulian */ 28299026Sjulianstruct thread * 28399026Sjulianthread_alloc(void) 28499026Sjulian{ 285173361Skib struct thread *td; 286163709Sjb 28799026Sjulian thread_reap(); /* check if any zombies to get */ 288173361Skib 289173361Skib td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK); 290173361Skib KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); 291173361Skib if (!vm_thread_new(td, 0)) { 292173361Skib uma_zfree(thread_zone, td); 293173361Skib return (NULL); 294173361Skib } 295173615Smarcel cpu_thread_alloc(td); 296173361Skib return (td); 29799026Sjulian} 29899026Sjulian 299103367Sjulian 300103367Sjulian/* 30199026Sjulian * Deallocate a thread. 30299026Sjulian */ 30399026Sjulianvoid 30499026Sjulianthread_free(struct thread *td) 30599026Sjulian{ 306177369Sjeff if (td->td_cpuset) 307177369Sjeff cpuset_rel(td->td_cpuset); 308176730Sjeff td->td_cpuset = NULL; 309173615Smarcel cpu_thread_free(td); 310173361Skib if (td->td_altkstack != 0) 311173361Skib vm_thread_dispose_altkstack(td); 312173361Skib if (td->td_kstack != 0) 313173361Skib vm_thread_dispose(td); 31499026Sjulian uma_zfree(thread_zone, td); 31599026Sjulian} 31699026Sjulian 31799026Sjulian/* 31899026Sjulian * Discard the current thread and exit from its context. 319130355Sjulian * Always called with scheduler locked. 32099026Sjulian * 32199026Sjulian * Because we can't free a thread while we're operating under its context, 322107719Sjulian * push the current thread into our CPU's deadthread holder. This means 323107719Sjulian * we needn't worry about someone else grabbing our context before we 324177091Sjeff * do a cpu_throw(). 32599026Sjulian */ 32699026Sjulianvoid 32799026Sjulianthread_exit(void) 32899026Sjulian{ 329156705Sdavidxu uint64_t new_switchtime; 33099026Sjulian struct thread *td; 331170174Sjeff struct thread *td2; 33299026Sjulian struct proc *p; 33399026Sjulian 33499026Sjulian td = curthread; 33599026Sjulian p = td->td_proc; 33699026Sjulian 337170296Sjeff PROC_SLOCK_ASSERT(p, MA_OWNED); 338134791Sjulian mtx_assert(&Giant, MA_NOTOWNED); 339170296Sjeff 340134791Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 341102581Sjulian KASSERT(p != NULL, ("thread exiting without a process")); 342133234Srwatson CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 343173601Sjulian (long)p->p_pid, td->td_name); 344151316Sdavidxu KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 34599026Sjulian 346155376Srwatson#ifdef AUDIT 347155376Srwatson AUDIT_SYSCALL_EXIT(0, td); 348155376Srwatson#endif 349161678Sdavidxu umtx_thread_exit(td); 350134791Sjulian /* 351134791Sjulian * drop FPU & debug register state storage, or any other 352134791Sjulian * architecture specific resources that 353134791Sjulian * would not be on a new untouched process. 354134791Sjulian */ 35599026Sjulian cpu_thread_exit(td); /* XXXSMP */ 35699026Sjulian 357156705Sdavidxu /* Do the same timestamp bookkeeping that mi_switch() would do. */ 358156705Sdavidxu new_switchtime = cpu_ticks(); 359156705Sdavidxu p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime)); 360156705Sdavidxu PCPU_SET(switchtime, new_switchtime); 361156705Sdavidxu PCPU_SET(switchticks, ticks); 362170292Sattilio PCPU_INC(cnt.v_swtch); 363170466Sattilio /* Save our resource usage in our process. */ 364170466Sattilio td->td_ru.ru_nvcsw++; 365170466Sattilio rucollect(&p->p_ru, &td->td_ru); 366134791Sjulian /* 367103002Sjulian * The last thread is left attached to the process 368103002Sjulian * So that the whole bundle gets recycled. Skip 369134791Sjulian * all this stuff if we never had threads. 370134791Sjulian * EXIT clears all sign of other threads when 371134791Sjulian * it goes to single threading, so the last thread always 372134791Sjulian * takes the short path. 373102581Sjulian */ 374134791Sjulian if (p->p_flag & P_HADTHREADS) { 375134791Sjulian if (p->p_numthreads > 1) { 376134791Sjulian thread_unlink(td); 377170174Sjeff td2 = FIRST_THREAD_IN_PROC(p); 378170174Sjeff sched_exit_thread(td2, td); 379134791Sjulian 380134791Sjulian /* 381134791Sjulian * The test below is NOT true if we are the 382134791Sjulian * sole exiting thread. P_STOPPED_SNGL is unset 383134791Sjulian * in exit1() after it is the only survivor. 384134791Sjulian */ 385134791Sjulian if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 386134791Sjulian if (p->p_numthreads == p->p_suspcount) { 387170296Sjeff thread_lock(p->p_singlethread); 388134791Sjulian thread_unsuspend_one(p->p_singlethread); 389170296Sjeff thread_unlock(p->p_singlethread); 390134791Sjulian } 391103002Sjulian } 392104695Sjulian 393170598Sjeff atomic_add_int(&td->td_proc->p_exitthreads, 1); 394134791Sjulian PCPU_SET(deadthread, td); 395134791Sjulian } else { 396134791Sjulian /* 397134791Sjulian * The last thread is exiting.. but not through exit() 398134791Sjulian */ 399134791Sjulian panic ("thread_exit: Last thread exiting on its own"); 400119488Sdavidxu } 401170296Sjeff } 402170296Sjeff PROC_UNLOCK(p); 403170296Sjeff thread_lock(td); 404170466Sattilio /* Save our tick information with both the thread and proc locked */ 405170296Sjeff ruxagg(&p->p_rux, td); 406170296Sjeff PROC_SUNLOCK(p); 407133396Sjulian td->td_state = TDS_INACTIVE; 408181695Sattilio#ifdef WITNESS 409181695Sattilio witness_thread_exit(td); 410181695Sattilio#endif 411133396Sjulian CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 412170296Sjeff sched_throw(td); 413112993Speter panic("I'm a teapot!"); 41499026Sjulian /* NOTREACHED */ 41599026Sjulian} 41699026Sjulian 417124350Sschweikh/* 418107719Sjulian * Do any thread specific cleanups that may be needed in wait() 419126932Speter * called with Giant, proc and schedlock not held. 420107719Sjulian */ 421107719Sjulianvoid 422107719Sjulianthread_wait(struct proc *p) 423107719Sjulian{ 424107719Sjulian struct thread *td; 425107719Sjulian 426126932Speter mtx_assert(&Giant, MA_NOTOWNED); 427124350Sschweikh KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 428170598Sjeff td = FIRST_THREAD_IN_PROC(p); 429170598Sjeff /* Lock the last thread so we spin until it exits cpu_throw(). */ 430170598Sjeff thread_lock(td); 431170598Sjeff thread_unlock(td); 432170598Sjeff /* Wait for any remaining threads to exit cpu_throw(). */ 433170598Sjeff while (p->p_exitthreads) 434170598Sjeff sched_relinquish(curthread); 435176730Sjeff cpuset_rel(td->td_cpuset); 436176730Sjeff td->td_cpuset = NULL; 437170598Sjeff cpu_thread_clean(td); 438170598Sjeff crfree(td->td_ucred); 439107719Sjulian thread_reap(); /* check for zombie threads etc. */ 440107719Sjulian} 441107719Sjulian 44299026Sjulian/* 44399026Sjulian * Link a thread to a process. 444103002Sjulian * set up anything that needs to be initialized for it to 445103002Sjulian * be used by the process. 44699026Sjulian */ 44799026Sjulianvoid 448163709Sjbthread_link(struct thread *td, struct proc *p) 44999026Sjulian{ 45099026Sjulian 451170296Sjeff /* 452170296Sjeff * XXX This can't be enabled because it's called for proc0 before 453177368Sjeff * its lock has been created. 454177368Sjeff * PROC_LOCK_ASSERT(p, MA_OWNED); 455170296Sjeff */ 456111028Sjeff td->td_state = TDS_INACTIVE; 457111028Sjeff td->td_proc = p; 458172207Sjeff td->td_flags = TDF_INMEM; 45999026Sjulian 460103002Sjulian LIST_INIT(&td->td_contested); 461174629Sjeff LIST_INIT(&td->td_lprof[0]); 462174629Sjeff LIST_INIT(&td->td_lprof[1]); 463151316Sdavidxu sigqueue_init(&td->td_sigqueue, p); 464119137Ssam callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 46599026Sjulian TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 46699026Sjulian p->p_numthreads++; 46799026Sjulian} 46899026Sjulian 469134791Sjulian/* 470136160Sjulian * Convert a process with one thread to an unthreaded process. 471136160Sjulian */ 472136160Sjulianvoid 473136160Sjulianthread_unthread(struct thread *td) 474136160Sjulian{ 475136160Sjulian struct proc *p = td->td_proc; 476136160Sjulian 477136160Sjulian KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); 478163709Sjb p->p_flag &= ~P_HADTHREADS; 479136160Sjulian} 480136160Sjulian 481136160Sjulian/* 482136160Sjulian * Called from: 483134791Sjulian * thread_exit() 484134791Sjulian */ 485113641Sjulianvoid 486113641Sjulianthread_unlink(struct thread *td) 487124350Sschweikh{ 488113641Sjulian struct proc *p = td->td_proc; 489113920Sjhb 490177368Sjeff PROC_LOCK_ASSERT(p, MA_OWNED); 491113641Sjulian TAILQ_REMOVE(&p->p_threads, td, td_plist); 492113641Sjulian p->p_numthreads--; 493113641Sjulian /* could clear a few other things here */ 494163709Sjb /* Must NOT clear links to proc! */ 495124350Sschweikh} 496113641Sjulian 497111028Sjeff/* 49899026Sjulian * Enforce single-threading. 49999026Sjulian * 50099026Sjulian * Returns 1 if the caller must abort (another thread is waiting to 50199026Sjulian * exit the process or similar). Process is locked! 50299026Sjulian * Returns 0 when you are successfully the only thread running. 50399026Sjulian * A process has successfully single threaded in the suspend mode when 50499026Sjulian * There are no threads in user mode. Threads in the kernel must be 50599026Sjulian * allowed to continue until they get to the user boundary. They may even 50699026Sjulian * copy out their return values and data before suspending. They may however be 507160048Smaxim * accelerated in reaching the user boundary as we will wake up 50899026Sjulian * any sleeping threads that are interruptable. (PCATCH). 50999026Sjulian */ 51099026Sjulianint 511136177Sdavidxuthread_single(int mode) 51299026Sjulian{ 51399026Sjulian struct thread *td; 51499026Sjulian struct thread *td2; 51599026Sjulian struct proc *p; 516181334Sjhb int remaining, wakeup_swapper; 51799026Sjulian 51899026Sjulian td = curthread; 51999026Sjulian p = td->td_proc; 520126932Speter mtx_assert(&Giant, MA_NOTOWNED); 52199026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 52299026Sjulian KASSERT((td != NULL), ("curthread is NULL")); 52399026Sjulian 524134791Sjulian if ((p->p_flag & P_HADTHREADS) == 0) 52599026Sjulian return (0); 52699026Sjulian 527100648Sjulian /* Is someone already single threading? */ 528136177Sdavidxu if (p->p_singlethread != NULL && p->p_singlethread != td) 52999026Sjulian return (1); 53099026Sjulian 531136177Sdavidxu if (mode == SINGLE_EXIT) { 532136177Sdavidxu p->p_flag |= P_SINGLE_EXIT; 533136177Sdavidxu p->p_flag &= ~P_SINGLE_BOUNDARY; 534136177Sdavidxu } else { 535136177Sdavidxu p->p_flag &= ~P_SINGLE_EXIT; 536136177Sdavidxu if (mode == SINGLE_BOUNDARY) 537136177Sdavidxu p->p_flag |= P_SINGLE_BOUNDARY; 538136177Sdavidxu else 539136177Sdavidxu p->p_flag &= ~P_SINGLE_BOUNDARY; 540136177Sdavidxu } 541102950Sdavidxu p->p_flag |= P_STOPPED_SINGLE; 542170296Sjeff PROC_SLOCK(p); 54399026Sjulian p->p_singlethread = td; 544136177Sdavidxu if (mode == SINGLE_EXIT) 545130674Sdavidxu remaining = p->p_numthreads; 546136177Sdavidxu else if (mode == SINGLE_BOUNDARY) 547136177Sdavidxu remaining = p->p_numthreads - p->p_boundary_count; 548136177Sdavidxu else 549130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 550130674Sdavidxu while (remaining != 1) { 551156942Sdavidxu if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 552156942Sdavidxu goto stopme; 553181334Sjhb wakeup_swapper = 0; 55499026Sjulian FOREACH_THREAD_IN_PROC(p, td2) { 55599026Sjulian if (td2 == td) 55699026Sjulian continue; 557170296Sjeff thread_lock(td2); 558177471Sjeff td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK; 559103216Sjulian if (TD_IS_INHIBITED(td2)) { 560136177Sdavidxu switch (mode) { 561136177Sdavidxu case SINGLE_EXIT: 562132087Sdavidxu if (td->td_flags & TDF_DBSUSPEND) 563132087Sdavidxu td->td_flags &= ~TDF_DBSUSPEND; 564136177Sdavidxu if (TD_IS_SUSPENDED(td2)) 565103216Sjulian thread_unsuspend_one(td2); 566105911Sjulian if (TD_ON_SLEEPQ(td2) && 567136177Sdavidxu (td2->td_flags & TDF_SINTR)) 568181334Sjhb wakeup_swapper = 569181334Sjhb sleepq_abort(td2, EINTR); 570136177Sdavidxu break; 571136177Sdavidxu case SINGLE_BOUNDARY: 572136177Sdavidxu break; 573136177Sdavidxu default: 574170296Sjeff if (TD_IS_SUSPENDED(td2)) { 575170296Sjeff thread_unlock(td2); 576105874Sdavidxu continue; 577170296Sjeff } 578111028Sjeff /* 579165693Srwatson * maybe other inhibited states too? 580111028Sjeff */ 581137281Sdavidxu if ((td2->td_flags & TDF_SINTR) && 582137281Sdavidxu (td2->td_inhibitors & 583137281Sdavidxu (TDI_SLEEPING | TDI_SWAPPED))) 584105911Sjulian thread_suspend_one(td2); 585136177Sdavidxu break; 58699026Sjulian } 58799026Sjulian } 588155594Sdavidxu#ifdef SMP 589155594Sdavidxu else if (TD_IS_RUNNING(td2) && td != td2) { 590155594Sdavidxu forward_signal(td2); 591155594Sdavidxu } 592155594Sdavidxu#endif 593170296Sjeff thread_unlock(td2); 59499026Sjulian } 595181334Sjhb if (wakeup_swapper) 596181334Sjhb kick_proc0(); 597136177Sdavidxu if (mode == SINGLE_EXIT) 598130674Sdavidxu remaining = p->p_numthreads; 599136177Sdavidxu else if (mode == SINGLE_BOUNDARY) 600136177Sdavidxu remaining = p->p_numthreads - p->p_boundary_count; 601130674Sdavidxu else 602130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 603130674Sdavidxu 604124350Sschweikh /* 605124350Sschweikh * Maybe we suspended some threads.. was it enough? 606105911Sjulian */ 607130674Sdavidxu if (remaining == 1) 608105911Sjulian break; 609105911Sjulian 610156942Sdavidxustopme: 61199026Sjulian /* 61299026Sjulian * Wake us up when everyone else has suspended. 613100648Sjulian * In the mean time we suspend as well. 61499026Sjulian */ 615170296Sjeff thread_suspend_switch(td); 616136177Sdavidxu if (mode == SINGLE_EXIT) 617130674Sdavidxu remaining = p->p_numthreads; 618136177Sdavidxu else if (mode == SINGLE_BOUNDARY) 619136177Sdavidxu remaining = p->p_numthreads - p->p_boundary_count; 620130674Sdavidxu else 621130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 62299026Sjulian } 623136177Sdavidxu if (mode == SINGLE_EXIT) { 624135269Sjulian /* 625135269Sjulian * We have gotten rid of all the other threads and we 626135269Sjulian * are about to either exit or exec. In either case, 627135269Sjulian * we try our utmost to revert to being a non-threaded 628135269Sjulian * process. 629135269Sjulian */ 630136160Sjulian p->p_singlethread = NULL; 631137279Sdavidxu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 632136160Sjulian thread_unthread(td); 633111028Sjeff } 634170296Sjeff PROC_SUNLOCK(p); 63599026Sjulian return (0); 63699026Sjulian} 63799026Sjulian 63899026Sjulian/* 63999026Sjulian * Called in from locations that can safely check to see 64099026Sjulian * whether we have to suspend or at least throttle for a 64199026Sjulian * single-thread event (e.g. fork). 64299026Sjulian * 64399026Sjulian * Such locations include userret(). 64499026Sjulian * If the "return_instead" argument is non zero, the thread must be able to 64599026Sjulian * accept 0 (caller may continue), or 1 (caller must abort) as a result. 64699026Sjulian * 64799026Sjulian * The 'return_instead' argument tells the function if it may do a 64899026Sjulian * thread_exit() or suspend, or whether the caller must abort and back 64999026Sjulian * out instead. 65099026Sjulian * 65199026Sjulian * If the thread that set the single_threading request has set the 65299026Sjulian * P_SINGLE_EXIT bit in the process flags then this call will never return 65399026Sjulian * if 'return_instead' is false, but will exit. 65499026Sjulian * 65599026Sjulian * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 65699026Sjulian *---------------+--------------------+--------------------- 65799026Sjulian * 0 | returns 0 | returns 0 or 1 65899026Sjulian * | when ST ends | immediatly 65999026Sjulian *---------------+--------------------+--------------------- 66099026Sjulian * 1 | thread exits | returns 1 66199026Sjulian * | | immediatly 66299026Sjulian * 0 = thread_exit() or suspension ok, 66399026Sjulian * other = return error instead of stopping the thread. 66499026Sjulian * 66599026Sjulian * While a full suspension is under effect, even a single threading 66699026Sjulian * thread would be suspended if it made this call (but it shouldn't). 66799026Sjulian * This call should only be made from places where 668124350Sschweikh * thread_exit() would be safe as that may be the outcome unless 66999026Sjulian * return_instead is set. 67099026Sjulian */ 67199026Sjulianint 67299026Sjulianthread_suspend_check(int return_instead) 67399026Sjulian{ 674104502Sjmallett struct thread *td; 675104502Sjmallett struct proc *p; 67699026Sjulian 67799026Sjulian td = curthread; 67899026Sjulian p = td->td_proc; 679126932Speter mtx_assert(&Giant, MA_NOTOWNED); 68099026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 681132087Sdavidxu while (P_SHOULDSTOP(p) || 682132087Sdavidxu ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) { 683102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 68499026Sjulian KASSERT(p->p_singlethread != NULL, 68599026Sjulian ("singlethread not set")); 68699026Sjulian /* 687100648Sjulian * The only suspension in action is a 688100648Sjulian * single-threading. Single threader need not stop. 689124350Sschweikh * XXX Should be safe to access unlocked 690100646Sjulian * as it can only be set to be true by us. 69199026Sjulian */ 692100648Sjulian if (p->p_singlethread == td) 69399026Sjulian return (0); /* Exempt from stopping. */ 694124350Sschweikh } 695134498Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 696155741Sdavidxu return (EINTR); 69799026Sjulian 698136177Sdavidxu /* Should we goto user boundary if we didn't come from there? */ 699136177Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 700136177Sdavidxu (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 701155741Sdavidxu return (ERESTART); 702136177Sdavidxu 703151316Sdavidxu /* If thread will exit, flush its pending signals */ 704151316Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 705151316Sdavidxu sigqueue_flush(&td->td_sigqueue); 706151316Sdavidxu 707170296Sjeff PROC_SLOCK(p); 708112071Sdavidxu thread_stopped(p); 70999026Sjulian /* 71099026Sjulian * If the process is waiting for us to exit, 71199026Sjulian * this thread should just suicide. 712102950Sdavidxu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 71399026Sjulian */ 714136177Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 715134791Sjulian thread_exit(); 716170296Sjeff if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 717170296Sjeff if (p->p_numthreads == p->p_suspcount + 1) { 718170296Sjeff thread_lock(p->p_singlethread); 719170296Sjeff thread_unsuspend_one(p->p_singlethread); 720170296Sjeff thread_unlock(p->p_singlethread); 721170296Sjeff } 722170296Sjeff } 723170296Sjeff PROC_UNLOCK(p); 724170296Sjeff thread_lock(td); 72599026Sjulian /* 72699026Sjulian * When a thread suspends, it just 727164936Sjulian * gets taken off all queues. 72899026Sjulian */ 729103216Sjulian thread_suspend_one(td); 730136177Sdavidxu if (return_instead == 0) { 731136177Sdavidxu p->p_boundary_count++; 732136177Sdavidxu td->td_flags |= TDF_BOUNDARY; 733136177Sdavidxu } 734170296Sjeff PROC_SUNLOCK(p); 735178272Sjeff mi_switch(SW_INVOL | SWT_SUSPEND, NULL); 736170296Sjeff if (return_instead == 0) 737136177Sdavidxu td->td_flags &= ~TDF_BOUNDARY; 738170296Sjeff thread_unlock(td); 73999026Sjulian PROC_LOCK(p); 740170296Sjeff if (return_instead == 0) 741170296Sjeff p->p_boundary_count--; 74299026Sjulian } 74399026Sjulian return (0); 74499026Sjulian} 74599026Sjulian 746102898Sdavidxuvoid 747170296Sjeffthread_suspend_switch(struct thread *td) 748170296Sjeff{ 749170296Sjeff struct proc *p; 750170296Sjeff 751170296Sjeff p = td->td_proc; 752170296Sjeff KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 753170296Sjeff PROC_LOCK_ASSERT(p, MA_OWNED); 754170296Sjeff PROC_SLOCK_ASSERT(p, MA_OWNED); 755170296Sjeff /* 756170296Sjeff * We implement thread_suspend_one in stages here to avoid 757170296Sjeff * dropping the proc lock while the thread lock is owned. 758170296Sjeff */ 759170296Sjeff thread_stopped(p); 760170296Sjeff p->p_suspcount++; 761170296Sjeff PROC_UNLOCK(p); 762170296Sjeff thread_lock(td); 763177471Sjeff td->td_flags &= ~TDF_NEEDSUSPCHK; 764170296Sjeff TD_SET_SUSPENDED(td); 765177085Sjeff sched_sleep(td, 0); 766170296Sjeff PROC_SUNLOCK(p); 767170296Sjeff DROP_GIANT(); 768178272Sjeff mi_switch(SW_VOL | SWT_SUSPEND, NULL); 769170296Sjeff thread_unlock(td); 770170296Sjeff PICKUP_GIANT(); 771170296Sjeff PROC_LOCK(p); 772170296Sjeff PROC_SLOCK(p); 773170296Sjeff} 774170296Sjeff 775170296Sjeffvoid 776102898Sdavidxuthread_suspend_one(struct thread *td) 777102898Sdavidxu{ 778102898Sdavidxu struct proc *p = td->td_proc; 779102898Sdavidxu 780170296Sjeff PROC_SLOCK_ASSERT(p, MA_OWNED); 781170296Sjeff THREAD_LOCK_ASSERT(td, MA_OWNED); 782112071Sdavidxu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 783102898Sdavidxu p->p_suspcount++; 784177471Sjeff td->td_flags &= ~TDF_NEEDSUSPCHK; 785103216Sjulian TD_SET_SUSPENDED(td); 786177085Sjeff sched_sleep(td, 0); 787102898Sdavidxu} 788102898Sdavidxu 789102898Sdavidxuvoid 790102898Sdavidxuthread_unsuspend_one(struct thread *td) 791102898Sdavidxu{ 792102898Sdavidxu struct proc *p = td->td_proc; 793102898Sdavidxu 794170296Sjeff PROC_SLOCK_ASSERT(p, MA_OWNED); 795170296Sjeff THREAD_LOCK_ASSERT(td, MA_OWNED); 796164936Sjulian KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 797103216Sjulian TD_CLR_SUSPENDED(td); 798102898Sdavidxu p->p_suspcount--; 799181334Sjhb if (setrunnable(td)) { 800181334Sjhb#ifdef INVARIANTS 801181334Sjhb panic("not waking up swapper"); 802181334Sjhb#endif 803181334Sjhb } 804102898Sdavidxu} 805102898Sdavidxu 80699026Sjulian/* 80799026Sjulian * Allow all threads blocked by single threading to continue running. 80899026Sjulian */ 80999026Sjulianvoid 81099026Sjulianthread_unsuspend(struct proc *p) 81199026Sjulian{ 81299026Sjulian struct thread *td; 81399026Sjulian 81499026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 815170296Sjeff PROC_SLOCK_ASSERT(p, MA_OWNED); 81699026Sjulian if (!P_SHOULDSTOP(p)) { 817164936Sjulian FOREACH_THREAD_IN_PROC(p, td) { 818170296Sjeff thread_lock(td); 819164936Sjulian if (TD_IS_SUSPENDED(td)) { 820164936Sjulian thread_unsuspend_one(td); 821164936Sjulian } 822170296Sjeff thread_unlock(td); 82399026Sjulian } 824102950Sdavidxu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 82599026Sjulian (p->p_numthreads == p->p_suspcount)) { 82699026Sjulian /* 82799026Sjulian * Stopping everything also did the job for the single 82899026Sjulian * threading request. Now we've downgraded to single-threaded, 82999026Sjulian * let it continue. 83099026Sjulian */ 831170296Sjeff thread_lock(p->p_singlethread); 832102898Sdavidxu thread_unsuspend_one(p->p_singlethread); 833170296Sjeff thread_unlock(p->p_singlethread); 83499026Sjulian } 83599026Sjulian} 83699026Sjulian 837134791Sjulian/* 838134791Sjulian * End the single threading mode.. 839134791Sjulian */ 84099026Sjulianvoid 84199026Sjulianthread_single_end(void) 84299026Sjulian{ 84399026Sjulian struct thread *td; 84499026Sjulian struct proc *p; 84599026Sjulian 84699026Sjulian td = curthread; 84799026Sjulian p = td->td_proc; 84899026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 849136177Sdavidxu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); 850170296Sjeff PROC_SLOCK(p); 85199026Sjulian p->p_singlethread = NULL; 852102292Sjulian /* 853102292Sjulian * If there are other threads they mey now run, 854102292Sjulian * unless of course there is a blanket 'stop order' 855102292Sjulian * on the process. The single threader must be allowed 856102292Sjulian * to continue however as this is a bad place to stop. 857102292Sjulian */ 858102292Sjulian if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 859164936Sjulian FOREACH_THREAD_IN_PROC(p, td) { 860170296Sjeff thread_lock(td); 861164936Sjulian if (TD_IS_SUSPENDED(td)) { 862164936Sjulian thread_unsuspend_one(td); 863164936Sjulian } 864170296Sjeff thread_unlock(td); 865102292Sjulian } 866102292Sjulian } 867170296Sjeff PROC_SUNLOCK(p); 86899026Sjulian} 869128721Sdeischen 870151990Sdavidxustruct thread * 871151990Sdavidxuthread_find(struct proc *p, lwpid_t tid) 872151990Sdavidxu{ 873151990Sdavidxu struct thread *td; 874151990Sdavidxu 875151990Sdavidxu PROC_LOCK_ASSERT(p, MA_OWNED); 876151990Sdavidxu FOREACH_THREAD_IN_PROC(p, td) { 877151990Sdavidxu if (td->td_tid == tid) 878151990Sdavidxu break; 879151990Sdavidxu } 880151990Sdavidxu return (td); 881151990Sdavidxu} 882