kern_thread.c revision 170292
1139804Simp/*- 299026Sjulian * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 399026Sjulian * All rights reserved. 499026Sjulian * 599026Sjulian * Redistribution and use in source and binary forms, with or without 699026Sjulian * modification, are permitted provided that the following conditions 799026Sjulian * are met: 899026Sjulian * 1. Redistributions of source code must retain the above copyright 999026Sjulian * notice(s), this list of conditions and the following disclaimer as 10124350Sschweikh * the first lines of this file unmodified other than the possible 1199026Sjulian * addition of one or more copyright notices. 1299026Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1399026Sjulian * notice(s), this list of conditions and the following disclaimer in the 1499026Sjulian * documentation and/or other materials provided with the distribution. 1599026Sjulian * 1699026Sjulian * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1799026Sjulian * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1899026Sjulian * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1999026Sjulian * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2099026Sjulian * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2199026Sjulian * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2299026Sjulian * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2399026Sjulian * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2499026Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2599026Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2699026Sjulian * DAMAGE. 2799026Sjulian */ 2899026Sjulian 29116182Sobrien#include <sys/cdefs.h> 30116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_thread.c 170292 2007-06-04 21:45:18Z attilio $"); 31116182Sobrien 3299026Sjulian#include <sys/param.h> 3399026Sjulian#include <sys/systm.h> 3499026Sjulian#include <sys/kernel.h> 3599026Sjulian#include <sys/lock.h> 3699026Sjulian#include <sys/mutex.h> 3799026Sjulian#include <sys/proc.h> 38156705Sdavidxu#include <sys/resourcevar.h> 39130355Sjulian#include <sys/smp.h> 4099026Sjulian#include <sys/sysctl.h> 41107126Sjeff#include <sys/sched.h> 42126326Sjhb#include <sys/sleepqueue.h> 43122514Sjhb#include <sys/turnstile.h> 4499026Sjulian#include <sys/ktr.h> 45143149Sdavidxu#include <sys/umtx.h> 4699026Sjulian 47155195Srwatson#include <security/audit/audit.h> 48155195Srwatson 4999026Sjulian#include <vm/vm.h> 50116355Salc#include <vm/vm_extern.h> 5199026Sjulian#include <vm/uma.h> 5299026Sjulian 5399026Sjulian/* 54163709Sjb * thread related storage. 55163709Sjb */ 5699026Sjulianstatic uma_zone_t thread_zone; 5799026Sjulian 5899026SjulianSYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 5999026Sjulian 60130199Sjulianint max_threads_per_proc = 1500; 61107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 62103367Sjulian &max_threads_per_proc, 0, "Limit on threads per proc"); 63103367Sjulian 64130199Sjulianint max_threads_hits; 65111115SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 66111115Sdavidxu &max_threads_hits, 0, ""); 67111115Sdavidxu 68163709Sjb#ifdef KSE 69130355Sjulianint virtual_cpu; 70111028Sjeff 71163709Sjb#endif 72111028SjeffTAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 73111028Sjeffstruct mtx kse_zombie_lock; 74111028SjeffMTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 7599026Sjulian 76163709Sjb#ifdef KSE 77130355Sjulianstatic int 78130355Sjuliansysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 79130355Sjulian{ 80130355Sjulian int error, new_val; 81130355Sjulian int def_val; 82111028Sjeff 83130355Sjulian def_val = mp_ncpus; 84130355Sjulian if (virtual_cpu == 0) 85130355Sjulian new_val = def_val; 86130355Sjulian else 87130355Sjulian new_val = virtual_cpu; 88130355Sjulian error = sysctl_handle_int(oidp, &new_val, 0, req); 89133713Sjulian if (error != 0 || req->newptr == NULL) 90130355Sjulian return (error); 91130355Sjulian if (new_val < 0) 92130355Sjulian return (EINVAL); 93130355Sjulian virtual_cpu = new_val; 94130355Sjulian return (0); 95130355Sjulian} 96130355Sjulian 97130355Sjulian/* DEBUG ONLY */ 98130355SjulianSYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 99130355Sjulian 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 100130355Sjulian "debug virtual cpus"); 101163709Sjb#endif 102130355Sjulian 103127794Smarcelstruct mtx tid_lock; 104143802Sphkstatic struct unrhdr *tid_unrhdr; 105127794Smarcel 106127794Smarcel/* 107107719Sjulian * Prepare a thread for use. 10899026Sjulian */ 109132987Sgreenstatic int 110132987Sgreenthread_ctor(void *mem, int size, void *arg, int flags) 11199026Sjulian{ 11299026Sjulian struct thread *td; 11399026Sjulian 11499026Sjulian td = (struct thread *)mem; 115103216Sjulian td->td_state = TDS_INACTIVE; 116135573Sjhb td->td_oncpu = NOCPU; 117130269Sjmallett 118143840Sphk td->td_tid = alloc_unr(tid_unrhdr); 119167352Smohans td->td_syscalls = 0; 120143840Sphk 121130269Sjmallett /* 122130269Sjmallett * Note that td_critnest begins life as 1 because the thread is not 123130269Sjmallett * running and is thereby implicitly waiting to be on the receiving 124130269Sjmallett * end of a context switch. A context switch must occur inside a 125130269Sjmallett * critical section, and in fact, includes hand-off of the sched_lock. 126130269Sjmallett * After a context switch to a newly created thread, it will release 127130269Sjmallett * sched_lock for the first time, and its td_critnest will hit 0 for 128130269Sjmallett * the first time. This happens on the far end of a context switch, 129130269Sjmallett * and when it context switches away from itself, it will in fact go 130130269Sjmallett * back into a critical section, and hand off the sched lock to the 131130269Sjmallett * next thread. 132130269Sjmallett */ 133118442Sjhb td->td_critnest = 1; 134155195Srwatson 135155195Srwatson#ifdef AUDIT 136155195Srwatson audit_thread_alloc(td); 137155195Srwatson#endif 138161678Sdavidxu umtx_thread_alloc(td); 139132987Sgreen return (0); 14099026Sjulian} 14199026Sjulian 14299026Sjulian/* 14399026Sjulian * Reclaim a thread after use. 14499026Sjulian */ 14599026Sjulianstatic void 14699026Sjulianthread_dtor(void *mem, int size, void *arg) 14799026Sjulian{ 148127794Smarcel struct thread *td; 14999026Sjulian 15099026Sjulian td = (struct thread *)mem; 15199026Sjulian 15299026Sjulian#ifdef INVARIANTS 15399026Sjulian /* Verify that this thread is in a safe state to free. */ 15499026Sjulian switch (td->td_state) { 155103216Sjulian case TDS_INHIBITED: 156103216Sjulian case TDS_RUNNING: 157103216Sjulian case TDS_CAN_RUN: 15899026Sjulian case TDS_RUNQ: 15999026Sjulian /* 16099026Sjulian * We must never unlink a thread that is in one of 16199026Sjulian * these states, because it is currently active. 16299026Sjulian */ 16399026Sjulian panic("bad state for thread unlinking"); 16499026Sjulian /* NOTREACHED */ 165103216Sjulian case TDS_INACTIVE: 16699026Sjulian break; 16799026Sjulian default: 16899026Sjulian panic("bad thread state"); 16999026Sjulian /* NOTREACHED */ 17099026Sjulian } 17199026Sjulian#endif 172155353Srwatson#ifdef AUDIT 173155353Srwatson audit_thread_free(td); 174155353Srwatson#endif 175143840Sphk free_unr(tid_unrhdr, td->td_tid); 176134791Sjulian sched_newthread(td); 17799026Sjulian} 17899026Sjulian 17999026Sjulian/* 18099026Sjulian * Initialize type-stable parts of a thread (when newly created). 18199026Sjulian */ 182132987Sgreenstatic int 183132987Sgreenthread_init(void *mem, int size, int flags) 18499026Sjulian{ 185131149Smarcel struct thread *td; 18699026Sjulian 18799026Sjulian td = (struct thread *)mem; 188131149Smarcel 189116355Salc vm_thread_new(td, 0); 19099026Sjulian cpu_thread_setup(td); 191126326Sjhb td->td_sleepqueue = sleepq_alloc(); 192122514Sjhb td->td_turnstile = turnstile_alloc(); 193107126Sjeff td->td_sched = (struct td_sched *)&td[1]; 194134791Sjulian sched_newthread(td); 195161678Sdavidxu umtx_thread_init(td); 196132987Sgreen return (0); 19799026Sjulian} 19899026Sjulian 19999026Sjulian/* 20099026Sjulian * Tear down type-stable parts of a thread (just before being discarded). 20199026Sjulian */ 20299026Sjulianstatic void 20399026Sjulianthread_fini(void *mem, int size) 20499026Sjulian{ 205131149Smarcel struct thread *td; 20699026Sjulian 20799026Sjulian td = (struct thread *)mem; 208122514Sjhb turnstile_free(td->td_turnstile); 209126326Sjhb sleepq_free(td->td_sleepqueue); 210161678Sdavidxu umtx_thread_fini(td); 211116355Salc vm_thread_dispose(td); 21299026Sjulian} 213111028Sjeff 214107126Sjeff/* 215111028Sjeff * For a newly created process, 216111028Sjeff * link up all the structures and its initial threads etc. 217134791Sjulian * called from: 218134791Sjulian * {arch}/{arch}/machdep.c ia64_init(), init386() etc. 219134791Sjulian * proc_dtor() (should go away) 220134791Sjulian * proc_init() 221105854Sjulian */ 222105854Sjulianvoid 223163709Sjbproc_linkup(struct proc *p, struct thread *td) 224105854Sjulian{ 225105854Sjulian TAILQ_INIT(&p->p_threads); /* all threads in proc */ 226164936Sjulian TAILQ_INIT(&p->p_upcalls); /* upcall list */ 227151316Sdavidxu sigqueue_init(&p->p_sigqueue, p); 228153253Sdavidxu p->p_ksi = ksiginfo_alloc(1); 229153253Sdavidxu if (p->p_ksi != NULL) { 230153253Sdavidxu /* XXX p_ksi may be null if ksiginfo zone is not ready */ 231153253Sdavidxu p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 232152185Sdavidxu } 233152948Sdavidxu LIST_INIT(&p->p_mqnotifier); 234105854Sjulian p->p_numthreads = 0; 235163709Sjb thread_link(td, p); 236105854Sjulian} 237105854Sjulian 238111028Sjeff/* 23999026Sjulian * Initialize global thread allocation resources. 24099026Sjulian */ 24199026Sjulianvoid 24299026Sjulianthreadinit(void) 24399026Sjulian{ 24499026Sjulian 245143802Sphk mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 246143802Sphk tid_unrhdr = new_unrhdr(PID_MAX + 1, INT_MAX, &tid_lock); 247143802Sphk 248107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 24999026Sjulian thread_ctor, thread_dtor, thread_init, thread_fini, 250167944Sjhb 16 - 1, 0); 251163709Sjb#ifdef KSE 252134791Sjulian kseinit(); /* set up kse specific stuff e.g. upcall zone*/ 253163709Sjb#endif 25499026Sjulian} 25599026Sjulian 25699026Sjulian/* 257103002Sjulian * Stash an embarasingly extra thread into the zombie thread queue. 258164936Sjulian * Use the slpq as that must be unused by now. 25999026Sjulian */ 26099026Sjulianvoid 26199026Sjulianthread_stash(struct thread *td) 26299026Sjulian{ 263111028Sjeff mtx_lock_spin(&kse_zombie_lock); 264164936Sjulian TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); 265111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 26699026Sjulian} 26799026Sjulian 268103410Smini/* 269111028Sjeff * Reap zombie kse resource. 27099026Sjulian */ 27199026Sjulianvoid 27299026Sjulianthread_reap(void) 27399026Sjulian{ 274105854Sjulian struct thread *td_first, *td_next; 27599026Sjulian 27699026Sjulian /* 277111028Sjeff * Don't even bother to lock if none at this instant, 278111028Sjeff * we really don't care about the next instant.. 27999026Sjulian */ 280163709Sjb if (!TAILQ_EMPTY(&zombie_threads)) { 281111028Sjeff mtx_lock_spin(&kse_zombie_lock); 282105854Sjulian td_first = TAILQ_FIRST(&zombie_threads); 283105854Sjulian if (td_first) 284105854Sjulian TAILQ_INIT(&zombie_threads); 285111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 286105854Sjulian while (td_first) { 287164936Sjulian td_next = TAILQ_NEXT(td_first, td_slpq); 288111028Sjeff if (td_first->td_ucred) 289111028Sjeff crfree(td_first->td_ucred); 290105854Sjulian thread_free(td_first); 291105854Sjulian td_first = td_next; 29299026Sjulian } 29399026Sjulian } 29499026Sjulian} 29599026Sjulian 29699026Sjulian/* 29799026Sjulian * Allocate a thread. 29899026Sjulian */ 29999026Sjulianstruct thread * 30099026Sjulianthread_alloc(void) 30199026Sjulian{ 302163709Sjb 30399026Sjulian thread_reap(); /* check if any zombies to get */ 304111119Simp return (uma_zalloc(thread_zone, M_WAITOK)); 30599026Sjulian} 30699026Sjulian 307103367Sjulian 308103367Sjulian/* 30999026Sjulian * Deallocate a thread. 31099026Sjulian */ 31199026Sjulianvoid 31299026Sjulianthread_free(struct thread *td) 31399026Sjulian{ 314107719Sjulian 315107719Sjulian cpu_thread_clean(td); 31699026Sjulian uma_zfree(thread_zone, td); 31799026Sjulian} 31899026Sjulian 31999026Sjulian/* 32099026Sjulian * Discard the current thread and exit from its context. 321130355Sjulian * Always called with scheduler locked. 32299026Sjulian * 32399026Sjulian * Because we can't free a thread while we're operating under its context, 324107719Sjulian * push the current thread into our CPU's deadthread holder. This means 325107719Sjulian * we needn't worry about someone else grabbing our context before we 326130355Sjulian * do a cpu_throw(). This may not be needed now as we are under schedlock. 327130355Sjulian * Maybe we can just do a thread_stash() as thr_exit1 does. 32899026Sjulian */ 329130355Sjulian/* XXX 330130355Sjulian * libthr expects its thread exit to return for the last 331130355Sjulian * thread, meaning that the program is back to non-threaded 332130355Sjulian * mode I guess. Because we do this (cpu_throw) unconditionally 333130355Sjulian * here, they have their own version of it. (thr_exit1()) 334130355Sjulian * that doesn't do it all if this was the last thread. 335130355Sjulian * It is also called from thread_suspend_check(). 336130355Sjulian * Of course in the end, they end up coming here through exit1 337130355Sjulian * anyhow.. After fixing 'thr' to play by the rules we should be able 338130355Sjulian * to merge these two functions together. 339134791Sjulian * 340134791Sjulian * called from: 341134791Sjulian * exit1() 342134791Sjulian * kse_exit() 343134791Sjulian * thr_exit() 344163709Sjb * ifdef KSE 345134791Sjulian * thread_user_enter() 346134791Sjulian * thread_userret() 347163709Sjb * endif 348134791Sjulian * thread_suspend_check() 349130355Sjulian */ 35099026Sjulianvoid 35199026Sjulianthread_exit(void) 35299026Sjulian{ 353156705Sdavidxu uint64_t new_switchtime; 35499026Sjulian struct thread *td; 355170174Sjeff struct thread *td2; 35699026Sjulian struct proc *p; 35799026Sjulian 35899026Sjulian td = curthread; 35999026Sjulian p = td->td_proc; 36099026Sjulian 36199026Sjulian mtx_assert(&sched_lock, MA_OWNED); 362134791Sjulian mtx_assert(&Giant, MA_NOTOWNED); 363134791Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 364102581Sjulian KASSERT(p != NULL, ("thread exiting without a process")); 365133234Srwatson CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 366133234Srwatson (long)p->p_pid, p->p_comm); 367151316Sdavidxu KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 36899026Sjulian 369155376Srwatson#ifdef AUDIT 370155376Srwatson AUDIT_SYSCALL_EXIT(0, td); 371155376Srwatson#endif 372155376Srwatson 373163709Sjb#ifdef KSE 374104695Sjulian if (td->td_standin != NULL) { 375134791Sjulian /* 376134791Sjulian * Note that we don't need to free the cred here as it 377134791Sjulian * is done in thread_reap(). 378134791Sjulian */ 379104695Sjulian thread_stash(td->td_standin); 380104695Sjulian td->td_standin = NULL; 381104695Sjulian } 382163709Sjb#endif 383104695Sjulian 384161678Sdavidxu umtx_thread_exit(td); 385161678Sdavidxu 386134791Sjulian /* 387134791Sjulian * drop FPU & debug register state storage, or any other 388134791Sjulian * architecture specific resources that 389134791Sjulian * would not be on a new untouched process. 390134791Sjulian */ 39199026Sjulian cpu_thread_exit(td); /* XXXSMP */ 39299026Sjulian 393163709Sjb#ifdef KSE 394102581Sjulian /* 395134791Sjulian * The thread is exiting. scheduler can release its stuff 396134791Sjulian * and collect stats etc. 397156705Sdavidxu * XXX this is not very right, since PROC_UNLOCK may still 398156705Sdavidxu * need scheduler stuff. 399134791Sjulian */ 400134791Sjulian sched_thread_exit(td); 401163709Sjb#endif 402134791Sjulian 403156705Sdavidxu /* Do the same timestamp bookkeeping that mi_switch() would do. */ 404156705Sdavidxu new_switchtime = cpu_ticks(); 405156705Sdavidxu p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime)); 406156705Sdavidxu PCPU_SET(switchtime, new_switchtime); 407156705Sdavidxu PCPU_SET(switchticks, ticks); 408170292Sattilio PCPU_INC(cnt.v_swtch); 409170174Sjeff /* 410170174Sjeff * Aggregate this thread's tick stats in the parent so they are not 411170174Sjeff * lost. Also add the child usage to our own when the final thread 412170174Sjeff * exits. 413170174Sjeff */ 414170174Sjeff ruxagg(&p->p_rux, td); 415156705Sdavidxu if (p->p_numthreads == 1) 416156705Sdavidxu ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux); 417134791Sjulian /* 418103002Sjulian * The last thread is left attached to the process 419103002Sjulian * So that the whole bundle gets recycled. Skip 420134791Sjulian * all this stuff if we never had threads. 421134791Sjulian * EXIT clears all sign of other threads when 422134791Sjulian * it goes to single threading, so the last thread always 423134791Sjulian * takes the short path. 424102581Sjulian */ 425134791Sjulian if (p->p_flag & P_HADTHREADS) { 426134791Sjulian if (p->p_numthreads > 1) { 427134791Sjulian thread_unlink(td); 428170174Sjeff /* Impart our resource usage on another thread */ 429170174Sjeff td2 = FIRST_THREAD_IN_PROC(p); 430170174Sjeff rucollect(&td2->td_ru, &td->td_ru); 431170174Sjeff sched_exit_thread(td2, td); 432134791Sjulian 433134791Sjulian /* 434134791Sjulian * The test below is NOT true if we are the 435134791Sjulian * sole exiting thread. P_STOPPED_SNGL is unset 436134791Sjulian * in exit1() after it is the only survivor. 437134791Sjulian */ 438134791Sjulian if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 439134791Sjulian if (p->p_numthreads == p->p_suspcount) { 440134791Sjulian thread_unsuspend_one(p->p_singlethread); 441134791Sjulian } 442103002Sjulian } 443104695Sjulian 444163709Sjb#ifdef KSE 445134791Sjulian /* 446134791Sjulian * Because each upcall structure has an owner thread, 447134791Sjulian * owner thread exits only when process is in exiting 448134791Sjulian * state, so upcall to userland is no longer needed, 449134791Sjulian * deleting upcall structure is safe here. 450134791Sjulian * So when all threads in a group is exited, all upcalls 451134791Sjulian * in the group should be automatically freed. 452134791Sjulian * XXXKSE This is a KSE thing and should be exported 453134791Sjulian * there somehow. 454134791Sjulian */ 455111028Sjeff upcall_remove(td); 456164936Sjulian#endif 457124350Sschweikh 458134791Sjulian PROC_UNLOCK(p); 459134791Sjulian PCPU_SET(deadthread, td); 460134791Sjulian } else { 461134791Sjulian /* 462134791Sjulian * The last thread is exiting.. but not through exit() 463134791Sjulian * what should we do? 464134791Sjulian * Theoretically this can't happen 465134791Sjulian * exit1() - clears threading flags before coming here 466134791Sjulian * kse_exit() - treats last thread specially 467134791Sjulian * thr_exit() - treats last thread specially 468163709Sjb * ifdef KSE 469134791Sjulian * thread_user_enter() - only if more exist 470134791Sjulian * thread_userret() - only if more exist 471163709Sjb * endif 472134791Sjulian * thread_suspend_check() - only if more exist 473134791Sjulian */ 474134791Sjulian panic ("thread_exit: Last thread exiting on its own"); 475119488Sdavidxu } 476103002Sjulian } else { 477134791Sjulian /* 478134791Sjulian * non threaded process comes here. 479134791Sjulian * This includes an EX threaded process that is coming 480134791Sjulian * here via exit1(). (exit1 dethreads the proc first). 481134791Sjulian */ 482103002Sjulian PROC_UNLOCK(p); 48399026Sjulian } 484133396Sjulian td->td_state = TDS_INACTIVE; 485133396Sjulian CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 486112993Speter cpu_throw(td, choosethread()); 487112993Speter panic("I'm a teapot!"); 48899026Sjulian /* NOTREACHED */ 48999026Sjulian} 49099026Sjulian 491124350Sschweikh/* 492107719Sjulian * Do any thread specific cleanups that may be needed in wait() 493126932Speter * called with Giant, proc and schedlock not held. 494107719Sjulian */ 495107719Sjulianvoid 496107719Sjulianthread_wait(struct proc *p) 497107719Sjulian{ 498107719Sjulian struct thread *td; 499107719Sjulian 500126932Speter mtx_assert(&Giant, MA_NOTOWNED); 501124350Sschweikh KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 502107719Sjulian FOREACH_THREAD_IN_PROC(p, td) { 503163709Sjb#ifdef KSE 504107719Sjulian if (td->td_standin != NULL) { 505143944Sjulian if (td->td_standin->td_ucred != NULL) { 506143944Sjulian crfree(td->td_standin->td_ucred); 507143944Sjulian td->td_standin->td_ucred = NULL; 508143944Sjulian } 509107719Sjulian thread_free(td->td_standin); 510107719Sjulian td->td_standin = NULL; 511107719Sjulian } 512163709Sjb#endif 513107719Sjulian cpu_thread_clean(td); 514134791Sjulian crfree(td->td_ucred); 515107719Sjulian } 516107719Sjulian thread_reap(); /* check for zombie threads etc. */ 517107719Sjulian} 518107719Sjulian 51999026Sjulian/* 52099026Sjulian * Link a thread to a process. 521103002Sjulian * set up anything that needs to be initialized for it to 522103002Sjulian * be used by the process. 52399026Sjulian * 52499026Sjulian * Note that we do not link to the proc's ucred here. 52599026Sjulian * The thread is linked as if running but no KSE assigned. 526134791Sjulian * Called from: 527134791Sjulian * proc_linkup() 528134791Sjulian * thread_schedule_upcall() 529134791Sjulian * thr_create() 53099026Sjulian */ 53199026Sjulianvoid 532163709Sjbthread_link(struct thread *td, struct proc *p) 53399026Sjulian{ 53499026Sjulian 535111028Sjeff td->td_state = TDS_INACTIVE; 536111028Sjeff td->td_proc = p; 537111028Sjeff td->td_flags = 0; 53899026Sjulian 539103002Sjulian LIST_INIT(&td->td_contested); 540151316Sdavidxu sigqueue_init(&td->td_sigqueue, p); 541119137Ssam callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 54299026Sjulian TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 54399026Sjulian p->p_numthreads++; 54499026Sjulian} 54599026Sjulian 546134791Sjulian/* 547136160Sjulian * Convert a process with one thread to an unthreaded process. 548134791Sjulian * Called from: 549136160Sjulian * thread_single(exit) (called from execve and exit) 550136160Sjulian * kse_exit() XXX may need cleaning up wrt KSE stuff 551136160Sjulian */ 552136160Sjulianvoid 553136160Sjulianthread_unthread(struct thread *td) 554136160Sjulian{ 555136160Sjulian struct proc *p = td->td_proc; 556136160Sjulian 557136160Sjulian KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); 558163709Sjb#ifdef KSE 559136160Sjulian upcall_remove(td); 560136160Sjulian p->p_flag &= ~(P_SA|P_HADTHREADS); 561136160Sjulian td->td_mailbox = NULL; 562136160Sjulian td->td_pflags &= ~(TDP_SA | TDP_CAN_UNBIND); 563136160Sjulian if (td->td_standin != NULL) { 564136160Sjulian thread_stash(td->td_standin); 565136160Sjulian td->td_standin = NULL; 566136160Sjulian } 567164936Sjulian sched_set_concurrency(p, 1); 568163709Sjb#else 569163709Sjb p->p_flag &= ~P_HADTHREADS; 570163709Sjb#endif 571136160Sjulian} 572136160Sjulian 573136160Sjulian/* 574136160Sjulian * Called from: 575134791Sjulian * thread_exit() 576134791Sjulian */ 577113641Sjulianvoid 578113641Sjulianthread_unlink(struct thread *td) 579124350Sschweikh{ 580113641Sjulian struct proc *p = td->td_proc; 581113920Sjhb 582113920Sjhb mtx_assert(&sched_lock, MA_OWNED); 583113641Sjulian TAILQ_REMOVE(&p->p_threads, td, td_plist); 584113641Sjulian p->p_numthreads--; 585113641Sjulian /* could clear a few other things here */ 586163709Sjb /* Must NOT clear links to proc! */ 587124350Sschweikh} 588113641Sjulian 589111028Sjeff/* 59099026Sjulian * Enforce single-threading. 59199026Sjulian * 59299026Sjulian * Returns 1 if the caller must abort (another thread is waiting to 59399026Sjulian * exit the process or similar). Process is locked! 59499026Sjulian * Returns 0 when you are successfully the only thread running. 59599026Sjulian * A process has successfully single threaded in the suspend mode when 59699026Sjulian * There are no threads in user mode. Threads in the kernel must be 59799026Sjulian * allowed to continue until they get to the user boundary. They may even 59899026Sjulian * copy out their return values and data before suspending. They may however be 599160048Smaxim * accelerated in reaching the user boundary as we will wake up 60099026Sjulian * any sleeping threads that are interruptable. (PCATCH). 60199026Sjulian */ 60299026Sjulianint 603136177Sdavidxuthread_single(int mode) 60499026Sjulian{ 60599026Sjulian struct thread *td; 60699026Sjulian struct thread *td2; 60799026Sjulian struct proc *p; 608130674Sdavidxu int remaining; 60999026Sjulian 61099026Sjulian td = curthread; 61199026Sjulian p = td->td_proc; 612126932Speter mtx_assert(&Giant, MA_NOTOWNED); 61399026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 61499026Sjulian KASSERT((td != NULL), ("curthread is NULL")); 61599026Sjulian 616134791Sjulian if ((p->p_flag & P_HADTHREADS) == 0) 61799026Sjulian return (0); 61899026Sjulian 619100648Sjulian /* Is someone already single threading? */ 620136177Sdavidxu if (p->p_singlethread != NULL && p->p_singlethread != td) 62199026Sjulian return (1); 62299026Sjulian 623136177Sdavidxu if (mode == SINGLE_EXIT) { 624136177Sdavidxu p->p_flag |= P_SINGLE_EXIT; 625136177Sdavidxu p->p_flag &= ~P_SINGLE_BOUNDARY; 626136177Sdavidxu } else { 627136177Sdavidxu p->p_flag &= ~P_SINGLE_EXIT; 628136177Sdavidxu if (mode == SINGLE_BOUNDARY) 629136177Sdavidxu p->p_flag |= P_SINGLE_BOUNDARY; 630136177Sdavidxu else 631136177Sdavidxu p->p_flag &= ~P_SINGLE_BOUNDARY; 632136177Sdavidxu } 633102950Sdavidxu p->p_flag |= P_STOPPED_SINGLE; 634113920Sjhb mtx_lock_spin(&sched_lock); 63599026Sjulian p->p_singlethread = td; 636136177Sdavidxu if (mode == SINGLE_EXIT) 637130674Sdavidxu remaining = p->p_numthreads; 638136177Sdavidxu else if (mode == SINGLE_BOUNDARY) 639136177Sdavidxu remaining = p->p_numthreads - p->p_boundary_count; 640136177Sdavidxu else 641130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 642130674Sdavidxu while (remaining != 1) { 643156942Sdavidxu if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 644156942Sdavidxu goto stopme; 64599026Sjulian FOREACH_THREAD_IN_PROC(p, td2) { 64699026Sjulian if (td2 == td) 64799026Sjulian continue; 648113705Sdavidxu td2->td_flags |= TDF_ASTPENDING; 649103216Sjulian if (TD_IS_INHIBITED(td2)) { 650136177Sdavidxu switch (mode) { 651136177Sdavidxu case SINGLE_EXIT: 652132087Sdavidxu if (td->td_flags & TDF_DBSUSPEND) 653132087Sdavidxu td->td_flags &= ~TDF_DBSUSPEND; 654136177Sdavidxu if (TD_IS_SUSPENDED(td2)) 655103216Sjulian thread_unsuspend_one(td2); 656105911Sjulian if (TD_ON_SLEEPQ(td2) && 657136177Sdavidxu (td2->td_flags & TDF_SINTR)) 658155741Sdavidxu sleepq_abort(td2, EINTR); 659136177Sdavidxu break; 660136177Sdavidxu case SINGLE_BOUNDARY: 661136177Sdavidxu if (TD_IS_SUSPENDED(td2) && 662136177Sdavidxu !(td2->td_flags & TDF_BOUNDARY)) 663136177Sdavidxu thread_unsuspend_one(td2); 664136177Sdavidxu if (TD_ON_SLEEPQ(td2) && 665136177Sdavidxu (td2->td_flags & TDF_SINTR)) 666155741Sdavidxu sleepq_abort(td2, ERESTART); 667136177Sdavidxu break; 668136177Sdavidxu default: 669105911Sjulian if (TD_IS_SUSPENDED(td2)) 670105874Sdavidxu continue; 671111028Sjeff /* 672165693Srwatson * maybe other inhibited states too? 673111028Sjeff */ 674137281Sdavidxu if ((td2->td_flags & TDF_SINTR) && 675137281Sdavidxu (td2->td_inhibitors & 676137281Sdavidxu (TDI_SLEEPING | TDI_SWAPPED))) 677105911Sjulian thread_suspend_one(td2); 678136177Sdavidxu break; 67999026Sjulian } 68099026Sjulian } 681155594Sdavidxu#ifdef SMP 682155594Sdavidxu else if (TD_IS_RUNNING(td2) && td != td2) { 683155594Sdavidxu forward_signal(td2); 684155594Sdavidxu } 685155594Sdavidxu#endif 68699026Sjulian } 687136177Sdavidxu if (mode == SINGLE_EXIT) 688130674Sdavidxu remaining = p->p_numthreads; 689136177Sdavidxu else if (mode == SINGLE_BOUNDARY) 690136177Sdavidxu remaining = p->p_numthreads - p->p_boundary_count; 691130674Sdavidxu else 692130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 693130674Sdavidxu 694124350Sschweikh /* 695124350Sschweikh * Maybe we suspended some threads.. was it enough? 696105911Sjulian */ 697130674Sdavidxu if (remaining == 1) 698105911Sjulian break; 699105911Sjulian 700156942Sdavidxustopme: 70199026Sjulian /* 70299026Sjulian * Wake us up when everyone else has suspended. 703100648Sjulian * In the mean time we suspend as well. 70499026Sjulian */ 705156934Sdavidxu thread_stopped(p); 706103216Sjulian thread_suspend_one(td); 70799026Sjulian PROC_UNLOCK(p); 708131473Sjhb mi_switch(SW_VOL, NULL); 70999026Sjulian mtx_unlock_spin(&sched_lock); 71099026Sjulian PROC_LOCK(p); 711113920Sjhb mtx_lock_spin(&sched_lock); 712136177Sdavidxu if (mode == SINGLE_EXIT) 713130674Sdavidxu remaining = p->p_numthreads; 714136177Sdavidxu else if (mode == SINGLE_BOUNDARY) 715136177Sdavidxu remaining = p->p_numthreads - p->p_boundary_count; 716130674Sdavidxu else 717130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 71899026Sjulian } 719136177Sdavidxu if (mode == SINGLE_EXIT) { 720135269Sjulian /* 721135269Sjulian * We have gotten rid of all the other threads and we 722135269Sjulian * are about to either exit or exec. In either case, 723135269Sjulian * we try our utmost to revert to being a non-threaded 724135269Sjulian * process. 725135269Sjulian */ 726136160Sjulian p->p_singlethread = NULL; 727137279Sdavidxu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 728136160Sjulian thread_unthread(td); 729111028Sjeff } 730136160Sjulian mtx_unlock_spin(&sched_lock); 73199026Sjulian return (0); 73299026Sjulian} 73399026Sjulian 73499026Sjulian/* 73599026Sjulian * Called in from locations that can safely check to see 73699026Sjulian * whether we have to suspend or at least throttle for a 73799026Sjulian * single-thread event (e.g. fork). 73899026Sjulian * 73999026Sjulian * Such locations include userret(). 74099026Sjulian * If the "return_instead" argument is non zero, the thread must be able to 74199026Sjulian * accept 0 (caller may continue), or 1 (caller must abort) as a result. 74299026Sjulian * 74399026Sjulian * The 'return_instead' argument tells the function if it may do a 74499026Sjulian * thread_exit() or suspend, or whether the caller must abort and back 74599026Sjulian * out instead. 74699026Sjulian * 74799026Sjulian * If the thread that set the single_threading request has set the 74899026Sjulian * P_SINGLE_EXIT bit in the process flags then this call will never return 74999026Sjulian * if 'return_instead' is false, but will exit. 75099026Sjulian * 75199026Sjulian * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 75299026Sjulian *---------------+--------------------+--------------------- 75399026Sjulian * 0 | returns 0 | returns 0 or 1 75499026Sjulian * | when ST ends | immediatly 75599026Sjulian *---------------+--------------------+--------------------- 75699026Sjulian * 1 | thread exits | returns 1 75799026Sjulian * | | immediatly 75899026Sjulian * 0 = thread_exit() or suspension ok, 75999026Sjulian * other = return error instead of stopping the thread. 76099026Sjulian * 76199026Sjulian * While a full suspension is under effect, even a single threading 76299026Sjulian * thread would be suspended if it made this call (but it shouldn't). 76399026Sjulian * This call should only be made from places where 764124350Sschweikh * thread_exit() would be safe as that may be the outcome unless 76599026Sjulian * return_instead is set. 76699026Sjulian */ 76799026Sjulianint 76899026Sjulianthread_suspend_check(int return_instead) 76999026Sjulian{ 770104502Sjmallett struct thread *td; 771104502Sjmallett struct proc *p; 77299026Sjulian 77399026Sjulian td = curthread; 77499026Sjulian p = td->td_proc; 775126932Speter mtx_assert(&Giant, MA_NOTOWNED); 77699026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 777132087Sdavidxu while (P_SHOULDSTOP(p) || 778132087Sdavidxu ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) { 779102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 78099026Sjulian KASSERT(p->p_singlethread != NULL, 78199026Sjulian ("singlethread not set")); 78299026Sjulian /* 783100648Sjulian * The only suspension in action is a 784100648Sjulian * single-threading. Single threader need not stop. 785124350Sschweikh * XXX Should be safe to access unlocked 786100646Sjulian * as it can only be set to be true by us. 78799026Sjulian */ 788100648Sjulian if (p->p_singlethread == td) 78999026Sjulian return (0); /* Exempt from stopping. */ 790124350Sschweikh } 791134498Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 792155741Sdavidxu return (EINTR); 79399026Sjulian 794136177Sdavidxu /* Should we goto user boundary if we didn't come from there? */ 795136177Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 796136177Sdavidxu (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 797155741Sdavidxu return (ERESTART); 798136177Sdavidxu 799151316Sdavidxu /* If thread will exit, flush its pending signals */ 800151316Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 801151316Sdavidxu sigqueue_flush(&td->td_sigqueue); 802151316Sdavidxu 803112071Sdavidxu mtx_lock_spin(&sched_lock); 804112071Sdavidxu thread_stopped(p); 80599026Sjulian /* 80699026Sjulian * If the process is waiting for us to exit, 80799026Sjulian * this thread should just suicide. 808102950Sdavidxu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 80999026Sjulian */ 810136177Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 811134791Sjulian thread_exit(); 81299026Sjulian 81399026Sjulian /* 81499026Sjulian * When a thread suspends, it just 815164936Sjulian * gets taken off all queues. 81699026Sjulian */ 817103216Sjulian thread_suspend_one(td); 818136177Sdavidxu if (return_instead == 0) { 819136177Sdavidxu p->p_boundary_count++; 820136177Sdavidxu td->td_flags |= TDF_BOUNDARY; 821136177Sdavidxu } 822102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 823136177Sdavidxu if (p->p_numthreads == p->p_suspcount) 824103216Sjulian thread_unsuspend_one(p->p_singlethread); 825100632Sjulian } 826113864Sjhb PROC_UNLOCK(p); 827131473Sjhb mi_switch(SW_INVOL, NULL); 828136177Sdavidxu if (return_instead == 0) { 829136177Sdavidxu p->p_boundary_count--; 830136177Sdavidxu td->td_flags &= ~TDF_BOUNDARY; 831136177Sdavidxu } 83299026Sjulian mtx_unlock_spin(&sched_lock); 83399026Sjulian PROC_LOCK(p); 83499026Sjulian } 83599026Sjulian return (0); 83699026Sjulian} 83799026Sjulian 838102898Sdavidxuvoid 839102898Sdavidxuthread_suspend_one(struct thread *td) 840102898Sdavidxu{ 841102898Sdavidxu struct proc *p = td->td_proc; 842102898Sdavidxu 843102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 844113920Sjhb PROC_LOCK_ASSERT(p, MA_OWNED); 845112071Sdavidxu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 846102898Sdavidxu p->p_suspcount++; 847103216Sjulian TD_SET_SUSPENDED(td); 848102898Sdavidxu} 849102898Sdavidxu 850102898Sdavidxuvoid 851102898Sdavidxuthread_unsuspend_one(struct thread *td) 852102898Sdavidxu{ 853102898Sdavidxu struct proc *p = td->td_proc; 854102898Sdavidxu 855102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 856113920Sjhb PROC_LOCK_ASSERT(p, MA_OWNED); 857164936Sjulian KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 858103216Sjulian TD_CLR_SUSPENDED(td); 859102898Sdavidxu p->p_suspcount--; 860103216Sjulian setrunnable(td); 861102898Sdavidxu} 862102898Sdavidxu 86399026Sjulian/* 86499026Sjulian * Allow all threads blocked by single threading to continue running. 86599026Sjulian */ 86699026Sjulianvoid 86799026Sjulianthread_unsuspend(struct proc *p) 86899026Sjulian{ 86999026Sjulian struct thread *td; 87099026Sjulian 871100646Sjulian mtx_assert(&sched_lock, MA_OWNED); 87299026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 87399026Sjulian if (!P_SHOULDSTOP(p)) { 874164936Sjulian FOREACH_THREAD_IN_PROC(p, td) { 875164936Sjulian if (TD_IS_SUSPENDED(td)) { 876164936Sjulian thread_unsuspend_one(td); 877164936Sjulian } 87899026Sjulian } 879102950Sdavidxu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 88099026Sjulian (p->p_numthreads == p->p_suspcount)) { 88199026Sjulian /* 88299026Sjulian * Stopping everything also did the job for the single 88399026Sjulian * threading request. Now we've downgraded to single-threaded, 88499026Sjulian * let it continue. 88599026Sjulian */ 886102898Sdavidxu thread_unsuspend_one(p->p_singlethread); 88799026Sjulian } 88899026Sjulian} 88999026Sjulian 890134791Sjulian/* 891134791Sjulian * End the single threading mode.. 892134791Sjulian */ 89399026Sjulianvoid 89499026Sjulianthread_single_end(void) 89599026Sjulian{ 89699026Sjulian struct thread *td; 89799026Sjulian struct proc *p; 89899026Sjulian 89999026Sjulian td = curthread; 90099026Sjulian p = td->td_proc; 90199026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 902136177Sdavidxu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); 903113920Sjhb mtx_lock_spin(&sched_lock); 90499026Sjulian p->p_singlethread = NULL; 905102292Sjulian /* 906102292Sjulian * If there are other threads they mey now run, 907102292Sjulian * unless of course there is a blanket 'stop order' 908102292Sjulian * on the process. The single threader must be allowed 909102292Sjulian * to continue however as this is a bad place to stop. 910102292Sjulian */ 911102292Sjulian if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 912164936Sjulian FOREACH_THREAD_IN_PROC(p, td) { 913164936Sjulian if (TD_IS_SUSPENDED(td)) { 914164936Sjulian thread_unsuspend_one(td); 915164936Sjulian } 916102292Sjulian } 917102292Sjulian } 918113920Sjhb mtx_unlock_spin(&sched_lock); 91999026Sjulian} 920128721Sdeischen 921151990Sdavidxustruct thread * 922151990Sdavidxuthread_find(struct proc *p, lwpid_t tid) 923151990Sdavidxu{ 924151990Sdavidxu struct thread *td; 925151990Sdavidxu 926151990Sdavidxu PROC_LOCK_ASSERT(p, MA_OWNED); 927151990Sdavidxu mtx_lock_spin(&sched_lock); 928151990Sdavidxu FOREACH_THREAD_IN_PROC(p, td) { 929151990Sdavidxu if (td->td_tid == tid) 930151990Sdavidxu break; 931151990Sdavidxu } 932151990Sdavidxu mtx_unlock_spin(&sched_lock); 933151990Sdavidxu return (td); 934151990Sdavidxu} 935