kern_thread.c revision 292892
1139804Simp/*- 299026Sjulian * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 399026Sjulian * All rights reserved. 499026Sjulian * 599026Sjulian * Redistribution and use in source and binary forms, with or without 699026Sjulian * modification, are permitted provided that the following conditions 799026Sjulian * are met: 899026Sjulian * 1. Redistributions of source code must retain the above copyright 999026Sjulian * notice(s), this list of conditions and the following disclaimer as 10124350Sschweikh * the first lines of this file unmodified other than the possible 1199026Sjulian * addition of one or more copyright notices. 1299026Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1399026Sjulian * notice(s), this list of conditions and the following disclaimer in the 1499026Sjulian * documentation and/or other materials provided with the distribution. 1599026Sjulian * 1699026Sjulian * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1799026Sjulian * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1899026Sjulian * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1999026Sjulian * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2099026Sjulian * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2199026Sjulian * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2299026Sjulian * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2399026Sjulian * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2499026Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2599026Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2699026Sjulian * DAMAGE. 2799026Sjulian */ 2899026Sjulian 29181695Sattilio#include "opt_witness.h" 30198464Sjkoshy#include "opt_hwpmc_hooks.h" 31181695Sattilio 32116182Sobrien#include <sys/cdefs.h> 33116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_thread.c 292892 2015-12-29 23:16:20Z jhb $"); 34116182Sobrien 3599026Sjulian#include <sys/param.h> 3699026Sjulian#include <sys/systm.h> 3799026Sjulian#include <sys/kernel.h> 3899026Sjulian#include <sys/lock.h> 3999026Sjulian#include <sys/mutex.h> 4099026Sjulian#include <sys/proc.h> 41236317Skib#include <sys/rangelock.h> 42156705Sdavidxu#include <sys/resourcevar.h> 43235459Srstone#include <sys/sdt.h> 44130355Sjulian#include <sys/smp.h> 45107126Sjeff#include <sys/sched.h> 46126326Sjhb#include <sys/sleepqueue.h> 47174647Sjeff#include <sys/selinfo.h> 48292892Sjhb#include <sys/syscallsubr.h> 49283382Sdchagin#include <sys/sysent.h> 50122514Sjhb#include <sys/turnstile.h> 5199026Sjulian#include <sys/ktr.h> 52213642Sdavidxu#include <sys/rwlock.h> 53143149Sdavidxu#include <sys/umtx.h> 54176730Sjeff#include <sys/cpuset.h> 55198464Sjkoshy#ifdef HWPMC_HOOKS 56198464Sjkoshy#include <sys/pmckern.h> 57198464Sjkoshy#endif 5899026Sjulian 59155195Srwatson#include <security/audit/audit.h> 60155195Srwatson 6199026Sjulian#include <vm/vm.h> 62116355Salc#include <vm/vm_extern.h> 6399026Sjulian#include <vm/uma.h> 64285387Sadrian#include <vm/vm_domain.h> 65173631Srrs#include <sys/eventhandler.h> 6699026Sjulian 67235459SrstoneSDT_PROVIDER_DECLARE(proc); 68258622SavgSDT_PROBE_DEFINE(proc, , , lwp__exit); 69235459Srstone 7099026Sjulian/* 71163709Sjb * thread related storage. 72163709Sjb */ 7399026Sjulianstatic uma_zone_t thread_zone; 7499026Sjulian 75111028SjeffTAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 76172256Sattiliostatic struct mtx zombie_lock; 77170296SjeffMTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); 7899026Sjulian 79170598Sjeffstatic void thread_zombie(struct thread *); 80282944Skibstatic int thread_unsuspend_one(struct thread *td, struct proc *p, 81282944Skib bool boundary); 82170598Sjeff 83216314Sdavidxu#define TID_BUFFER_SIZE 1024 84216314Sdavidxu 85127794Smarcelstruct mtx tid_lock; 86143802Sphkstatic struct unrhdr *tid_unrhdr; 87216314Sdavidxustatic lwpid_t tid_buffer[TID_BUFFER_SIZE]; 88216314Sdavidxustatic int tid_head, tid_tail; 89213642Sdavidxustatic MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash"); 90213642Sdavidxu 91213642Sdavidxustruct tidhashhead *tidhashtbl; 92213642Sdavidxuu_long tidhash; 93213642Sdavidxustruct rwlock tidhash_lock; 94213642Sdavidxu 95216314Sdavidxustatic lwpid_t 96216314Sdavidxutid_alloc(void) 97216314Sdavidxu{ 98216314Sdavidxu lwpid_t tid; 99216314Sdavidxu 100216314Sdavidxu tid = alloc_unr(tid_unrhdr); 101216314Sdavidxu if (tid != -1) 102216314Sdavidxu return (tid); 103216314Sdavidxu mtx_lock(&tid_lock); 104216314Sdavidxu if (tid_head == tid_tail) { 105216314Sdavidxu mtx_unlock(&tid_lock); 106216314Sdavidxu return (-1); 107216314Sdavidxu } 108240951Skib tid = tid_buffer[tid_head]; 109240951Skib tid_head = (tid_head + 1) % TID_BUFFER_SIZE; 110216314Sdavidxu mtx_unlock(&tid_lock); 111216314Sdavidxu return (tid); 112216314Sdavidxu} 113216314Sdavidxu 114216314Sdavidxustatic void 115216314Sdavidxutid_free(lwpid_t tid) 116216314Sdavidxu{ 117216314Sdavidxu lwpid_t tmp_tid = -1; 118216314Sdavidxu 119216314Sdavidxu mtx_lock(&tid_lock); 120216314Sdavidxu if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) { 121240951Skib tmp_tid = tid_buffer[tid_head]; 122240951Skib tid_head = (tid_head + 1) % TID_BUFFER_SIZE; 123216314Sdavidxu } 124240951Skib tid_buffer[tid_tail] = tid; 125240951Skib tid_tail = (tid_tail + 1) % TID_BUFFER_SIZE; 126216314Sdavidxu mtx_unlock(&tid_lock); 127216314Sdavidxu if (tmp_tid != -1) 128216314Sdavidxu free_unr(tid_unrhdr, tmp_tid); 129216314Sdavidxu} 130216314Sdavidxu 131127794Smarcel/* 132107719Sjulian * Prepare a thread for use. 13399026Sjulian */ 134132987Sgreenstatic int 135132987Sgreenthread_ctor(void *mem, int size, void *arg, int flags) 13699026Sjulian{ 13799026Sjulian struct thread *td; 13899026Sjulian 13999026Sjulian td = (struct thread *)mem; 140103216Sjulian td->td_state = TDS_INACTIVE; 141135573Sjhb td->td_oncpu = NOCPU; 142130269Sjmallett 143216314Sdavidxu td->td_tid = tid_alloc(); 144143840Sphk 145130269Sjmallett /* 146130269Sjmallett * Note that td_critnest begins life as 1 because the thread is not 147130269Sjmallett * running and is thereby implicitly waiting to be on the receiving 148170296Sjeff * end of a context switch. 149130269Sjmallett */ 150118442Sjhb td->td_critnest = 1; 151216313Sdavidxu td->td_lend_user_pri = PRI_MAX; 152173631Srrs EVENTHANDLER_INVOKE(thread_ctor, td); 153155195Srwatson#ifdef AUDIT 154155195Srwatson audit_thread_alloc(td); 155155195Srwatson#endif 156161678Sdavidxu umtx_thread_alloc(td); 157132987Sgreen return (0); 15899026Sjulian} 15999026Sjulian 16099026Sjulian/* 16199026Sjulian * Reclaim a thread after use. 16299026Sjulian */ 16399026Sjulianstatic void 16499026Sjulianthread_dtor(void *mem, int size, void *arg) 16599026Sjulian{ 166127794Smarcel struct thread *td; 16799026Sjulian 16899026Sjulian td = (struct thread *)mem; 16999026Sjulian 17099026Sjulian#ifdef INVARIANTS 17199026Sjulian /* Verify that this thread is in a safe state to free. */ 17299026Sjulian switch (td->td_state) { 173103216Sjulian case TDS_INHIBITED: 174103216Sjulian case TDS_RUNNING: 175103216Sjulian case TDS_CAN_RUN: 17699026Sjulian case TDS_RUNQ: 17799026Sjulian /* 17899026Sjulian * We must never unlink a thread that is in one of 17999026Sjulian * these states, because it is currently active. 18099026Sjulian */ 18199026Sjulian panic("bad state for thread unlinking"); 18299026Sjulian /* NOTREACHED */ 183103216Sjulian case TDS_INACTIVE: 18499026Sjulian break; 18599026Sjulian default: 18699026Sjulian panic("bad thread state"); 18799026Sjulian /* NOTREACHED */ 18899026Sjulian } 18999026Sjulian#endif 190155353Srwatson#ifdef AUDIT 191155353Srwatson audit_thread_free(td); 192155353Srwatson#endif 193185029Spjd /* Free all OSD associated to this thread. */ 194185029Spjd osd_thread_exit(td); 195185029Spjd 196173631Srrs EVENTHANDLER_INVOKE(thread_dtor, td); 197216314Sdavidxu tid_free(td->td_tid); 19899026Sjulian} 19999026Sjulian 20099026Sjulian/* 20199026Sjulian * Initialize type-stable parts of a thread (when newly created). 20299026Sjulian */ 203132987Sgreenstatic int 204132987Sgreenthread_init(void *mem, int size, int flags) 20599026Sjulian{ 206131149Smarcel struct thread *td; 20799026Sjulian 20899026Sjulian td = (struct thread *)mem; 209131149Smarcel 210126326Sjhb td->td_sleepqueue = sleepq_alloc(); 211122514Sjhb td->td_turnstile = turnstile_alloc(); 212236317Skib td->td_rlqe = NULL; 213173631Srrs EVENTHANDLER_INVOKE(thread_init, td); 214107126Sjeff td->td_sched = (struct td_sched *)&td[1]; 215161678Sdavidxu umtx_thread_init(td); 216173361Skib td->td_kstack = 0; 217281696Skib td->td_sel = NULL; 218132987Sgreen return (0); 21999026Sjulian} 22099026Sjulian 22199026Sjulian/* 22299026Sjulian * Tear down type-stable parts of a thread (just before being discarded). 22399026Sjulian */ 22499026Sjulianstatic void 22599026Sjulianthread_fini(void *mem, int size) 22699026Sjulian{ 227131149Smarcel struct thread *td; 22899026Sjulian 22999026Sjulian td = (struct thread *)mem; 230173631Srrs EVENTHANDLER_INVOKE(thread_fini, td); 231236317Skib rlqentry_free(td->td_rlqe); 232122514Sjhb turnstile_free(td->td_turnstile); 233126326Sjhb sleepq_free(td->td_sleepqueue); 234161678Sdavidxu umtx_thread_fini(td); 235174647Sjeff seltdfini(td); 23699026Sjulian} 237111028Sjeff 238107126Sjeff/* 239111028Sjeff * For a newly created process, 240111028Sjeff * link up all the structures and its initial threads etc. 241134791Sjulian * called from: 242268351Smarcel * {arch}/{arch}/machdep.c {arch}_init(), init386() etc. 243134791Sjulian * proc_dtor() (should go away) 244134791Sjulian * proc_init() 245105854Sjulian */ 246105854Sjulianvoid 247173361Skibproc_linkup0(struct proc *p, struct thread *td) 248173361Skib{ 249173361Skib TAILQ_INIT(&p->p_threads); /* all threads in proc */ 250173361Skib proc_linkup(p, td); 251173361Skib} 252173361Skib 253173361Skibvoid 254163709Sjbproc_linkup(struct proc *p, struct thread *td) 255105854Sjulian{ 256170296Sjeff 257151316Sdavidxu sigqueue_init(&p->p_sigqueue, p); 258153253Sdavidxu p->p_ksi = ksiginfo_alloc(1); 259153253Sdavidxu if (p->p_ksi != NULL) { 260153253Sdavidxu /* XXX p_ksi may be null if ksiginfo zone is not ready */ 261153253Sdavidxu p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 262152185Sdavidxu } 263152948Sdavidxu LIST_INIT(&p->p_mqnotifier); 264105854Sjulian p->p_numthreads = 0; 265163709Sjb thread_link(td, p); 266105854Sjulian} 267105854Sjulian 268111028Sjeff/* 26999026Sjulian * Initialize global thread allocation resources. 27099026Sjulian */ 27199026Sjulianvoid 27299026Sjulianthreadinit(void) 27399026Sjulian{ 27499026Sjulian 275143802Sphk mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 276239301Skib 277239301Skib /* 278239328Skib * pid_max cannot be greater than PID_MAX. 279239301Skib * leave one number for thread0. 280239301Skib */ 281174848Sjulian tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock); 282143802Sphk 283107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 28499026Sjulian thread_ctor, thread_dtor, thread_init, thread_fini, 285289661Skib 16 - 1, UMA_ZONE_NOFREE); 286213642Sdavidxu tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash); 287213642Sdavidxu rw_init(&tidhash_lock, "tidhash"); 28899026Sjulian} 28999026Sjulian 29099026Sjulian/* 291170598Sjeff * Place an unused thread on the zombie list. 292164936Sjulian * Use the slpq as that must be unused by now. 29399026Sjulian */ 29499026Sjulianvoid 295170598Sjeffthread_zombie(struct thread *td) 29699026Sjulian{ 297170296Sjeff mtx_lock_spin(&zombie_lock); 298164936Sjulian TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); 299170296Sjeff mtx_unlock_spin(&zombie_lock); 30099026Sjulian} 30199026Sjulian 302103410Smini/* 303170598Sjeff * Release a thread that has exited after cpu_throw(). 304170598Sjeff */ 305170598Sjeffvoid 306170598Sjeffthread_stash(struct thread *td) 307170598Sjeff{ 308170598Sjeff atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); 309170598Sjeff thread_zombie(td); 310170598Sjeff} 311170598Sjeff 312170598Sjeff/* 313177091Sjeff * Reap zombie resources. 31499026Sjulian */ 31599026Sjulianvoid 31699026Sjulianthread_reap(void) 31799026Sjulian{ 318105854Sjulian struct thread *td_first, *td_next; 31999026Sjulian 32099026Sjulian /* 321111028Sjeff * Don't even bother to lock if none at this instant, 322111028Sjeff * we really don't care about the next instant.. 32399026Sjulian */ 324163709Sjb if (!TAILQ_EMPTY(&zombie_threads)) { 325170296Sjeff mtx_lock_spin(&zombie_lock); 326105854Sjulian td_first = TAILQ_FIRST(&zombie_threads); 327105854Sjulian if (td_first) 328105854Sjulian TAILQ_INIT(&zombie_threads); 329170296Sjeff mtx_unlock_spin(&zombie_lock); 330105854Sjulian while (td_first) { 331164936Sjulian td_next = TAILQ_NEXT(td_first, td_slpq); 332284214Smjg thread_cow_free(td_first); 333105854Sjulian thread_free(td_first); 334105854Sjulian td_first = td_next; 33599026Sjulian } 33699026Sjulian } 33799026Sjulian} 33899026Sjulian 33999026Sjulian/* 34099026Sjulian * Allocate a thread. 34199026Sjulian */ 34299026Sjulianstruct thread * 343196730Skibthread_alloc(int pages) 34499026Sjulian{ 345173361Skib struct thread *td; 346163709Sjb 34799026Sjulian thread_reap(); /* check if any zombies to get */ 348173361Skib 349173361Skib td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK); 350173361Skib KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); 351196730Skib if (!vm_thread_new(td, pages)) { 352173361Skib uma_zfree(thread_zone, td); 353173361Skib return (NULL); 354173361Skib } 355173615Smarcel cpu_thread_alloc(td); 356285387Sadrian vm_domain_policy_init(&td->td_vm_dom_policy); 357173361Skib return (td); 35899026Sjulian} 35999026Sjulian 360196730Skibint 361196730Skibthread_alloc_stack(struct thread *td, int pages) 362196730Skib{ 363103367Sjulian 364196730Skib KASSERT(td->td_kstack == 0, 365196730Skib ("thread_alloc_stack called on a thread with kstack")); 366196730Skib if (!vm_thread_new(td, pages)) 367196730Skib return (0); 368196730Skib cpu_thread_alloc(td); 369196730Skib return (1); 370196730Skib} 371196730Skib 372103367Sjulian/* 37399026Sjulian * Deallocate a thread. 37499026Sjulian */ 37599026Sjulianvoid 37699026Sjulianthread_free(struct thread *td) 37799026Sjulian{ 378189845Sjeff 379189845Sjeff lock_profile_thread_exit(td); 380177369Sjeff if (td->td_cpuset) 381177369Sjeff cpuset_rel(td->td_cpuset); 382176730Sjeff td->td_cpuset = NULL; 383173615Smarcel cpu_thread_free(td); 384173361Skib if (td->td_kstack != 0) 385173361Skib vm_thread_dispose(td); 386285387Sadrian vm_domain_policy_cleanup(&td->td_vm_dom_policy); 38799026Sjulian uma_zfree(thread_zone, td); 38899026Sjulian} 38999026Sjulian 390284214Smjgvoid 391284214Smjgthread_cow_get_proc(struct thread *newtd, struct proc *p) 392284214Smjg{ 393284214Smjg 394284214Smjg PROC_LOCK_ASSERT(p, MA_OWNED); 395284214Smjg newtd->td_ucred = crhold(p->p_ucred); 396284215Smjg newtd->td_limit = lim_hold(p->p_limit); 397284214Smjg newtd->td_cowgen = p->p_cowgen; 398284214Smjg} 399284214Smjg 400284214Smjgvoid 401284214Smjgthread_cow_get(struct thread *newtd, struct thread *td) 402284214Smjg{ 403284214Smjg 404284214Smjg newtd->td_ucred = crhold(td->td_ucred); 405284215Smjg newtd->td_limit = lim_hold(td->td_limit); 406284214Smjg newtd->td_cowgen = td->td_cowgen; 407284214Smjg} 408284214Smjg 409284214Smjgvoid 410284214Smjgthread_cow_free(struct thread *td) 411284214Smjg{ 412284214Smjg 413285633Smjg if (td->td_ucred != NULL) 414284214Smjg crfree(td->td_ucred); 415285633Smjg if (td->td_limit != NULL) 416284215Smjg lim_free(td->td_limit); 417284214Smjg} 418284214Smjg 419284214Smjgvoid 420284214Smjgthread_cow_update(struct thread *td) 421284214Smjg{ 422284214Smjg struct proc *p; 423285633Smjg struct ucred *oldcred; 424285633Smjg struct plimit *oldlimit; 425284214Smjg 426284214Smjg p = td->td_proc; 427285633Smjg oldcred = NULL; 428285633Smjg oldlimit = NULL; 429284214Smjg PROC_LOCK(p); 430285633Smjg if (td->td_ucred != p->p_ucred) { 431285633Smjg oldcred = td->td_ucred; 432285633Smjg td->td_ucred = crhold(p->p_ucred); 433285633Smjg } 434285633Smjg if (td->td_limit != p->p_limit) { 435285633Smjg oldlimit = td->td_limit; 436285633Smjg td->td_limit = lim_hold(p->p_limit); 437285633Smjg } 438284214Smjg td->td_cowgen = p->p_cowgen; 439284214Smjg PROC_UNLOCK(p); 440285633Smjg if (oldcred != NULL) 441285633Smjg crfree(oldcred); 442285633Smjg if (oldlimit != NULL) 443285633Smjg lim_free(oldlimit); 444284214Smjg} 445284214Smjg 44699026Sjulian/* 44799026Sjulian * Discard the current thread and exit from its context. 448130355Sjulian * Always called with scheduler locked. 44999026Sjulian * 45099026Sjulian * Because we can't free a thread while we're operating under its context, 451107719Sjulian * push the current thread into our CPU's deadthread holder. This means 452107719Sjulian * we needn't worry about someone else grabbing our context before we 453177091Sjeff * do a cpu_throw(). 45499026Sjulian */ 45599026Sjulianvoid 45699026Sjulianthread_exit(void) 45799026Sjulian{ 458229429Sjhb uint64_t runtime, new_switchtime; 45999026Sjulian struct thread *td; 460170174Sjeff struct thread *td2; 46199026Sjulian struct proc *p; 462182011Sjhb int wakeup_swapper; 46399026Sjulian 46499026Sjulian td = curthread; 46599026Sjulian p = td->td_proc; 46699026Sjulian 467170296Sjeff PROC_SLOCK_ASSERT(p, MA_OWNED); 468134791Sjulian mtx_assert(&Giant, MA_NOTOWNED); 469170296Sjeff 470134791Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 471102581Sjulian KASSERT(p != NULL, ("thread exiting without a process")); 472133234Srwatson CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 473173601Sjulian (long)p->p_pid, td->td_name); 474151316Sdavidxu KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 47599026Sjulian 476155376Srwatson#ifdef AUDIT 477155376Srwatson AUDIT_SYSCALL_EXIT(0, td); 478155376Srwatson#endif 479134791Sjulian /* 480134791Sjulian * drop FPU & debug register state storage, or any other 481134791Sjulian * architecture specific resources that 482134791Sjulian * would not be on a new untouched process. 483134791Sjulian */ 48499026Sjulian cpu_thread_exit(td); /* XXXSMP */ 48599026Sjulian 486134791Sjulian /* 487103002Sjulian * The last thread is left attached to the process 488103002Sjulian * So that the whole bundle gets recycled. Skip 489134791Sjulian * all this stuff if we never had threads. 490134791Sjulian * EXIT clears all sign of other threads when 491134791Sjulian * it goes to single threading, so the last thread always 492134791Sjulian * takes the short path. 493102581Sjulian */ 494134791Sjulian if (p->p_flag & P_HADTHREADS) { 495134791Sjulian if (p->p_numthreads > 1) { 496271000Skib atomic_add_int(&td->td_proc->p_exitthreads, 1); 497134791Sjulian thread_unlink(td); 498170174Sjeff td2 = FIRST_THREAD_IN_PROC(p); 499170174Sjeff sched_exit_thread(td2, td); 500134791Sjulian 501134791Sjulian /* 502134791Sjulian * The test below is NOT true if we are the 503207606Skib * sole exiting thread. P_STOPPED_SINGLE is unset 504134791Sjulian * in exit1() after it is the only survivor. 505134791Sjulian */ 506134791Sjulian if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 507134791Sjulian if (p->p_numthreads == p->p_suspcount) { 508170296Sjeff thread_lock(p->p_singlethread); 509182011Sjhb wakeup_swapper = thread_unsuspend_one( 510282944Skib p->p_singlethread, p, false); 511170296Sjeff thread_unlock(p->p_singlethread); 512182011Sjhb if (wakeup_swapper) 513182011Sjhb kick_proc0(); 514134791Sjulian } 515103002Sjulian } 516104695Sjulian 517134791Sjulian PCPU_SET(deadthread, td); 518134791Sjulian } else { 519134791Sjulian /* 520134791Sjulian * The last thread is exiting.. but not through exit() 521134791Sjulian */ 522134791Sjulian panic ("thread_exit: Last thread exiting on its own"); 523119488Sdavidxu } 524170296Sjeff } 525198464Sjkoshy#ifdef HWPMC_HOOKS 526198464Sjkoshy /* 527198464Sjkoshy * If this thread is part of a process that is being tracked by hwpmc(4), 528198464Sjkoshy * inform the module of the thread's impending exit. 529198464Sjkoshy */ 530198464Sjkoshy if (PMC_PROC_IS_USING_PMCS(td->td_proc)) 531198464Sjkoshy PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); 532198464Sjkoshy#endif 533170296Sjeff PROC_UNLOCK(p); 534275121Skib PROC_STATLOCK(p); 535275121Skib thread_lock(td); 536275121Skib PROC_SUNLOCK(p); 537229429Sjhb 538229429Sjhb /* Do the same timestamp bookkeeping that mi_switch() would do. */ 539229429Sjhb new_switchtime = cpu_ticks(); 540229429Sjhb runtime = new_switchtime - PCPU_GET(switchtime); 541229429Sjhb td->td_runtime += runtime; 542229429Sjhb td->td_incruntime += runtime; 543229429Sjhb PCPU_SET(switchtime, new_switchtime); 544229429Sjhb PCPU_SET(switchticks, ticks); 545229429Sjhb PCPU_INC(cnt.v_swtch); 546229429Sjhb 547229429Sjhb /* Save our resource usage in our process. */ 548229429Sjhb td->td_ru.ru_nvcsw++; 549208488Skib ruxagg(p, td); 550229429Sjhb rucollect(&p->p_ru, &td->td_ru); 551275121Skib PROC_STATUNLOCK(p); 552229429Sjhb 553133396Sjulian td->td_state = TDS_INACTIVE; 554181695Sattilio#ifdef WITNESS 555181695Sattilio witness_thread_exit(td); 556181695Sattilio#endif 557133396Sjulian CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 558170296Sjeff sched_throw(td); 559112993Speter panic("I'm a teapot!"); 56099026Sjulian /* NOTREACHED */ 56199026Sjulian} 56299026Sjulian 563124350Sschweikh/* 564107719Sjulian * Do any thread specific cleanups that may be needed in wait() 565126932Speter * called with Giant, proc and schedlock not held. 566107719Sjulian */ 567107719Sjulianvoid 568107719Sjulianthread_wait(struct proc *p) 569107719Sjulian{ 570107719Sjulian struct thread *td; 571107719Sjulian 572126932Speter mtx_assert(&Giant, MA_NOTOWNED); 573271008Skib KASSERT(p->p_numthreads == 1, ("multiple threads in thread_wait()")); 574271008Skib KASSERT(p->p_exitthreads == 0, ("p_exitthreads leaking")); 575170598Sjeff td = FIRST_THREAD_IN_PROC(p); 576170598Sjeff /* Lock the last thread so we spin until it exits cpu_throw(). */ 577170598Sjeff thread_lock(td); 578170598Sjeff thread_unlock(td); 579189845Sjeff lock_profile_thread_exit(td); 580176730Sjeff cpuset_rel(td->td_cpuset); 581176730Sjeff td->td_cpuset = NULL; 582170598Sjeff cpu_thread_clean(td); 583284214Smjg thread_cow_free(td); 584107719Sjulian thread_reap(); /* check for zombie threads etc. */ 585107719Sjulian} 586107719Sjulian 58799026Sjulian/* 58899026Sjulian * Link a thread to a process. 589103002Sjulian * set up anything that needs to be initialized for it to 590103002Sjulian * be used by the process. 59199026Sjulian */ 59299026Sjulianvoid 593163709Sjbthread_link(struct thread *td, struct proc *p) 59499026Sjulian{ 59599026Sjulian 596170296Sjeff /* 597170296Sjeff * XXX This can't be enabled because it's called for proc0 before 598177368Sjeff * its lock has been created. 599177368Sjeff * PROC_LOCK_ASSERT(p, MA_OWNED); 600170296Sjeff */ 601111028Sjeff td->td_state = TDS_INACTIVE; 602111028Sjeff td->td_proc = p; 603172207Sjeff td->td_flags = TDF_INMEM; 60499026Sjulian 605103002Sjulian LIST_INIT(&td->td_contested); 606174629Sjeff LIST_INIT(&td->td_lprof[0]); 607174629Sjeff LIST_INIT(&td->td_lprof[1]); 608151316Sdavidxu sigqueue_init(&td->td_sigqueue, p); 609283291Sjkim callout_init(&td->td_slpcallout, 1); 610269095Sdeischen TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist); 61199026Sjulian p->p_numthreads++; 61299026Sjulian} 61399026Sjulian 614134791Sjulian/* 615136160Sjulian * Called from: 616134791Sjulian * thread_exit() 617134791Sjulian */ 618113641Sjulianvoid 619113641Sjulianthread_unlink(struct thread *td) 620124350Sschweikh{ 621113641Sjulian struct proc *p = td->td_proc; 622113920Sjhb 623177368Sjeff PROC_LOCK_ASSERT(p, MA_OWNED); 624113641Sjulian TAILQ_REMOVE(&p->p_threads, td, td_plist); 625113641Sjulian p->p_numthreads--; 626113641Sjulian /* could clear a few other things here */ 627163709Sjb /* Must NOT clear links to proc! */ 628124350Sschweikh} 629113641Sjulian 630195701Skibstatic int 631195701Skibcalc_remaining(struct proc *p, int mode) 632195701Skib{ 633195701Skib int remaining; 634195701Skib 635227657Skib PROC_LOCK_ASSERT(p, MA_OWNED); 636227657Skib PROC_SLOCK_ASSERT(p, MA_OWNED); 637195701Skib if (mode == SINGLE_EXIT) 638195701Skib remaining = p->p_numthreads; 639195701Skib else if (mode == SINGLE_BOUNDARY) 640195701Skib remaining = p->p_numthreads - p->p_boundary_count; 641275745Skib else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC) 642195701Skib remaining = p->p_numthreads - p->p_suspcount; 643195701Skib else 644195701Skib panic("calc_remaining: wrong mode %d", mode); 645195701Skib return (remaining); 646195701Skib} 647195701Skib 648275617Skibstatic int 649275617Skibremain_for_mode(int mode) 650275617Skib{ 651275617Skib 652275745Skib return (mode == SINGLE_ALLPROC ? 0 : 1); 653275617Skib} 654275617Skib 655275617Skibstatic int 656275617Skibweed_inhib(int mode, struct thread *td2, struct proc *p) 657275617Skib{ 658275617Skib int wakeup_swapper; 659275617Skib 660275617Skib PROC_LOCK_ASSERT(p, MA_OWNED); 661275617Skib PROC_SLOCK_ASSERT(p, MA_OWNED); 662275617Skib THREAD_LOCK_ASSERT(td2, MA_OWNED); 663275617Skib 664275617Skib wakeup_swapper = 0; 665275617Skib switch (mode) { 666275617Skib case SINGLE_EXIT: 667275617Skib if (TD_IS_SUSPENDED(td2)) 668282944Skib wakeup_swapper |= thread_unsuspend_one(td2, p, true); 669275617Skib if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) 670275617Skib wakeup_swapper |= sleepq_abort(td2, EINTR); 671275617Skib break; 672275617Skib case SINGLE_BOUNDARY: 673275617Skib if (TD_IS_SUSPENDED(td2) && (td2->td_flags & TDF_BOUNDARY) == 0) 674282944Skib wakeup_swapper |= thread_unsuspend_one(td2, p, false); 675275617Skib if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) 676275617Skib wakeup_swapper |= sleepq_abort(td2, ERESTART); 677275617Skib break; 678275617Skib case SINGLE_NO_EXIT: 679275617Skib if (TD_IS_SUSPENDED(td2) && (td2->td_flags & TDF_BOUNDARY) == 0) 680282944Skib wakeup_swapper |= thread_unsuspend_one(td2, p, false); 681275617Skib if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) 682275617Skib wakeup_swapper |= sleepq_abort(td2, ERESTART); 683275820Skib break; 684275745Skib case SINGLE_ALLPROC: 685275745Skib /* 686275745Skib * ALLPROC suspend tries to avoid spurious EINTR for 687275745Skib * threads sleeping interruptable, by suspending the 688275745Skib * thread directly, similarly to sig_suspend_threads(). 689275745Skib * Since such sleep is not performed at the user 690275745Skib * boundary, TDF_BOUNDARY flag is not set, and TDF_ALLPROCSUSP 691275745Skib * is used to avoid immediate un-suspend. 692275745Skib */ 693275745Skib if (TD_IS_SUSPENDED(td2) && (td2->td_flags & (TDF_BOUNDARY | 694275745Skib TDF_ALLPROCSUSP)) == 0) 695282944Skib wakeup_swapper |= thread_unsuspend_one(td2, p, false); 696275745Skib if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) { 697275745Skib if ((td2->td_flags & TDF_SBDRY) == 0) { 698275745Skib thread_suspend_one(td2); 699275745Skib td2->td_flags |= TDF_ALLPROCSUSP; 700275745Skib } else { 701275745Skib wakeup_swapper |= sleepq_abort(td2, ERESTART); 702275745Skib } 703275745Skib } 704275617Skib break; 705275617Skib } 706275617Skib return (wakeup_swapper); 707275617Skib} 708275617Skib 709111028Sjeff/* 71099026Sjulian * Enforce single-threading. 71199026Sjulian * 71299026Sjulian * Returns 1 if the caller must abort (another thread is waiting to 71399026Sjulian * exit the process or similar). Process is locked! 71499026Sjulian * Returns 0 when you are successfully the only thread running. 71599026Sjulian * A process has successfully single threaded in the suspend mode when 71699026Sjulian * There are no threads in user mode. Threads in the kernel must be 71799026Sjulian * allowed to continue until they get to the user boundary. They may even 71899026Sjulian * copy out their return values and data before suspending. They may however be 719160048Smaxim * accelerated in reaching the user boundary as we will wake up 72099026Sjulian * any sleeping threads that are interruptable. (PCATCH). 72199026Sjulian */ 72299026Sjulianint 723275745Skibthread_single(struct proc *p, int mode) 72499026Sjulian{ 72599026Sjulian struct thread *td; 72699026Sjulian struct thread *td2; 727181334Sjhb int remaining, wakeup_swapper; 72899026Sjulian 72999026Sjulian td = curthread; 730275745Skib KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY || 731275745Skib mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT, 732275745Skib ("invalid mode %d", mode)); 733275745Skib /* 734275745Skib * If allowing non-ALLPROC singlethreading for non-curproc 735275745Skib * callers, calc_remaining() and remain_for_mode() should be 736275745Skib * adjusted to also account for td->td_proc != p. For now 737275745Skib * this is not implemented because it is not used. 738275745Skib */ 739275745Skib KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) || 740275745Skib (mode != SINGLE_ALLPROC && td->td_proc == p), 741275745Skib ("mode %d proc %p curproc %p", mode, p, td->td_proc)); 742126932Speter mtx_assert(&Giant, MA_NOTOWNED); 74399026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 74499026Sjulian 745275745Skib if ((p->p_flag & P_HADTHREADS) == 0 && mode != SINGLE_ALLPROC) 74699026Sjulian return (0); 74799026Sjulian 748100648Sjulian /* Is someone already single threading? */ 749136177Sdavidxu if (p->p_singlethread != NULL && p->p_singlethread != td) 75099026Sjulian return (1); 75199026Sjulian 752136177Sdavidxu if (mode == SINGLE_EXIT) { 753136177Sdavidxu p->p_flag |= P_SINGLE_EXIT; 754136177Sdavidxu p->p_flag &= ~P_SINGLE_BOUNDARY; 755136177Sdavidxu } else { 756136177Sdavidxu p->p_flag &= ~P_SINGLE_EXIT; 757136177Sdavidxu if (mode == SINGLE_BOUNDARY) 758136177Sdavidxu p->p_flag |= P_SINGLE_BOUNDARY; 759136177Sdavidxu else 760136177Sdavidxu p->p_flag &= ~P_SINGLE_BOUNDARY; 761136177Sdavidxu } 762275745Skib if (mode == SINGLE_ALLPROC) 763275745Skib p->p_flag |= P_TOTAL_STOP; 764102950Sdavidxu p->p_flag |= P_STOPPED_SINGLE; 765184667Sdavidxu PROC_SLOCK(p); 76699026Sjulian p->p_singlethread = td; 767195701Skib remaining = calc_remaining(p, mode); 768275617Skib while (remaining != remain_for_mode(mode)) { 769156942Sdavidxu if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 770156942Sdavidxu goto stopme; 771181334Sjhb wakeup_swapper = 0; 77299026Sjulian FOREACH_THREAD_IN_PROC(p, td2) { 77399026Sjulian if (td2 == td) 77499026Sjulian continue; 775170296Sjeff thread_lock(td2); 776177471Sjeff td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK; 777275745Skib if (TD_IS_INHIBITED(td2)) { 778275617Skib wakeup_swapper |= weed_inhib(mode, td2, p); 779155594Sdavidxu#ifdef SMP 780275745Skib } else if (TD_IS_RUNNING(td2) && td != td2) { 781155594Sdavidxu forward_signal(td2); 782275745Skib#endif 783155594Sdavidxu } 784170296Sjeff thread_unlock(td2); 78599026Sjulian } 786181334Sjhb if (wakeup_swapper) 787181334Sjhb kick_proc0(); 788195701Skib remaining = calc_remaining(p, mode); 789130674Sdavidxu 790124350Sschweikh /* 791124350Sschweikh * Maybe we suspended some threads.. was it enough? 792105911Sjulian */ 793275617Skib if (remaining == remain_for_mode(mode)) 794105911Sjulian break; 795105911Sjulian 796156942Sdavidxustopme: 79799026Sjulian /* 79899026Sjulian * Wake us up when everyone else has suspended. 799100648Sjulian * In the mean time we suspend as well. 80099026Sjulian */ 801275745Skib thread_suspend_switch(td, p); 802195701Skib remaining = calc_remaining(p, mode); 80399026Sjulian } 804136177Sdavidxu if (mode == SINGLE_EXIT) { 805135269Sjulian /* 806271007Skib * Convert the process to an unthreaded process. The 807271007Skib * SINGLE_EXIT is called by exit1() or execve(), in 808271007Skib * both cases other threads must be retired. 809135269Sjulian */ 810271007Skib KASSERT(p->p_numthreads == 1, ("Unthreading with >1 threads")); 811136160Sjulian p->p_singlethread = NULL; 812271007Skib p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS); 813271000Skib 814271000Skib /* 815271000Skib * Wait for any remaining threads to exit cpu_throw(). 816271000Skib */ 817271000Skib while (p->p_exitthreads != 0) { 818271000Skib PROC_SUNLOCK(p); 819271000Skib PROC_UNLOCK(p); 820271000Skib sched_relinquish(td); 821271000Skib PROC_LOCK(p); 822271000Skib PROC_SLOCK(p); 823271000Skib } 824282679Skib } else if (mode == SINGLE_BOUNDARY) { 825282679Skib /* 826282679Skib * Wait until all suspended threads are removed from 827282679Skib * the processors. The thread_suspend_check() 828282679Skib * increments p_boundary_count while it is still 829282679Skib * running, which makes it possible for the execve() 830282679Skib * to destroy vmspace while our other threads are 831282679Skib * still using the address space. 832282679Skib * 833282679Skib * We lock the thread, which is only allowed to 834282679Skib * succeed after context switch code finished using 835282679Skib * the address space. 836282679Skib */ 837282679Skib FOREACH_THREAD_IN_PROC(p, td2) { 838282679Skib if (td2 == td) 839282679Skib continue; 840282679Skib thread_lock(td2); 841282679Skib KASSERT((td2->td_flags & TDF_BOUNDARY) != 0, 842282679Skib ("td %p not on boundary", td2)); 843282679Skib KASSERT(TD_IS_SUSPENDED(td2), 844282679Skib ("td %p is not suspended", td2)); 845282679Skib thread_unlock(td2); 846282679Skib } 847111028Sjeff } 848184667Sdavidxu PROC_SUNLOCK(p); 84999026Sjulian return (0); 85099026Sjulian} 85199026Sjulian 852275616Skibbool 853275616Skibthread_suspend_check_needed(void) 854275616Skib{ 855275616Skib struct proc *p; 856275616Skib struct thread *td; 857275616Skib 858275616Skib td = curthread; 859275616Skib p = td->td_proc; 860275616Skib PROC_LOCK_ASSERT(p, MA_OWNED); 861275616Skib return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 && 862275616Skib (td->td_dbgflags & TDB_SUSPEND) != 0)); 863275616Skib} 864275616Skib 86599026Sjulian/* 86699026Sjulian * Called in from locations that can safely check to see 86799026Sjulian * whether we have to suspend or at least throttle for a 86899026Sjulian * single-thread event (e.g. fork). 86999026Sjulian * 87099026Sjulian * Such locations include userret(). 87199026Sjulian * If the "return_instead" argument is non zero, the thread must be able to 87299026Sjulian * accept 0 (caller may continue), or 1 (caller must abort) as a result. 87399026Sjulian * 87499026Sjulian * The 'return_instead' argument tells the function if it may do a 87599026Sjulian * thread_exit() or suspend, or whether the caller must abort and back 87699026Sjulian * out instead. 87799026Sjulian * 87899026Sjulian * If the thread that set the single_threading request has set the 87999026Sjulian * P_SINGLE_EXIT bit in the process flags then this call will never return 88099026Sjulian * if 'return_instead' is false, but will exit. 88199026Sjulian * 88299026Sjulian * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 88399026Sjulian *---------------+--------------------+--------------------- 88499026Sjulian * 0 | returns 0 | returns 0 or 1 885246996Sjhb * | when ST ends | immediately 88699026Sjulian *---------------+--------------------+--------------------- 88799026Sjulian * 1 | thread exits | returns 1 888246996Sjhb * | | immediately 88999026Sjulian * 0 = thread_exit() or suspension ok, 89099026Sjulian * other = return error instead of stopping the thread. 89199026Sjulian * 89299026Sjulian * While a full suspension is under effect, even a single threading 89399026Sjulian * thread would be suspended if it made this call (but it shouldn't). 89499026Sjulian * This call should only be made from places where 895124350Sschweikh * thread_exit() would be safe as that may be the outcome unless 89699026Sjulian * return_instead is set. 89799026Sjulian */ 89899026Sjulianint 89999026Sjulianthread_suspend_check(int return_instead) 90099026Sjulian{ 901104502Sjmallett struct thread *td; 902104502Sjmallett struct proc *p; 903182011Sjhb int wakeup_swapper; 90499026Sjulian 90599026Sjulian td = curthread; 90699026Sjulian p = td->td_proc; 907126932Speter mtx_assert(&Giant, MA_NOTOWNED); 90899026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 909275616Skib while (thread_suspend_check_needed()) { 910102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 91199026Sjulian KASSERT(p->p_singlethread != NULL, 91299026Sjulian ("singlethread not set")); 91399026Sjulian /* 914100648Sjulian * The only suspension in action is a 915100648Sjulian * single-threading. Single threader need not stop. 916124350Sschweikh * XXX Should be safe to access unlocked 917100646Sjulian * as it can only be set to be true by us. 91899026Sjulian */ 919100648Sjulian if (p->p_singlethread == td) 92099026Sjulian return (0); /* Exempt from stopping. */ 921124350Sschweikh } 922134498Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 923155741Sdavidxu return (EINTR); 92499026Sjulian 925136177Sdavidxu /* Should we goto user boundary if we didn't come from there? */ 926136177Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 927136177Sdavidxu (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 928155741Sdavidxu return (ERESTART); 929136177Sdavidxu 93099026Sjulian /* 931283320Skib * Ignore suspend requests if they are deferred. 932248584Sjhb */ 933283320Skib if ((td->td_flags & TDF_SBDRY) != 0) { 934248584Sjhb KASSERT(return_instead, 935248584Sjhb ("TDF_SBDRY set for unsafe thread_suspend_check")); 936248584Sjhb return (0); 937248584Sjhb } 938248584Sjhb 939248584Sjhb /* 94099026Sjulian * If the process is waiting for us to exit, 94199026Sjulian * this thread should just suicide. 942102950Sdavidxu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 94399026Sjulian */ 944213642Sdavidxu if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 945213642Sdavidxu PROC_UNLOCK(p); 946283382Sdchagin 947283382Sdchagin /* 948283382Sdchagin * Allow Linux emulation layer to do some work 949283382Sdchagin * before thread suicide. 950283382Sdchagin */ 951283382Sdchagin if (__predict_false(p->p_sysent->sv_thread_detach != NULL)) 952283382Sdchagin (p->p_sysent->sv_thread_detach)(td); 953292892Sjhb kern_thr_exit(td); 954292892Sjhb panic("stopped thread did not exit"); 955213642Sdavidxu } 956213950Sdavidxu 957213950Sdavidxu PROC_SLOCK(p); 958213950Sdavidxu thread_stopped(p); 959170296Sjeff if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 960170296Sjeff if (p->p_numthreads == p->p_suspcount + 1) { 961170296Sjeff thread_lock(p->p_singlethread); 962282944Skib wakeup_swapper = thread_unsuspend_one( 963282944Skib p->p_singlethread, p, false); 964170296Sjeff thread_unlock(p->p_singlethread); 965182011Sjhb if (wakeup_swapper) 966182011Sjhb kick_proc0(); 967170296Sjeff } 968170296Sjeff } 969184667Sdavidxu PROC_UNLOCK(p); 970184199Sdavidxu thread_lock(td); 97199026Sjulian /* 97299026Sjulian * When a thread suspends, it just 973164936Sjulian * gets taken off all queues. 97499026Sjulian */ 975103216Sjulian thread_suspend_one(td); 976136177Sdavidxu if (return_instead == 0) { 977136177Sdavidxu p->p_boundary_count++; 978136177Sdavidxu td->td_flags |= TDF_BOUNDARY; 979136177Sdavidxu } 980184667Sdavidxu PROC_SUNLOCK(p); 981178272Sjeff mi_switch(SW_INVOL | SWT_SUSPEND, NULL); 982170296Sjeff thread_unlock(td); 98399026Sjulian PROC_LOCK(p); 98499026Sjulian } 98599026Sjulian return (0); 98699026Sjulian} 98799026Sjulian 988102898Sdavidxuvoid 989275745Skibthread_suspend_switch(struct thread *td, struct proc *p) 990170296Sjeff{ 991170296Sjeff 992170296Sjeff KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 993170296Sjeff PROC_LOCK_ASSERT(p, MA_OWNED); 994184667Sdavidxu PROC_SLOCK_ASSERT(p, MA_OWNED); 995170296Sjeff /* 996170296Sjeff * We implement thread_suspend_one in stages here to avoid 997170296Sjeff * dropping the proc lock while the thread lock is owned. 998170296Sjeff */ 999275745Skib if (p == td->td_proc) { 1000275745Skib thread_stopped(p); 1001275745Skib p->p_suspcount++; 1002275745Skib } 1003184667Sdavidxu PROC_UNLOCK(p); 1004184199Sdavidxu thread_lock(td); 1005177471Sjeff td->td_flags &= ~TDF_NEEDSUSPCHK; 1006170296Sjeff TD_SET_SUSPENDED(td); 1007177085Sjeff sched_sleep(td, 0); 1008184667Sdavidxu PROC_SUNLOCK(p); 1009170296Sjeff DROP_GIANT(); 1010178272Sjeff mi_switch(SW_VOL | SWT_SUSPEND, NULL); 1011170296Sjeff thread_unlock(td); 1012170296Sjeff PICKUP_GIANT(); 1013170296Sjeff PROC_LOCK(p); 1014184667Sdavidxu PROC_SLOCK(p); 1015170296Sjeff} 1016170296Sjeff 1017170296Sjeffvoid 1018102898Sdavidxuthread_suspend_one(struct thread *td) 1019102898Sdavidxu{ 1020275745Skib struct proc *p; 1021102898Sdavidxu 1022275745Skib p = td->td_proc; 1023184667Sdavidxu PROC_SLOCK_ASSERT(p, MA_OWNED); 1024170296Sjeff THREAD_LOCK_ASSERT(td, MA_OWNED); 1025112071Sdavidxu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 1026102898Sdavidxu p->p_suspcount++; 1027177471Sjeff td->td_flags &= ~TDF_NEEDSUSPCHK; 1028103216Sjulian TD_SET_SUSPENDED(td); 1029177085Sjeff sched_sleep(td, 0); 1030102898Sdavidxu} 1031102898Sdavidxu 1032282944Skibstatic int 1033282944Skibthread_unsuspend_one(struct thread *td, struct proc *p, bool boundary) 1034102898Sdavidxu{ 1035102898Sdavidxu 1036170296Sjeff THREAD_LOCK_ASSERT(td, MA_OWNED); 1037164936Sjulian KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 1038103216Sjulian TD_CLR_SUSPENDED(td); 1039275745Skib td->td_flags &= ~TDF_ALLPROCSUSP; 1040275745Skib if (td->td_proc == p) { 1041275745Skib PROC_SLOCK_ASSERT(p, MA_OWNED); 1042275745Skib p->p_suspcount--; 1043282944Skib if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) { 1044282944Skib td->td_flags &= ~TDF_BOUNDARY; 1045282944Skib p->p_boundary_count--; 1046282944Skib } 1047275745Skib } 1048182011Sjhb return (setrunnable(td)); 1049102898Sdavidxu} 1050102898Sdavidxu 105199026Sjulian/* 105299026Sjulian * Allow all threads blocked by single threading to continue running. 105399026Sjulian */ 105499026Sjulianvoid 105599026Sjulianthread_unsuspend(struct proc *p) 105699026Sjulian{ 105799026Sjulian struct thread *td; 1058182011Sjhb int wakeup_swapper; 105999026Sjulian 106099026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 1061184667Sdavidxu PROC_SLOCK_ASSERT(p, MA_OWNED); 1062182011Sjhb wakeup_swapper = 0; 106399026Sjulian if (!P_SHOULDSTOP(p)) { 1064164936Sjulian FOREACH_THREAD_IN_PROC(p, td) { 1065170296Sjeff thread_lock(td); 1066164936Sjulian if (TD_IS_SUSPENDED(td)) { 1067282944Skib wakeup_swapper |= thread_unsuspend_one(td, p, 1068282944Skib true); 1069164936Sjulian } 1070170296Sjeff thread_unlock(td); 107199026Sjulian } 1072282944Skib } else if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 1073282944Skib p->p_numthreads == p->p_suspcount) { 107499026Sjulian /* 107599026Sjulian * Stopping everything also did the job for the single 107699026Sjulian * threading request. Now we've downgraded to single-threaded, 107799026Sjulian * let it continue. 107899026Sjulian */ 1079275745Skib if (p->p_singlethread->td_proc == p) { 1080275745Skib thread_lock(p->p_singlethread); 1081275745Skib wakeup_swapper = thread_unsuspend_one( 1082282944Skib p->p_singlethread, p, false); 1083275745Skib thread_unlock(p->p_singlethread); 1084275745Skib } 108599026Sjulian } 1086182011Sjhb if (wakeup_swapper) 1087182011Sjhb kick_proc0(); 108899026Sjulian} 108999026Sjulian 1090134791Sjulian/* 1091134791Sjulian * End the single threading mode.. 1092134791Sjulian */ 109399026Sjulianvoid 1094275745Skibthread_single_end(struct proc *p, int mode) 109599026Sjulian{ 109699026Sjulian struct thread *td; 1097182011Sjhb int wakeup_swapper; 109899026Sjulian 1099275745Skib KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY || 1100275745Skib mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT, 1101275745Skib ("invalid mode %d", mode)); 110299026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 1103275745Skib KASSERT((mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) || 1104275745Skib (mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0), 1105275745Skib ("mode %d does not match P_TOTAL_STOP", mode)); 1106282944Skib KASSERT(mode == SINGLE_ALLPROC || p->p_singlethread == curthread, 1107282944Skib ("thread_single_end from other thread %p %p", 1108282944Skib curthread, p->p_singlethread)); 1109282944Skib KASSERT(mode != SINGLE_BOUNDARY || 1110282944Skib (p->p_flag & P_SINGLE_BOUNDARY) != 0, 1111282944Skib ("mis-matched SINGLE_BOUNDARY flags %x", p->p_flag)); 1112275745Skib p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY | 1113275745Skib P_TOTAL_STOP); 1114184667Sdavidxu PROC_SLOCK(p); 111599026Sjulian p->p_singlethread = NULL; 1116182011Sjhb wakeup_swapper = 0; 1117102292Sjulian /* 1118182011Sjhb * If there are other threads they may now run, 1119102292Sjulian * unless of course there is a blanket 'stop order' 1120102292Sjulian * on the process. The single threader must be allowed 1121102292Sjulian * to continue however as this is a bad place to stop. 1122102292Sjulian */ 1123275745Skib if (p->p_numthreads != remain_for_mode(mode) && !P_SHOULDSTOP(p)) { 1124164936Sjulian FOREACH_THREAD_IN_PROC(p, td) { 1125170296Sjeff thread_lock(td); 1126164936Sjulian if (TD_IS_SUSPENDED(td)) { 1127282944Skib wakeup_swapper |= thread_unsuspend_one(td, p, 1128282944Skib mode == SINGLE_BOUNDARY); 1129164936Sjulian } 1130170296Sjeff thread_unlock(td); 1131102292Sjulian } 1132102292Sjulian } 1133282944Skib KASSERT(mode != SINGLE_BOUNDARY || p->p_boundary_count == 0, 1134282944Skib ("inconsistent boundary count %d", p->p_boundary_count)); 1135184667Sdavidxu PROC_SUNLOCK(p); 1136182011Sjhb if (wakeup_swapper) 1137182011Sjhb kick_proc0(); 113899026Sjulian} 1139128721Sdeischen 1140151990Sdavidxustruct thread * 1141151990Sdavidxuthread_find(struct proc *p, lwpid_t tid) 1142151990Sdavidxu{ 1143151990Sdavidxu struct thread *td; 1144151990Sdavidxu 1145151990Sdavidxu PROC_LOCK_ASSERT(p, MA_OWNED); 1146151990Sdavidxu FOREACH_THREAD_IN_PROC(p, td) { 1147151990Sdavidxu if (td->td_tid == tid) 1148151990Sdavidxu break; 1149151990Sdavidxu } 1150151990Sdavidxu return (td); 1151151990Sdavidxu} 1152213642Sdavidxu 1153213642Sdavidxu/* Locate a thread by number; return with proc lock held. */ 1154213642Sdavidxustruct thread * 1155213642Sdavidxutdfind(lwpid_t tid, pid_t pid) 1156213642Sdavidxu{ 1157213642Sdavidxu#define RUN_THRESH 16 1158213642Sdavidxu struct thread *td; 1159213642Sdavidxu int run = 0; 1160213642Sdavidxu 1161213642Sdavidxu rw_rlock(&tidhash_lock); 1162213642Sdavidxu LIST_FOREACH(td, TIDHASH(tid), td_hash) { 1163213642Sdavidxu if (td->td_tid == tid) { 1164213642Sdavidxu if (pid != -1 && td->td_proc->p_pid != pid) { 1165213642Sdavidxu td = NULL; 1166213642Sdavidxu break; 1167213642Sdavidxu } 1168219968Sjhb PROC_LOCK(td->td_proc); 1169213642Sdavidxu if (td->td_proc->p_state == PRS_NEW) { 1170219968Sjhb PROC_UNLOCK(td->td_proc); 1171213642Sdavidxu td = NULL; 1172213642Sdavidxu break; 1173213642Sdavidxu } 1174213642Sdavidxu if (run > RUN_THRESH) { 1175213642Sdavidxu if (rw_try_upgrade(&tidhash_lock)) { 1176213642Sdavidxu LIST_REMOVE(td, td_hash); 1177213642Sdavidxu LIST_INSERT_HEAD(TIDHASH(td->td_tid), 1178213642Sdavidxu td, td_hash); 1179213642Sdavidxu rw_wunlock(&tidhash_lock); 1180213642Sdavidxu return (td); 1181213642Sdavidxu } 1182213642Sdavidxu } 1183213642Sdavidxu break; 1184213642Sdavidxu } 1185213642Sdavidxu run++; 1186213642Sdavidxu } 1187213642Sdavidxu rw_runlock(&tidhash_lock); 1188213642Sdavidxu return (td); 1189213642Sdavidxu} 1190213642Sdavidxu 1191213642Sdavidxuvoid 1192213642Sdavidxutidhash_add(struct thread *td) 1193213642Sdavidxu{ 1194213642Sdavidxu rw_wlock(&tidhash_lock); 1195213950Sdavidxu LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); 1196213642Sdavidxu rw_wunlock(&tidhash_lock); 1197213642Sdavidxu} 1198213642Sdavidxu 1199213642Sdavidxuvoid 1200213642Sdavidxutidhash_remove(struct thread *td) 1201213642Sdavidxu{ 1202213642Sdavidxu rw_wlock(&tidhash_lock); 1203213950Sdavidxu LIST_REMOVE(td, td_hash); 1204213642Sdavidxu rw_wunlock(&tidhash_lock); 1205213642Sdavidxu} 1206