kern_thread.c revision 132987
1124350Sschweikh/* 299026Sjulian * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 399026Sjulian * All rights reserved. 499026Sjulian * 599026Sjulian * Redistribution and use in source and binary forms, with or without 699026Sjulian * modification, are permitted provided that the following conditions 799026Sjulian * are met: 899026Sjulian * 1. Redistributions of source code must retain the above copyright 999026Sjulian * notice(s), this list of conditions and the following disclaimer as 10124350Sschweikh * the first lines of this file unmodified other than the possible 1199026Sjulian * addition of one or more copyright notices. 1299026Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1399026Sjulian * notice(s), this list of conditions and the following disclaimer in the 1499026Sjulian * documentation and/or other materials provided with the distribution. 1599026Sjulian * 1699026Sjulian * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1799026Sjulian * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1899026Sjulian * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1999026Sjulian * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2099026Sjulian * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2199026Sjulian * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2299026Sjulian * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2399026Sjulian * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2499026Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2599026Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2699026Sjulian * DAMAGE. 2799026Sjulian */ 2899026Sjulian 29116182Sobrien#include <sys/cdefs.h> 30116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_thread.c 132987 2004-08-02 00:18:36Z green $"); 31116182Sobrien 3299026Sjulian#include <sys/param.h> 3399026Sjulian#include <sys/systm.h> 3499026Sjulian#include <sys/kernel.h> 3599026Sjulian#include <sys/lock.h> 3699026Sjulian#include <sys/mutex.h> 3799026Sjulian#include <sys/proc.h> 38130355Sjulian#include <sys/smp.h> 3999026Sjulian#include <sys/sysctl.h> 40107126Sjeff#include <sys/sched.h> 41126326Sjhb#include <sys/sleepqueue.h> 42122514Sjhb#include <sys/turnstile.h> 4399026Sjulian#include <sys/ktr.h> 4499026Sjulian 4599026Sjulian#include <vm/vm.h> 46116355Salc#include <vm/vm_extern.h> 4799026Sjulian#include <vm/uma.h> 4899026Sjulian 4999026Sjulian/* 50103367Sjulian * KSEGRP related storage. 5199026Sjulian */ 52103367Sjulianstatic uma_zone_t ksegrp_zone; 53103367Sjulianstatic uma_zone_t kse_zone; 5499026Sjulianstatic uma_zone_t thread_zone; 5599026Sjulian 56103367Sjulian/* DEBUG ONLY */ 5799026SjulianSYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 58107719Sjulianstatic int thread_debug = 0; 59107719SjulianSYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 60107719Sjulian &thread_debug, 0, "thread debug"); 6199026Sjulian 62130199Sjulianint max_threads_per_proc = 1500; 63107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 64103367Sjulian &max_threads_per_proc, 0, "Limit on threads per proc"); 65103367Sjulian 66130199Sjulianint max_groups_per_proc = 500; 67107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 68107006Sdavidxu &max_groups_per_proc, 0, "Limit on thread groups per proc"); 69107006Sdavidxu 70130199Sjulianint max_threads_hits; 71111115SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 72111115Sdavidxu &max_threads_hits, 0, ""); 73111115Sdavidxu 74130355Sjulianint virtual_cpu; 75111028Sjeff 76130355Sjulian#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 77130355Sjulian 78111028SjeffTAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 79105854SjulianTAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 80105854SjulianTAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 81111028Sjeffstruct mtx kse_zombie_lock; 82111028SjeffMTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 8399026Sjulian 84130199Sjulianvoid kse_purge(struct proc *p, struct thread *td); 85130199Sjulianvoid kse_purge_group(struct thread *td); 86105854Sjulian 87130199Sjulian/* move to proc.h */ 88130199Sjulianextern void kseinit(void); 89130199Sjulianextern void kse_GC(void); 90111028Sjeff 91111028Sjeff 92130355Sjulianstatic int 93130355Sjuliansysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 94130355Sjulian{ 95130355Sjulian int error, new_val; 96130355Sjulian int def_val; 97111028Sjeff 98130355Sjulian def_val = mp_ncpus; 99130355Sjulian if (virtual_cpu == 0) 100130355Sjulian new_val = def_val; 101130355Sjulian else 102130355Sjulian new_val = virtual_cpu; 103130355Sjulian error = sysctl_handle_int(oidp, &new_val, 0, req); 104130355Sjulian if (error != 0 || req->newptr == NULL) 105130355Sjulian return (error); 106130355Sjulian if (new_val < 0) 107130355Sjulian return (EINVAL); 108130355Sjulian virtual_cpu = new_val; 109130355Sjulian return (0); 110130355Sjulian} 111130355Sjulian 112130355Sjulian/* DEBUG ONLY */ 113130355SjulianSYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 114130355Sjulian 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 115130355Sjulian "debug virtual cpus"); 116130355Sjulian 11799026Sjulian/* 118127794Smarcel * Thread ID allocator. The allocator keeps track of assigned IDs by 119127794Smarcel * using a bitmap. The bitmap is created in parts. The parts are linked 120127794Smarcel * together. 121127794Smarcel */ 122127794Smarceltypedef u_long tid_bitmap_word; 123127794Smarcel 124127794Smarcel#define TID_IDS_PER_PART 1024 125127794Smarcel#define TID_IDS_PER_IDX (sizeof(tid_bitmap_word) << 3) 126127794Smarcel#define TID_BITMAP_SIZE (TID_IDS_PER_PART / TID_IDS_PER_IDX) 127127794Smarcel#define TID_MIN (PID_MAX + 1) 128127794Smarcel 129127794Smarcelstruct tid_bitmap_part { 130127794Smarcel STAILQ_ENTRY(tid_bitmap_part) bmp_next; 131127794Smarcel tid_bitmap_word bmp_bitmap[TID_BITMAP_SIZE]; 132130735Smarcel lwpid_t bmp_base; 133127794Smarcel int bmp_free; 134127794Smarcel}; 135127794Smarcel 136127794Smarcelstatic STAILQ_HEAD(, tid_bitmap_part) tid_bitmap = 137127794Smarcel STAILQ_HEAD_INITIALIZER(tid_bitmap); 138127794Smarcelstatic uma_zone_t tid_zone; 139127794Smarcel 140127794Smarcelstruct mtx tid_lock; 141127794SmarcelMTX_SYSINIT(tid_lock, &tid_lock, "TID lock", MTX_DEF); 142127794Smarcel 143127794Smarcel/* 144107719Sjulian * Prepare a thread for use. 14599026Sjulian */ 146132987Sgreenstatic int 147132987Sgreenthread_ctor(void *mem, int size, void *arg, int flags) 14899026Sjulian{ 14999026Sjulian struct thread *td; 15099026Sjulian 15199026Sjulian td = (struct thread *)mem; 152103216Sjulian td->td_state = TDS_INACTIVE; 153113339Sjulian td->td_oncpu = NOCPU; 154130269Sjmallett 155130269Sjmallett /* 156130269Sjmallett * Note that td_critnest begins life as 1 because the thread is not 157130269Sjmallett * running and is thereby implicitly waiting to be on the receiving 158130269Sjmallett * end of a context switch. A context switch must occur inside a 159130269Sjmallett * critical section, and in fact, includes hand-off of the sched_lock. 160130269Sjmallett * After a context switch to a newly created thread, it will release 161130269Sjmallett * sched_lock for the first time, and its td_critnest will hit 0 for 162130269Sjmallett * the first time. This happens on the far end of a context switch, 163130269Sjmallett * and when it context switches away from itself, it will in fact go 164130269Sjmallett * back into a critical section, and hand off the sched lock to the 165130269Sjmallett * next thread. 166130269Sjmallett */ 167118442Sjhb td->td_critnest = 1; 168132987Sgreen return (0); 16999026Sjulian} 17099026Sjulian 17199026Sjulian/* 17299026Sjulian * Reclaim a thread after use. 17399026Sjulian */ 17499026Sjulianstatic void 17599026Sjulianthread_dtor(void *mem, int size, void *arg) 17699026Sjulian{ 177127794Smarcel struct thread *td; 17899026Sjulian 17999026Sjulian td = (struct thread *)mem; 18099026Sjulian 18199026Sjulian#ifdef INVARIANTS 18299026Sjulian /* Verify that this thread is in a safe state to free. */ 18399026Sjulian switch (td->td_state) { 184103216Sjulian case TDS_INHIBITED: 185103216Sjulian case TDS_RUNNING: 186103216Sjulian case TDS_CAN_RUN: 18799026Sjulian case TDS_RUNQ: 18899026Sjulian /* 18999026Sjulian * We must never unlink a thread that is in one of 19099026Sjulian * these states, because it is currently active. 19199026Sjulian */ 19299026Sjulian panic("bad state for thread unlinking"); 19399026Sjulian /* NOTREACHED */ 194103216Sjulian case TDS_INACTIVE: 19599026Sjulian break; 19699026Sjulian default: 19799026Sjulian panic("bad thread state"); 19899026Sjulian /* NOTREACHED */ 19999026Sjulian } 20099026Sjulian#endif 20199026Sjulian} 20299026Sjulian 20399026Sjulian/* 20499026Sjulian * Initialize type-stable parts of a thread (when newly created). 20599026Sjulian */ 206132987Sgreenstatic int 207132987Sgreenthread_init(void *mem, int size, int flags) 20899026Sjulian{ 209131149Smarcel struct thread *td; 210131149Smarcel struct tid_bitmap_part *bmp, *new; 211131149Smarcel int bit, idx; 21299026Sjulian 21399026Sjulian td = (struct thread *)mem; 214131149Smarcel 215131149Smarcel mtx_lock(&tid_lock); 216131149Smarcel STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { 217131149Smarcel if (bmp->bmp_free) 218131149Smarcel break; 219131149Smarcel } 220131149Smarcel /* Create a new bitmap if we run out of free bits. */ 221131149Smarcel if (bmp == NULL) { 222131149Smarcel mtx_unlock(&tid_lock); 223131149Smarcel new = uma_zalloc(tid_zone, M_WAITOK); 224131149Smarcel mtx_lock(&tid_lock); 225131149Smarcel bmp = STAILQ_LAST(&tid_bitmap, tid_bitmap_part, bmp_next); 226131149Smarcel if (bmp == NULL || bmp->bmp_free < TID_IDS_PER_PART/2) { 227131149Smarcel /* 1=free, 0=assigned. This way we can use ffsl(). */ 228131149Smarcel memset(new->bmp_bitmap, ~0U, sizeof(new->bmp_bitmap)); 229131149Smarcel new->bmp_base = (bmp == NULL) ? TID_MIN : 230131149Smarcel bmp->bmp_base + TID_IDS_PER_PART; 231131149Smarcel new->bmp_free = TID_IDS_PER_PART; 232131149Smarcel STAILQ_INSERT_TAIL(&tid_bitmap, new, bmp_next); 233131149Smarcel bmp = new; 234131149Smarcel new = NULL; 235131149Smarcel } 236131149Smarcel } else 237131149Smarcel new = NULL; 238131149Smarcel /* We have a bitmap with available IDs. */ 239131149Smarcel idx = 0; 240131149Smarcel while (idx < TID_BITMAP_SIZE && bmp->bmp_bitmap[idx] == 0UL) 241131149Smarcel idx++; 242131149Smarcel bit = ffsl(bmp->bmp_bitmap[idx]) - 1; 243131149Smarcel td->td_tid = bmp->bmp_base + idx * TID_IDS_PER_IDX + bit; 244131149Smarcel bmp->bmp_bitmap[idx] &= ~(1UL << bit); 245131149Smarcel bmp->bmp_free--; 246131149Smarcel mtx_unlock(&tid_lock); 247131149Smarcel if (new != NULL) 248131149Smarcel uma_zfree(tid_zone, new); 249131149Smarcel 250116355Salc vm_thread_new(td, 0); 25199026Sjulian cpu_thread_setup(td); 252126326Sjhb td->td_sleepqueue = sleepq_alloc(); 253122514Sjhb td->td_turnstile = turnstile_alloc(); 254107126Sjeff td->td_sched = (struct td_sched *)&td[1]; 255132987Sgreen return (0); 25699026Sjulian} 25799026Sjulian 25899026Sjulian/* 25999026Sjulian * Tear down type-stable parts of a thread (just before being discarded). 26099026Sjulian */ 26199026Sjulianstatic void 26299026Sjulianthread_fini(void *mem, int size) 26399026Sjulian{ 264131149Smarcel struct thread *td; 265131149Smarcel struct tid_bitmap_part *bmp; 266131149Smarcel lwpid_t tid; 267131149Smarcel int bit, idx; 26899026Sjulian 26999026Sjulian td = (struct thread *)mem; 270122514Sjhb turnstile_free(td->td_turnstile); 271126326Sjhb sleepq_free(td->td_sleepqueue); 272116355Salc vm_thread_dispose(td); 273131149Smarcel 274131149Smarcel STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { 275131149Smarcel if (td->td_tid >= bmp->bmp_base && 276131149Smarcel td->td_tid < bmp->bmp_base + TID_IDS_PER_PART) 277131149Smarcel break; 278131149Smarcel } 279131149Smarcel KASSERT(bmp != NULL, ("No TID bitmap?")); 280131149Smarcel mtx_lock(&tid_lock); 281131149Smarcel tid = td->td_tid - bmp->bmp_base; 282131149Smarcel idx = tid / TID_IDS_PER_IDX; 283131149Smarcel bit = 1UL << (tid % TID_IDS_PER_IDX); 284131149Smarcel bmp->bmp_bitmap[idx] |= bit; 285131149Smarcel bmp->bmp_free++; 286131149Smarcel mtx_unlock(&tid_lock); 28799026Sjulian} 288111028Sjeff 289107126Sjeff/* 290107126Sjeff * Initialize type-stable parts of a kse (when newly created). 291107126Sjeff */ 292132987Sgreenstatic int 293132987Sgreenkse_init(void *mem, int size, int flags) 294107126Sjeff{ 295107126Sjeff struct kse *ke; 29699026Sjulian 297107126Sjeff ke = (struct kse *)mem; 298107126Sjeff ke->ke_sched = (struct ke_sched *)&ke[1]; 299132987Sgreen return (0); 300107126Sjeff} 301111028Sjeff 302107126Sjeff/* 303107126Sjeff * Initialize type-stable parts of a ksegrp (when newly created). 304107126Sjeff */ 305132987Sgreenstatic int 306132987Sgreenksegrp_init(void *mem, int size, int flags) 307107126Sjeff{ 308107126Sjeff struct ksegrp *kg; 309107126Sjeff 310107126Sjeff kg = (struct ksegrp *)mem; 311107126Sjeff kg->kg_sched = (struct kg_sched *)&kg[1]; 312132987Sgreen return (0); 313107126Sjeff} 314107126Sjeff 315124350Sschweikh/* 316111028Sjeff * KSE is linked into kse group. 317105854Sjulian */ 318105854Sjulianvoid 319105854Sjuliankse_link(struct kse *ke, struct ksegrp *kg) 320105854Sjulian{ 321105854Sjulian struct proc *p = kg->kg_proc; 322105854Sjulian 323105854Sjulian TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 324105854Sjulian kg->kg_kses++; 325111028Sjeff ke->ke_state = KES_UNQUEUED; 326105854Sjulian ke->ke_proc = p; 327105854Sjulian ke->ke_ksegrp = kg; 328105854Sjulian ke->ke_thread = NULL; 329111028Sjeff ke->ke_oncpu = NOCPU; 330111028Sjeff ke->ke_flags = 0; 331105854Sjulian} 332105854Sjulian 333105854Sjulianvoid 334105854Sjuliankse_unlink(struct kse *ke) 335105854Sjulian{ 336105854Sjulian struct ksegrp *kg; 337105854Sjulian 338105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 339105854Sjulian kg = ke->ke_ksegrp; 340105854Sjulian TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 341111028Sjeff if (ke->ke_state == KES_IDLE) { 342111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 343111028Sjeff kg->kg_idle_kses--; 344105854Sjulian } 345119488Sdavidxu --kg->kg_kses; 346105854Sjulian /* 347105854Sjulian * Aggregate stats from the KSE 348105854Sjulian */ 349105854Sjulian kse_stash(ke); 350105854Sjulian} 351105854Sjulian 352105854Sjulianvoid 353105854Sjulianksegrp_link(struct ksegrp *kg, struct proc *p) 354105854Sjulian{ 355105854Sjulian 356105854Sjulian TAILQ_INIT(&kg->kg_threads); 357105854Sjulian TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 358105854Sjulian TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 359105854Sjulian TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 360111028Sjeff TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 361111028Sjeff TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 362111028Sjeff kg->kg_proc = p; 363111028Sjeff /* 364111028Sjeff * the following counters are in the -zero- section 365111028Sjeff * and may not need clearing 366111028Sjeff */ 367105854Sjulian kg->kg_numthreads = 0; 368111028Sjeff kg->kg_runnable = 0; 369111028Sjeff kg->kg_kses = 0; 370111028Sjeff kg->kg_runq_kses = 0; /* XXXKSE change name */ 371111028Sjeff kg->kg_idle_kses = 0; 372111028Sjeff kg->kg_numupcalls = 0; 373111028Sjeff /* link it in now that it's consistent */ 374105854Sjulian p->p_numksegrps++; 375105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 376105854Sjulian} 377105854Sjulian 378105854Sjulianvoid 379105854Sjulianksegrp_unlink(struct ksegrp *kg) 380105854Sjulian{ 381105854Sjulian struct proc *p; 382105854Sjulian 383105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 384111028Sjeff KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 385111028Sjeff KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 386111028Sjeff KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 387111028Sjeff 388105854Sjulian p = kg->kg_proc; 389105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 390105854Sjulian p->p_numksegrps--; 391105854Sjulian /* 392105854Sjulian * Aggregate stats from the KSE 393105854Sjulian */ 394105854Sjulian ksegrp_stash(kg); 395105854Sjulian} 396105854Sjulian 39799026Sjulian/* 398111028Sjeff * For a newly created process, 399111028Sjeff * link up all the structures and its initial threads etc. 400105854Sjulian */ 401105854Sjulianvoid 402105854Sjulianproc_linkup(struct proc *p, struct ksegrp *kg, 403111028Sjeff struct kse *ke, struct thread *td) 404105854Sjulian{ 405105854Sjulian 406105854Sjulian TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 407105854Sjulian TAILQ_INIT(&p->p_threads); /* all threads in proc */ 408105854Sjulian TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 409105854Sjulian p->p_numksegrps = 0; 410105854Sjulian p->p_numthreads = 0; 411105854Sjulian 412105854Sjulian ksegrp_link(kg, p); 413105854Sjulian kse_link(ke, kg); 414105854Sjulian thread_link(td, kg); 415105854Sjulian} 416105854Sjulian 417111028Sjeff/* 41899026Sjulian * Initialize global thread allocation resources. 41999026Sjulian */ 42099026Sjulianvoid 42199026Sjulianthreadinit(void) 42299026Sjulian{ 42399026Sjulian 424107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 42599026Sjulian thread_ctor, thread_dtor, thread_init, thread_fini, 42699026Sjulian UMA_ALIGN_CACHE, 0); 427127794Smarcel tid_zone = uma_zcreate("TID", sizeof(struct tid_bitmap_part), 428127794Smarcel NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 429107126Sjeff ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 430107126Sjeff NULL, NULL, ksegrp_init, NULL, 431103367Sjulian UMA_ALIGN_CACHE, 0); 432107126Sjeff kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 433107126Sjeff NULL, NULL, kse_init, NULL, 434103367Sjulian UMA_ALIGN_CACHE, 0); 435130199Sjulian kseinit(); 43699026Sjulian} 43799026Sjulian 43899026Sjulian/* 439103002Sjulian * Stash an embarasingly extra thread into the zombie thread queue. 44099026Sjulian */ 44199026Sjulianvoid 44299026Sjulianthread_stash(struct thread *td) 44399026Sjulian{ 444111028Sjeff mtx_lock_spin(&kse_zombie_lock); 44599026Sjulian TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 446111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 44799026Sjulian} 44899026Sjulian 449103410Smini/* 450105854Sjulian * Stash an embarasingly extra kse into the zombie kse queue. 451105854Sjulian */ 452105854Sjulianvoid 453105854Sjuliankse_stash(struct kse *ke) 454105854Sjulian{ 455111028Sjeff mtx_lock_spin(&kse_zombie_lock); 456105854Sjulian TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 457111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 458105854Sjulian} 459105854Sjulian 460105854Sjulian/* 461105854Sjulian * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 462105854Sjulian */ 463105854Sjulianvoid 464105854Sjulianksegrp_stash(struct ksegrp *kg) 465105854Sjulian{ 466111028Sjeff mtx_lock_spin(&kse_zombie_lock); 467105854Sjulian TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 468111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 469105854Sjulian} 470105854Sjulian 471105854Sjulian/* 472111028Sjeff * Reap zombie kse resource. 47399026Sjulian */ 47499026Sjulianvoid 47599026Sjulianthread_reap(void) 47699026Sjulian{ 477105854Sjulian struct thread *td_first, *td_next; 478105854Sjulian struct kse *ke_first, *ke_next; 479105854Sjulian struct ksegrp *kg_first, * kg_next; 48099026Sjulian 48199026Sjulian /* 482111028Sjeff * Don't even bother to lock if none at this instant, 483111028Sjeff * we really don't care about the next instant.. 48499026Sjulian */ 485105854Sjulian if ((!TAILQ_EMPTY(&zombie_threads)) 486105854Sjulian || (!TAILQ_EMPTY(&zombie_kses)) 487130199Sjulian || (!TAILQ_EMPTY(&zombie_ksegrps))) { 488111028Sjeff mtx_lock_spin(&kse_zombie_lock); 489105854Sjulian td_first = TAILQ_FIRST(&zombie_threads); 490105854Sjulian ke_first = TAILQ_FIRST(&zombie_kses); 491105854Sjulian kg_first = TAILQ_FIRST(&zombie_ksegrps); 492105854Sjulian if (td_first) 493105854Sjulian TAILQ_INIT(&zombie_threads); 494105854Sjulian if (ke_first) 495105854Sjulian TAILQ_INIT(&zombie_kses); 496105854Sjulian if (kg_first) 497105854Sjulian TAILQ_INIT(&zombie_ksegrps); 498111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 499105854Sjulian while (td_first) { 500105854Sjulian td_next = TAILQ_NEXT(td_first, td_runq); 501111028Sjeff if (td_first->td_ucred) 502111028Sjeff crfree(td_first->td_ucred); 503105854Sjulian thread_free(td_first); 504105854Sjulian td_first = td_next; 50599026Sjulian } 506105854Sjulian while (ke_first) { 507105854Sjulian ke_next = TAILQ_NEXT(ke_first, ke_procq); 508105854Sjulian kse_free(ke_first); 509105854Sjulian ke_first = ke_next; 510105854Sjulian } 511105854Sjulian while (kg_first) { 512105854Sjulian kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 513105854Sjulian ksegrp_free(kg_first); 514105854Sjulian kg_first = kg_next; 515105854Sjulian } 51699026Sjulian } 517130199Sjulian kse_GC(); 51899026Sjulian} 51999026Sjulian 52099026Sjulian/* 521103367Sjulian * Allocate a ksegrp. 522103367Sjulian */ 523103367Sjulianstruct ksegrp * 524103367Sjulianksegrp_alloc(void) 525103367Sjulian{ 526111119Simp return (uma_zalloc(ksegrp_zone, M_WAITOK)); 527103367Sjulian} 528103367Sjulian 529103367Sjulian/* 530103367Sjulian * Allocate a kse. 531103367Sjulian */ 532103367Sjulianstruct kse * 533103367Sjuliankse_alloc(void) 534103367Sjulian{ 535111119Simp return (uma_zalloc(kse_zone, M_WAITOK)); 536103367Sjulian} 537103367Sjulian 538103367Sjulian/* 53999026Sjulian * Allocate a thread. 54099026Sjulian */ 54199026Sjulianstruct thread * 54299026Sjulianthread_alloc(void) 54399026Sjulian{ 54499026Sjulian thread_reap(); /* check if any zombies to get */ 545111119Simp return (uma_zalloc(thread_zone, M_WAITOK)); 54699026Sjulian} 54799026Sjulian 54899026Sjulian/* 549103367Sjulian * Deallocate a ksegrp. 550103367Sjulian */ 551103367Sjulianvoid 552103367Sjulianksegrp_free(struct ksegrp *td) 553103367Sjulian{ 554103367Sjulian uma_zfree(ksegrp_zone, td); 555103367Sjulian} 556103367Sjulian 557103367Sjulian/* 558103367Sjulian * Deallocate a kse. 559103367Sjulian */ 560103367Sjulianvoid 561103367Sjuliankse_free(struct kse *td) 562103367Sjulian{ 563103367Sjulian uma_zfree(kse_zone, td); 564103367Sjulian} 565103367Sjulian 566103367Sjulian/* 56799026Sjulian * Deallocate a thread. 56899026Sjulian */ 56999026Sjulianvoid 57099026Sjulianthread_free(struct thread *td) 57199026Sjulian{ 572107719Sjulian 573107719Sjulian cpu_thread_clean(td); 57499026Sjulian uma_zfree(thread_zone, td); 57599026Sjulian} 57699026Sjulian 57799026Sjulian/* 57899026Sjulian * Discard the current thread and exit from its context. 579130355Sjulian * Always called with scheduler locked. 58099026Sjulian * 58199026Sjulian * Because we can't free a thread while we're operating under its context, 582107719Sjulian * push the current thread into our CPU's deadthread holder. This means 583107719Sjulian * we needn't worry about someone else grabbing our context before we 584130355Sjulian * do a cpu_throw(). This may not be needed now as we are under schedlock. 585130355Sjulian * Maybe we can just do a thread_stash() as thr_exit1 does. 58699026Sjulian */ 587130355Sjulian/* XXX 588130355Sjulian * libthr expects its thread exit to return for the last 589130355Sjulian * thread, meaning that the program is back to non-threaded 590130355Sjulian * mode I guess. Because we do this (cpu_throw) unconditionally 591130355Sjulian * here, they have their own version of it. (thr_exit1()) 592130355Sjulian * that doesn't do it all if this was the last thread. 593130355Sjulian * It is also called from thread_suspend_check(). 594130355Sjulian * Of course in the end, they end up coming here through exit1 595130355Sjulian * anyhow.. After fixing 'thr' to play by the rules we should be able 596130355Sjulian * to merge these two functions together. 597130355Sjulian */ 59899026Sjulianvoid 59999026Sjulianthread_exit(void) 60099026Sjulian{ 60199026Sjulian struct thread *td; 60299026Sjulian struct kse *ke; 60399026Sjulian struct proc *p; 60499026Sjulian struct ksegrp *kg; 60599026Sjulian 60699026Sjulian td = curthread; 60799026Sjulian kg = td->td_ksegrp; 60899026Sjulian p = td->td_proc; 60999026Sjulian ke = td->td_kse; 61099026Sjulian 61199026Sjulian mtx_assert(&sched_lock, MA_OWNED); 612102581Sjulian KASSERT(p != NULL, ("thread exiting without a process")); 613102581Sjulian KASSERT(ke != NULL, ("thread exiting without a kse")); 614102581Sjulian KASSERT(kg != NULL, ("thread exiting without a kse group")); 61599026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 61699026Sjulian CTR1(KTR_PROC, "thread_exit: thread %p", td); 617125158Sjhb mtx_assert(&Giant, MA_NOTOWNED); 61899026Sjulian 619104695Sjulian if (td->td_standin != NULL) { 620104695Sjulian thread_stash(td->td_standin); 621104695Sjulian td->td_standin = NULL; 622104695Sjulian } 623104695Sjulian 62499026Sjulian cpu_thread_exit(td); /* XXXSMP */ 62599026Sjulian 626102581Sjulian /* 627103002Sjulian * The last thread is left attached to the process 628103002Sjulian * So that the whole bundle gets recycled. Skip 629103002Sjulian * all this stuff. 630102581Sjulian */ 631103002Sjulian if (p->p_numthreads > 1) { 632113641Sjulian thread_unlink(td); 633111115Sdavidxu if (p->p_maxthrwaits) 634111115Sdavidxu wakeup(&p->p_numthreads); 635103002Sjulian /* 636103002Sjulian * The test below is NOT true if we are the 637103002Sjulian * sole exiting thread. P_STOPPED_SNGL is unset 638103002Sjulian * in exit1() after it is the only survivor. 639103002Sjulian */ 640103002Sjulian if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 641103002Sjulian if (p->p_numthreads == p->p_suspcount) { 642103216Sjulian thread_unsuspend_one(p->p_singlethread); 643103002Sjulian } 64499026Sjulian } 645104695Sjulian 646111028Sjeff /* 647111028Sjeff * Because each upcall structure has an owner thread, 648111028Sjeff * owner thread exits only when process is in exiting 649111028Sjeff * state, so upcall to userland is no longer needed, 650111028Sjeff * deleting upcall structure is safe here. 651111028Sjeff * So when all threads in a group is exited, all upcalls 652111028Sjeff * in the group should be automatically freed. 653111028Sjeff */ 654111028Sjeff if (td->td_upcall) 655111028Sjeff upcall_remove(td); 656124350Sschweikh 657119488Sdavidxu sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); 658132372Sjulian sched_exit_kse(FIRST_KSE_IN_PROC(p), td); 659104695Sjulian ke->ke_state = KES_UNQUEUED; 660111028Sjeff ke->ke_thread = NULL; 661124350Sschweikh /* 662108338Sjulian * Decide what to do with the KSE attached to this thread. 663104695Sjulian */ 664119488Sdavidxu if (ke->ke_flags & KEF_EXIT) { 665105854Sjulian kse_unlink(ke); 666119488Sdavidxu if (kg->kg_kses == 0) { 667132372Sjulian sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); 668119488Sdavidxu ksegrp_unlink(kg); 669119488Sdavidxu } 670119488Sdavidxu } 671111028Sjeff else 672105854Sjulian kse_reassign(ke); 673105854Sjulian PROC_UNLOCK(p); 674111028Sjeff td->td_kse = NULL; 675113244Sdavidxu#if 0 676105854Sjulian td->td_proc = NULL; 677113244Sdavidxu#endif 678105854Sjulian td->td_ksegrp = NULL; 679105854Sjulian td->td_last_kse = NULL; 680107719Sjulian PCPU_SET(deadthread, td); 681103002Sjulian } else { 682103002Sjulian PROC_UNLOCK(p); 68399026Sjulian } 684130877Sjulian td->td_state = TDS_INACTIVE; 685112888Sjeff /* XXX Shouldn't cpu_throw() here. */ 686112993Speter mtx_assert(&sched_lock, MA_OWNED); 687112993Speter cpu_throw(td, choosethread()); 688112993Speter panic("I'm a teapot!"); 68999026Sjulian /* NOTREACHED */ 69099026Sjulian} 69199026Sjulian 692124350Sschweikh/* 693107719Sjulian * Do any thread specific cleanups that may be needed in wait() 694126932Speter * called with Giant, proc and schedlock not held. 695107719Sjulian */ 696107719Sjulianvoid 697107719Sjulianthread_wait(struct proc *p) 698107719Sjulian{ 699107719Sjulian struct thread *td; 700107719Sjulian 701126932Speter mtx_assert(&Giant, MA_NOTOWNED); 702124350Sschweikh KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 703124350Sschweikh KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); 704107719Sjulian FOREACH_THREAD_IN_PROC(p, td) { 705107719Sjulian if (td->td_standin != NULL) { 706107719Sjulian thread_free(td->td_standin); 707107719Sjulian td->td_standin = NULL; 708107719Sjulian } 709107719Sjulian cpu_thread_clean(td); 710107719Sjulian } 711107719Sjulian thread_reap(); /* check for zombie threads etc. */ 712107719Sjulian} 713107719Sjulian 71499026Sjulian/* 71599026Sjulian * Link a thread to a process. 716103002Sjulian * set up anything that needs to be initialized for it to 717103002Sjulian * be used by the process. 71899026Sjulian * 71999026Sjulian * Note that we do not link to the proc's ucred here. 72099026Sjulian * The thread is linked as if running but no KSE assigned. 72199026Sjulian */ 72299026Sjulianvoid 72399026Sjulianthread_link(struct thread *td, struct ksegrp *kg) 72499026Sjulian{ 72599026Sjulian struct proc *p; 72699026Sjulian 72799026Sjulian p = kg->kg_proc; 728111028Sjeff td->td_state = TDS_INACTIVE; 729111028Sjeff td->td_proc = p; 730111028Sjeff td->td_ksegrp = kg; 731111028Sjeff td->td_last_kse = NULL; 732111028Sjeff td->td_flags = 0; 733128721Sdeischen td->td_kflags = 0; 734111028Sjeff td->td_kse = NULL; 73599026Sjulian 736103002Sjulian LIST_INIT(&td->td_contested); 737119137Ssam callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 73899026Sjulian TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 73999026Sjulian TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 74099026Sjulian p->p_numthreads++; 74199026Sjulian kg->kg_numthreads++; 74299026Sjulian} 74399026Sjulian 744113641Sjulianvoid 745113641Sjulianthread_unlink(struct thread *td) 746124350Sschweikh{ 747113641Sjulian struct proc *p = td->td_proc; 748113641Sjulian struct ksegrp *kg = td->td_ksegrp; 749113920Sjhb 750113920Sjhb mtx_assert(&sched_lock, MA_OWNED); 751113641Sjulian TAILQ_REMOVE(&p->p_threads, td, td_plist); 752113641Sjulian p->p_numthreads--; 753113641Sjulian TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 754113641Sjulian kg->kg_numthreads--; 755113641Sjulian /* could clear a few other things here */ 756124350Sschweikh} 757113641Sjulian 758111028Sjeff/* 759111028Sjeff * Purge a ksegrp resource. When a ksegrp is preparing to 760124350Sschweikh * exit, it calls this function. 761111028Sjeff */ 762130199Sjulianvoid 763111028Sjeffkse_purge_group(struct thread *td) 764111028Sjeff{ 765111028Sjeff struct ksegrp *kg; 766111028Sjeff struct kse *ke; 767111028Sjeff 768111028Sjeff kg = td->td_ksegrp; 769111028Sjeff KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 770111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 771111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 772111028Sjeff ("%s: wrong idle KSE state", __func__)); 773111028Sjeff kse_unlink(ke); 774111028Sjeff } 775111028Sjeff KASSERT((kg->kg_kses == 1), 776111028Sjeff ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 777111028Sjeff KASSERT((kg->kg_numupcalls == 0), 778111028Sjeff ("%s: ksegrp still has %d upcall datas", 779111028Sjeff __func__, kg->kg_numupcalls)); 780111028Sjeff} 781111028Sjeff 782111028Sjeff/* 783124350Sschweikh * Purge a process's KSE resource. When a process is preparing to 784124350Sschweikh * exit, it calls kse_purge to release any extra KSE resources in 785111028Sjeff * the process. 786111028Sjeff */ 787130199Sjulianvoid 788105854Sjuliankse_purge(struct proc *p, struct thread *td) 789105854Sjulian{ 790105854Sjulian struct ksegrp *kg; 791111028Sjeff struct kse *ke; 792105854Sjulian 793105854Sjulian KASSERT(p->p_numthreads == 1, ("bad thread number")); 794105854Sjulian while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 795105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 796105854Sjulian p->p_numksegrps--; 797111028Sjeff /* 798111028Sjeff * There is no ownership for KSE, after all threads 799124350Sschweikh * in the group exited, it is possible that some KSEs 800111028Sjeff * were left in idle queue, gc them now. 801111028Sjeff */ 802111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 803111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 804111028Sjeff ("%s: wrong idle KSE state", __func__)); 805111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 806111028Sjeff kg->kg_idle_kses--; 807111028Sjeff TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 808111028Sjeff kg->kg_kses--; 809111028Sjeff kse_stash(ke); 810111028Sjeff } 811105854Sjulian KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 812111028Sjeff ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 813111028Sjeff ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 814111028Sjeff KASSERT((kg->kg_numupcalls == 0), 815111028Sjeff ("%s: ksegrp still has %d upcall datas", 816111028Sjeff __func__, kg->kg_numupcalls)); 817124350Sschweikh 818111028Sjeff if (kg != td->td_ksegrp) 819105854Sjulian ksegrp_stash(kg); 820105854Sjulian } 821105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 822105854Sjulian p->p_numksegrps++; 823105854Sjulian} 824105854Sjulian 825111028Sjeff/* 82699026Sjulian * Enforce single-threading. 82799026Sjulian * 82899026Sjulian * Returns 1 if the caller must abort (another thread is waiting to 82999026Sjulian * exit the process or similar). Process is locked! 83099026Sjulian * Returns 0 when you are successfully the only thread running. 83199026Sjulian * A process has successfully single threaded in the suspend mode when 83299026Sjulian * There are no threads in user mode. Threads in the kernel must be 83399026Sjulian * allowed to continue until they get to the user boundary. They may even 83499026Sjulian * copy out their return values and data before suspending. They may however be 83599026Sjulian * accellerated in reaching the user boundary as we will wake up 83699026Sjulian * any sleeping threads that are interruptable. (PCATCH). 83799026Sjulian */ 83899026Sjulianint 83999026Sjulianthread_single(int force_exit) 84099026Sjulian{ 84199026Sjulian struct thread *td; 84299026Sjulian struct thread *td2; 84399026Sjulian struct proc *p; 844130674Sdavidxu int remaining; 84599026Sjulian 84699026Sjulian td = curthread; 84799026Sjulian p = td->td_proc; 848126932Speter mtx_assert(&Giant, MA_NOTOWNED); 84999026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 85099026Sjulian KASSERT((td != NULL), ("curthread is NULL")); 85199026Sjulian 852116361Sdavidxu if ((p->p_flag & P_SA) == 0 && p->p_numthreads == 1) 85399026Sjulian return (0); 85499026Sjulian 855100648Sjulian /* Is someone already single threading? */ 856124350Sschweikh if (p->p_singlethread) 85799026Sjulian return (1); 85899026Sjulian 859108338Sjulian if (force_exit == SINGLE_EXIT) { 86099026Sjulian p->p_flag |= P_SINGLE_EXIT; 861108338Sjulian } else 86299026Sjulian p->p_flag &= ~P_SINGLE_EXIT; 863102950Sdavidxu p->p_flag |= P_STOPPED_SINGLE; 864113920Sjhb mtx_lock_spin(&sched_lock); 86599026Sjulian p->p_singlethread = td; 866130674Sdavidxu if (force_exit == SINGLE_EXIT) 867130674Sdavidxu remaining = p->p_numthreads; 868130674Sdavidxu else 869130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 870130674Sdavidxu while (remaining != 1) { 87199026Sjulian FOREACH_THREAD_IN_PROC(p, td2) { 87299026Sjulian if (td2 == td) 87399026Sjulian continue; 874113705Sdavidxu td2->td_flags |= TDF_ASTPENDING; 875103216Sjulian if (TD_IS_INHIBITED(td2)) { 876105911Sjulian if (force_exit == SINGLE_EXIT) { 877132087Sdavidxu if (td->td_flags & TDF_DBSUSPEND) 878132087Sdavidxu td->td_flags &= ~TDF_DBSUSPEND; 879105911Sjulian if (TD_IS_SUSPENDED(td2)) { 880103216Sjulian thread_unsuspend_one(td2); 881105911Sjulian } 882105911Sjulian if (TD_ON_SLEEPQ(td2) && 883105911Sjulian (td2->td_flags & TDF_SINTR)) { 884126326Sjhb sleepq_abort(td2); 885105911Sjulian } 886105911Sjulian } else { 887105911Sjulian if (TD_IS_SUSPENDED(td2)) 888105874Sdavidxu continue; 889111028Sjeff /* 890111028Sjeff * maybe other inhibitted states too? 891111028Sjeff * XXXKSE Is it totally safe to 892111028Sjeff * suspend a non-interruptable thread? 893111028Sjeff */ 894108338Sjulian if (td2->td_inhibitors & 895111028Sjeff (TDI_SLEEPING | TDI_SWAPPED)) 896105911Sjulian thread_suspend_one(td2); 89799026Sjulian } 89899026Sjulian } 89999026Sjulian } 900130674Sdavidxu if (force_exit == SINGLE_EXIT) 901130674Sdavidxu remaining = p->p_numthreads; 902130674Sdavidxu else 903130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 904130674Sdavidxu 905124350Sschweikh /* 906124350Sschweikh * Maybe we suspended some threads.. was it enough? 907105911Sjulian */ 908130674Sdavidxu if (remaining == 1) 909105911Sjulian break; 910105911Sjulian 91199026Sjulian /* 91299026Sjulian * Wake us up when everyone else has suspended. 913100648Sjulian * In the mean time we suspend as well. 91499026Sjulian */ 915103216Sjulian thread_suspend_one(td); 91699026Sjulian PROC_UNLOCK(p); 917131473Sjhb mi_switch(SW_VOL, NULL); 91899026Sjulian mtx_unlock_spin(&sched_lock); 91999026Sjulian PROC_LOCK(p); 920113920Sjhb mtx_lock_spin(&sched_lock); 921130674Sdavidxu if (force_exit == SINGLE_EXIT) 922130674Sdavidxu remaining = p->p_numthreads; 923130674Sdavidxu else 924130674Sdavidxu remaining = p->p_numthreads - p->p_suspcount; 92599026Sjulian } 926124350Sschweikh if (force_exit == SINGLE_EXIT) { 927113920Sjhb if (td->td_upcall) 928111028Sjeff upcall_remove(td); 929105854Sjulian kse_purge(p, td); 930111028Sjeff } 931113920Sjhb mtx_unlock_spin(&sched_lock); 93299026Sjulian return (0); 93399026Sjulian} 93499026Sjulian 93599026Sjulian/* 93699026Sjulian * Called in from locations that can safely check to see 93799026Sjulian * whether we have to suspend or at least throttle for a 93899026Sjulian * single-thread event (e.g. fork). 93999026Sjulian * 94099026Sjulian * Such locations include userret(). 94199026Sjulian * If the "return_instead" argument is non zero, the thread must be able to 94299026Sjulian * accept 0 (caller may continue), or 1 (caller must abort) as a result. 94399026Sjulian * 94499026Sjulian * The 'return_instead' argument tells the function if it may do a 94599026Sjulian * thread_exit() or suspend, or whether the caller must abort and back 94699026Sjulian * out instead. 94799026Sjulian * 94899026Sjulian * If the thread that set the single_threading request has set the 94999026Sjulian * P_SINGLE_EXIT bit in the process flags then this call will never return 95099026Sjulian * if 'return_instead' is false, but will exit. 95199026Sjulian * 95299026Sjulian * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 95399026Sjulian *---------------+--------------------+--------------------- 95499026Sjulian * 0 | returns 0 | returns 0 or 1 95599026Sjulian * | when ST ends | immediatly 95699026Sjulian *---------------+--------------------+--------------------- 95799026Sjulian * 1 | thread exits | returns 1 95899026Sjulian * | | immediatly 95999026Sjulian * 0 = thread_exit() or suspension ok, 96099026Sjulian * other = return error instead of stopping the thread. 96199026Sjulian * 96299026Sjulian * While a full suspension is under effect, even a single threading 96399026Sjulian * thread would be suspended if it made this call (but it shouldn't). 96499026Sjulian * This call should only be made from places where 965124350Sschweikh * thread_exit() would be safe as that may be the outcome unless 96699026Sjulian * return_instead is set. 96799026Sjulian */ 96899026Sjulianint 96999026Sjulianthread_suspend_check(int return_instead) 97099026Sjulian{ 971104502Sjmallett struct thread *td; 972104502Sjmallett struct proc *p; 97399026Sjulian 97499026Sjulian td = curthread; 97599026Sjulian p = td->td_proc; 976126932Speter mtx_assert(&Giant, MA_NOTOWNED); 97799026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 978132087Sdavidxu while (P_SHOULDSTOP(p) || 979132087Sdavidxu ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) { 980102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 98199026Sjulian KASSERT(p->p_singlethread != NULL, 98299026Sjulian ("singlethread not set")); 98399026Sjulian /* 984100648Sjulian * The only suspension in action is a 985100648Sjulian * single-threading. Single threader need not stop. 986124350Sschweikh * XXX Should be safe to access unlocked 987100646Sjulian * as it can only be set to be true by us. 98899026Sjulian */ 989100648Sjulian if (p->p_singlethread == td) 99099026Sjulian return (0); /* Exempt from stopping. */ 991124350Sschweikh } 992100648Sjulian if (return_instead) 99399026Sjulian return (1); 99499026Sjulian 995112071Sdavidxu mtx_lock_spin(&sched_lock); 996112071Sdavidxu thread_stopped(p); 99799026Sjulian /* 99899026Sjulian * If the process is waiting for us to exit, 99999026Sjulian * this thread should just suicide. 1000102950Sdavidxu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 100199026Sjulian */ 100299026Sjulian if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1003116361Sdavidxu if (p->p_flag & P_SA) 1004112910Sjeff thread_exit(); 1005112910Sjeff else 1006112910Sjeff thr_exit1(); 100799026Sjulian } 100899026Sjulian 100999026Sjulian /* 101099026Sjulian * When a thread suspends, it just 101199026Sjulian * moves to the processes's suspend queue 101299026Sjulian * and stays there. 101399026Sjulian */ 1014103216Sjulian thread_suspend_one(td); 1015102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1016100632Sjulian if (p->p_numthreads == p->p_suspcount) { 1017103216Sjulian thread_unsuspend_one(p->p_singlethread); 1018100632Sjulian } 1019100632Sjulian } 1020113864Sjhb PROC_UNLOCK(p); 1021131473Sjhb mi_switch(SW_INVOL, NULL); 102299026Sjulian mtx_unlock_spin(&sched_lock); 102399026Sjulian PROC_LOCK(p); 102499026Sjulian } 102599026Sjulian return (0); 102699026Sjulian} 102799026Sjulian 1028102898Sdavidxuvoid 1029102898Sdavidxuthread_suspend_one(struct thread *td) 1030102898Sdavidxu{ 1031102898Sdavidxu struct proc *p = td->td_proc; 1032102898Sdavidxu 1033102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 1034113920Sjhb PROC_LOCK_ASSERT(p, MA_OWNED); 1035112071Sdavidxu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 1036102898Sdavidxu p->p_suspcount++; 1037103216Sjulian TD_SET_SUSPENDED(td); 1038102898Sdavidxu TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 1039103216Sjulian /* 1040103216Sjulian * Hack: If we are suspending but are on the sleep queue 1041103216Sjulian * then we are in msleep or the cv equivalent. We 1042103216Sjulian * want to look like we have two Inhibitors. 1043105911Sjulian * May already be set.. doesn't matter. 1044103216Sjulian */ 1045103216Sjulian if (TD_ON_SLEEPQ(td)) 1046103216Sjulian TD_SET_SLEEPING(td); 1047102898Sdavidxu} 1048102898Sdavidxu 1049102898Sdavidxuvoid 1050102898Sdavidxuthread_unsuspend_one(struct thread *td) 1051102898Sdavidxu{ 1052102898Sdavidxu struct proc *p = td->td_proc; 1053102898Sdavidxu 1054102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 1055113920Sjhb PROC_LOCK_ASSERT(p, MA_OWNED); 1056102898Sdavidxu TAILQ_REMOVE(&p->p_suspended, td, td_runq); 1057103216Sjulian TD_CLR_SUSPENDED(td); 1058102898Sdavidxu p->p_suspcount--; 1059103216Sjulian setrunnable(td); 1060102898Sdavidxu} 1061102898Sdavidxu 106299026Sjulian/* 106399026Sjulian * Allow all threads blocked by single threading to continue running. 106499026Sjulian */ 106599026Sjulianvoid 106699026Sjulianthread_unsuspend(struct proc *p) 106799026Sjulian{ 106899026Sjulian struct thread *td; 106999026Sjulian 1070100646Sjulian mtx_assert(&sched_lock, MA_OWNED); 107199026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 107299026Sjulian if (!P_SHOULDSTOP(p)) { 1073132265Sjhb while ((td = TAILQ_FIRST(&p->p_suspended))) { 1074102898Sdavidxu thread_unsuspend_one(td); 107599026Sjulian } 1076102950Sdavidxu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 107799026Sjulian (p->p_numthreads == p->p_suspcount)) { 107899026Sjulian /* 107999026Sjulian * Stopping everything also did the job for the single 108099026Sjulian * threading request. Now we've downgraded to single-threaded, 108199026Sjulian * let it continue. 108299026Sjulian */ 1083102898Sdavidxu thread_unsuspend_one(p->p_singlethread); 108499026Sjulian } 108599026Sjulian} 108699026Sjulian 108799026Sjulianvoid 108899026Sjulianthread_single_end(void) 108999026Sjulian{ 109099026Sjulian struct thread *td; 109199026Sjulian struct proc *p; 109299026Sjulian 109399026Sjulian td = curthread; 109499026Sjulian p = td->td_proc; 109599026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 1096129547Sdavidxu p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 1097113920Sjhb mtx_lock_spin(&sched_lock); 109899026Sjulian p->p_singlethread = NULL; 1099102292Sjulian /* 1100102292Sjulian * If there are other threads they mey now run, 1101102292Sjulian * unless of course there is a blanket 'stop order' 1102102292Sjulian * on the process. The single threader must be allowed 1103102292Sjulian * to continue however as this is a bad place to stop. 1104102292Sjulian */ 1105102292Sjulian if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 1106102292Sjulian while (( td = TAILQ_FIRST(&p->p_suspended))) { 1107103216Sjulian thread_unsuspend_one(td); 1108102292Sjulian } 1109102292Sjulian } 1110113920Sjhb mtx_unlock_spin(&sched_lock); 111199026Sjulian} 1112128721Sdeischen 1113