kern_thread.c revision 119137
199026Sjulian/* 299026Sjulian * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 399026Sjulian * All rights reserved. 499026Sjulian * 599026Sjulian * Redistribution and use in source and binary forms, with or without 699026Sjulian * modification, are permitted provided that the following conditions 799026Sjulian * are met: 899026Sjulian * 1. Redistributions of source code must retain the above copyright 999026Sjulian * notice(s), this list of conditions and the following disclaimer as 1099026Sjulian * the first lines of this file unmodified other than the possible 1199026Sjulian * addition of one or more copyright notices. 1299026Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1399026Sjulian * notice(s), this list of conditions and the following disclaimer in the 1499026Sjulian * documentation and/or other materials provided with the distribution. 1599026Sjulian * 1699026Sjulian * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1799026Sjulian * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1899026Sjulian * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1999026Sjulian * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2099026Sjulian * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2199026Sjulian * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2299026Sjulian * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2399026Sjulian * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2499026Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2599026Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2699026Sjulian * DAMAGE. 2799026Sjulian */ 2899026Sjulian 29116182Sobrien#include <sys/cdefs.h> 30116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_thread.c 119137 2003-08-19 17:51:11Z sam $"); 31116182Sobrien 3299026Sjulian#include <sys/param.h> 3399026Sjulian#include <sys/systm.h> 3499026Sjulian#include <sys/kernel.h> 3599026Sjulian#include <sys/lock.h> 3699026Sjulian#include <sys/malloc.h> 3799026Sjulian#include <sys/mutex.h> 3899026Sjulian#include <sys/proc.h> 39107029Sjulian#include <sys/smp.h> 4099026Sjulian#include <sys/sysctl.h> 41105854Sjulian#include <sys/sysproto.h> 4299026Sjulian#include <sys/filedesc.h> 43107126Sjeff#include <sys/sched.h> 4499026Sjulian#include <sys/signalvar.h> 4599026Sjulian#include <sys/sx.h> 46107126Sjeff#include <sys/tty.h> 4799026Sjulian#include <sys/user.h> 4899026Sjulian#include <sys/jail.h> 4999026Sjulian#include <sys/kse.h> 5099026Sjulian#include <sys/ktr.h> 51103410Smini#include <sys/ucontext.h> 5299026Sjulian 5399026Sjulian#include <vm/vm.h> 54116355Salc#include <vm/vm_extern.h> 5599026Sjulian#include <vm/vm_object.h> 5699026Sjulian#include <vm/pmap.h> 5799026Sjulian#include <vm/uma.h> 5899026Sjulian#include <vm/vm_map.h> 5999026Sjulian 60100273Speter#include <machine/frame.h> 61100273Speter 6299026Sjulian/* 63103367Sjulian * KSEGRP related storage. 6499026Sjulian */ 65103367Sjulianstatic uma_zone_t ksegrp_zone; 66103367Sjulianstatic uma_zone_t kse_zone; 6799026Sjulianstatic uma_zone_t thread_zone; 68111028Sjeffstatic uma_zone_t upcall_zone; 6999026Sjulian 70103367Sjulian/* DEBUG ONLY */ 7199026SjulianSYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 72107719Sjulianstatic int thread_debug = 0; 73107719SjulianSYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 74107719Sjulian &thread_debug, 0, "thread debug"); 7599026Sjulian 76114268Sdavidxustatic int max_threads_per_proc = 150; 77107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 78103367Sjulian &max_threads_per_proc, 0, "Limit on threads per proc"); 79103367Sjulian 80114268Sdavidxustatic int max_groups_per_proc = 50; 81107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 82107006Sdavidxu &max_groups_per_proc, 0, "Limit on thread groups per proc"); 83107006Sdavidxu 84111115Sdavidxustatic int max_threads_hits; 85111115SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 86111115Sdavidxu &max_threads_hits, 0, ""); 87111115Sdavidxu 88111028Sjeffstatic int virtual_cpu; 89111028Sjeff 9099026Sjulian#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 9199026Sjulian 92111028SjeffTAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 93105854SjulianTAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 94105854SjulianTAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 95111028SjeffTAILQ_HEAD(, kse_upcall) zombie_upcalls = 96111028Sjeff TAILQ_HEAD_INITIALIZER(zombie_upcalls); 97111028Sjeffstruct mtx kse_zombie_lock; 98111028SjeffMTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 9999026Sjulian 100107719Sjulianstatic void kse_purge(struct proc *p, struct thread *td); 101111028Sjeffstatic void kse_purge_group(struct thread *td); 102111515Sdavidxustatic int thread_update_usr_ticks(struct thread *td, int user); 103111028Sjeffstatic void thread_alloc_spare(struct thread *td, struct thread *spare); 104105854Sjulian 105111028Sjeffstatic int 106111028Sjeffsysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 107111028Sjeff{ 108111028Sjeff int error, new_val; 109111028Sjeff int def_val; 110111028Sjeff 111111028Sjeff#ifdef SMP 112111028Sjeff def_val = mp_ncpus; 113111028Sjeff#else 114111028Sjeff def_val = 1; 115111028Sjeff#endif 116111028Sjeff if (virtual_cpu == 0) 117111028Sjeff new_val = def_val; 118111028Sjeff else 119111028Sjeff new_val = virtual_cpu; 120111028Sjeff error = sysctl_handle_int(oidp, &new_val, 0, req); 121111028Sjeff if (error != 0 || req->newptr == NULL) 122111028Sjeff return (error); 123111028Sjeff if (new_val < 0) 124111028Sjeff return (EINVAL); 125111028Sjeff virtual_cpu = new_val; 126111028Sjeff return (0); 127111028Sjeff} 128111028Sjeff 129111028Sjeff/* DEBUG ONLY */ 130111028SjeffSYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 131111028Sjeff 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 132111028Sjeff "debug virtual cpus"); 133111028Sjeff 13499026Sjulian/* 135107719Sjulian * Prepare a thread for use. 13699026Sjulian */ 13799026Sjulianstatic void 13899026Sjulianthread_ctor(void *mem, int size, void *arg) 13999026Sjulian{ 14099026Sjulian struct thread *td; 14199026Sjulian 14299026Sjulian td = (struct thread *)mem; 143103216Sjulian td->td_state = TDS_INACTIVE; 144113339Sjulian td->td_oncpu = NOCPU; 145118442Sjhb td->td_critnest = 1; 14699026Sjulian} 14799026Sjulian 14899026Sjulian/* 14999026Sjulian * Reclaim a thread after use. 15099026Sjulian */ 15199026Sjulianstatic void 15299026Sjulianthread_dtor(void *mem, int size, void *arg) 15399026Sjulian{ 15499026Sjulian struct thread *td; 15599026Sjulian 15699026Sjulian td = (struct thread *)mem; 15799026Sjulian 15899026Sjulian#ifdef INVARIANTS 15999026Sjulian /* Verify that this thread is in a safe state to free. */ 16099026Sjulian switch (td->td_state) { 161103216Sjulian case TDS_INHIBITED: 162103216Sjulian case TDS_RUNNING: 163103216Sjulian case TDS_CAN_RUN: 16499026Sjulian case TDS_RUNQ: 16599026Sjulian /* 16699026Sjulian * We must never unlink a thread that is in one of 16799026Sjulian * these states, because it is currently active. 16899026Sjulian */ 16999026Sjulian panic("bad state for thread unlinking"); 17099026Sjulian /* NOTREACHED */ 171103216Sjulian case TDS_INACTIVE: 17299026Sjulian break; 17399026Sjulian default: 17499026Sjulian panic("bad thread state"); 17599026Sjulian /* NOTREACHED */ 17699026Sjulian } 17799026Sjulian#endif 17899026Sjulian} 17999026Sjulian 18099026Sjulian/* 18199026Sjulian * Initialize type-stable parts of a thread (when newly created). 18299026Sjulian */ 18399026Sjulianstatic void 18499026Sjulianthread_init(void *mem, int size) 18599026Sjulian{ 18699026Sjulian struct thread *td; 18799026Sjulian 18899026Sjulian td = (struct thread *)mem; 189103312Sjulian mtx_lock(&Giant); 190116355Salc vm_thread_new(td, 0); 191103312Sjulian mtx_unlock(&Giant); 19299026Sjulian cpu_thread_setup(td); 193107126Sjeff td->td_sched = (struct td_sched *)&td[1]; 19499026Sjulian} 19599026Sjulian 19699026Sjulian/* 19799026Sjulian * Tear down type-stable parts of a thread (just before being discarded). 19899026Sjulian */ 19999026Sjulianstatic void 20099026Sjulianthread_fini(void *mem, int size) 20199026Sjulian{ 20299026Sjulian struct thread *td; 20399026Sjulian 20499026Sjulian td = (struct thread *)mem; 205116355Salc vm_thread_dispose(td); 20699026Sjulian} 207111028Sjeff 208107126Sjeff/* 209107126Sjeff * Initialize type-stable parts of a kse (when newly created). 210107126Sjeff */ 211107126Sjeffstatic void 212107126Sjeffkse_init(void *mem, int size) 213107126Sjeff{ 214107126Sjeff struct kse *ke; 21599026Sjulian 216107126Sjeff ke = (struct kse *)mem; 217107126Sjeff ke->ke_sched = (struct ke_sched *)&ke[1]; 218107126Sjeff} 219111028Sjeff 220107126Sjeff/* 221107126Sjeff * Initialize type-stable parts of a ksegrp (when newly created). 222107126Sjeff */ 223107126Sjeffstatic void 224107126Sjeffksegrp_init(void *mem, int size) 225107126Sjeff{ 226107126Sjeff struct ksegrp *kg; 227107126Sjeff 228107126Sjeff kg = (struct ksegrp *)mem; 229107126Sjeff kg->kg_sched = (struct kg_sched *)&kg[1]; 230107126Sjeff} 231107126Sjeff 232105854Sjulian/* 233111028Sjeff * KSE is linked into kse group. 234105854Sjulian */ 235105854Sjulianvoid 236105854Sjuliankse_link(struct kse *ke, struct ksegrp *kg) 237105854Sjulian{ 238105854Sjulian struct proc *p = kg->kg_proc; 239105854Sjulian 240105854Sjulian TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 241105854Sjulian kg->kg_kses++; 242111028Sjeff ke->ke_state = KES_UNQUEUED; 243105854Sjulian ke->ke_proc = p; 244105854Sjulian ke->ke_ksegrp = kg; 245105854Sjulian ke->ke_thread = NULL; 246111028Sjeff ke->ke_oncpu = NOCPU; 247111028Sjeff ke->ke_flags = 0; 248105854Sjulian} 249105854Sjulian 250105854Sjulianvoid 251105854Sjuliankse_unlink(struct kse *ke) 252105854Sjulian{ 253105854Sjulian struct ksegrp *kg; 254105854Sjulian 255105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 256105854Sjulian kg = ke->ke_ksegrp; 257105854Sjulian TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 258111028Sjeff if (ke->ke_state == KES_IDLE) { 259111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 260111028Sjeff kg->kg_idle_kses--; 261105854Sjulian } 262111028Sjeff if (--kg->kg_kses == 0) 263111028Sjeff ksegrp_unlink(kg); 264105854Sjulian /* 265105854Sjulian * Aggregate stats from the KSE 266105854Sjulian */ 267105854Sjulian kse_stash(ke); 268105854Sjulian} 269105854Sjulian 270105854Sjulianvoid 271105854Sjulianksegrp_link(struct ksegrp *kg, struct proc *p) 272105854Sjulian{ 273105854Sjulian 274105854Sjulian TAILQ_INIT(&kg->kg_threads); 275105854Sjulian TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 276105854Sjulian TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 277105854Sjulian TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 278111028Sjeff TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 279111028Sjeff TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 280111028Sjeff kg->kg_proc = p; 281111028Sjeff /* 282111028Sjeff * the following counters are in the -zero- section 283111028Sjeff * and may not need clearing 284111028Sjeff */ 285105854Sjulian kg->kg_numthreads = 0; 286111028Sjeff kg->kg_runnable = 0; 287111028Sjeff kg->kg_kses = 0; 288111028Sjeff kg->kg_runq_kses = 0; /* XXXKSE change name */ 289111028Sjeff kg->kg_idle_kses = 0; 290111028Sjeff kg->kg_numupcalls = 0; 291111028Sjeff /* link it in now that it's consistent */ 292105854Sjulian p->p_numksegrps++; 293105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 294105854Sjulian} 295105854Sjulian 296105854Sjulianvoid 297105854Sjulianksegrp_unlink(struct ksegrp *kg) 298105854Sjulian{ 299105854Sjulian struct proc *p; 300105854Sjulian 301105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 302111028Sjeff KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 303111028Sjeff KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 304111028Sjeff KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 305111028Sjeff 306105854Sjulian p = kg->kg_proc; 307105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 308105854Sjulian p->p_numksegrps--; 309105854Sjulian /* 310105854Sjulian * Aggregate stats from the KSE 311105854Sjulian */ 312105854Sjulian ksegrp_stash(kg); 313105854Sjulian} 314105854Sjulian 315111028Sjeffstruct kse_upcall * 316111028Sjeffupcall_alloc(void) 317111028Sjeff{ 318111028Sjeff struct kse_upcall *ku; 319111028Sjeff 320111125Sdavidxu ku = uma_zalloc(upcall_zone, M_WAITOK); 321111028Sjeff bzero(ku, sizeof(*ku)); 322111028Sjeff return (ku); 323111028Sjeff} 324111028Sjeff 325111028Sjeffvoid 326111028Sjeffupcall_free(struct kse_upcall *ku) 327111028Sjeff{ 328111028Sjeff 329111028Sjeff uma_zfree(upcall_zone, ku); 330111028Sjeff} 331111028Sjeff 332111028Sjeffvoid 333111028Sjeffupcall_link(struct kse_upcall *ku, struct ksegrp *kg) 334111028Sjeff{ 335111028Sjeff 336111028Sjeff mtx_assert(&sched_lock, MA_OWNED); 337111028Sjeff TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 338111028Sjeff ku->ku_ksegrp = kg; 339111028Sjeff kg->kg_numupcalls++; 340111028Sjeff} 341111028Sjeff 342111028Sjeffvoid 343111028Sjeffupcall_unlink(struct kse_upcall *ku) 344111028Sjeff{ 345111028Sjeff struct ksegrp *kg = ku->ku_ksegrp; 346111028Sjeff 347111028Sjeff mtx_assert(&sched_lock, MA_OWNED); 348111028Sjeff KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 349111028Sjeff TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 350111028Sjeff kg->kg_numupcalls--; 351111028Sjeff upcall_stash(ku); 352111028Sjeff} 353111028Sjeff 354111028Sjeffvoid 355111028Sjeffupcall_remove(struct thread *td) 356111028Sjeff{ 357111028Sjeff 358111028Sjeff if (td->td_upcall) { 359111028Sjeff td->td_upcall->ku_owner = NULL; 360111028Sjeff upcall_unlink(td->td_upcall); 361111028Sjeff td->td_upcall = 0; 362111028Sjeff } 363111028Sjeff} 364111028Sjeff 36599026Sjulian/* 366111028Sjeff * For a newly created process, 367111028Sjeff * link up all the structures and its initial threads etc. 368105854Sjulian */ 369105854Sjulianvoid 370105854Sjulianproc_linkup(struct proc *p, struct ksegrp *kg, 371111028Sjeff struct kse *ke, struct thread *td) 372105854Sjulian{ 373105854Sjulian 374105854Sjulian TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 375105854Sjulian TAILQ_INIT(&p->p_threads); /* all threads in proc */ 376105854Sjulian TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 377105854Sjulian p->p_numksegrps = 0; 378105854Sjulian p->p_numthreads = 0; 379105854Sjulian 380105854Sjulian ksegrp_link(kg, p); 381105854Sjulian kse_link(ke, kg); 382105854Sjulian thread_link(td, kg); 383105854Sjulian} 384105854Sjulian 385111028Sjeff/* 386111028Sjeffstruct kse_thr_interrupt_args { 387111028Sjeff struct kse_thr_mailbox * tmbx; 388117704Sdavidxu int cmd; 389117704Sdavidxu long data; 390111028Sjeff}; 391111028Sjeff*/ 392105854Sjulianint 393105854Sjuliankse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 394105854Sjulian{ 395106180Sdavidxu struct proc *p; 396106180Sdavidxu struct thread *td2; 397105854Sjulian 398106242Sdavidxu p = td->td_proc; 399117704Sdavidxu if (!(p->p_flag & P_SA)) 400106242Sdavidxu return (EINVAL); 401116963Sdavidxu 402117704Sdavidxu switch (uap->cmd) { 403117704Sdavidxu case KSE_INTR_SENDSIG: 404117704Sdavidxu if (uap->data < 0 || uap->data > _SIG_MAXSIG) 405117704Sdavidxu return (EINVAL); 406117704Sdavidxu case KSE_INTR_INTERRUPT: 407117704Sdavidxu case KSE_INTR_RESTART: 408117704Sdavidxu PROC_LOCK(p); 409117704Sdavidxu mtx_lock_spin(&sched_lock); 410117704Sdavidxu FOREACH_THREAD_IN_PROC(p, td2) { 411117704Sdavidxu if (td2->td_mailbox == uap->tmbx) 412117704Sdavidxu break; 413117704Sdavidxu } 414117704Sdavidxu if (td2 == NULL) { 415117704Sdavidxu mtx_unlock_spin(&sched_lock); 416117704Sdavidxu PROC_UNLOCK(p); 417117704Sdavidxu return (ESRCH); 418117704Sdavidxu } 419117704Sdavidxu if (uap->cmd == KSE_INTR_SENDSIG) { 420117704Sdavidxu if (uap->data > 0) { 421117704Sdavidxu td2->td_flags &= ~TDF_INTERRUPT; 422117704Sdavidxu mtx_unlock_spin(&sched_lock); 423117704Sdavidxu tdsignal(td2, (int)uap->data, SIGTARGET_TD); 424117704Sdavidxu } else { 425117704Sdavidxu mtx_unlock_spin(&sched_lock); 426117704Sdavidxu } 427117704Sdavidxu } else { 428117704Sdavidxu td2->td_flags |= TDF_INTERRUPT | TDF_ASTPENDING; 429117704Sdavidxu if (TD_CAN_UNBIND(td2)) 430117704Sdavidxu td2->td_upcall->ku_flags |= KUF_DOUPCALL; 431117704Sdavidxu if (uap->cmd == KSE_INTR_INTERRUPT) 432117704Sdavidxu td2->td_intrval = EINTR; 433116963Sdavidxu else 434117704Sdavidxu td2->td_intrval = ERESTART; 435117704Sdavidxu if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { 436117704Sdavidxu if (td2->td_flags & TDF_CVWAITQ) 437117704Sdavidxu cv_abort(td2); 438117704Sdavidxu else 439117704Sdavidxu abortsleep(td2); 440117704Sdavidxu } 441117704Sdavidxu mtx_unlock_spin(&sched_lock); 442106180Sdavidxu } 443117704Sdavidxu PROC_UNLOCK(p); 444117704Sdavidxu break; 445117704Sdavidxu case KSE_INTR_SIGEXIT: 446117704Sdavidxu if (uap->data < 1 || uap->data > _SIG_MAXSIG) 447117704Sdavidxu return (EINVAL); 448117704Sdavidxu PROC_LOCK(p); 449117704Sdavidxu sigexit(td, (int)uap->data); 450117704Sdavidxu break; 451117704Sdavidxu default: 452117704Sdavidxu return (EINVAL); 453106180Sdavidxu } 454116963Sdavidxu return (0); 455105854Sjulian} 456105854Sjulian 457111028Sjeff/* 458111028Sjeffstruct kse_exit_args { 459111028Sjeff register_t dummy; 460111028Sjeff}; 461111028Sjeff*/ 462105854Sjulianint 463105854Sjuliankse_exit(struct thread *td, struct kse_exit_args *uap) 464105854Sjulian{ 465105854Sjulian struct proc *p; 466105854Sjulian struct ksegrp *kg; 467108640Sdavidxu struct kse *ke; 468115790Sjulian struct kse_upcall *ku, *ku2; 469115790Sjulian int error, count; 470105854Sjulian 471105854Sjulian p = td->td_proc; 472115790Sjulian if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 473106182Sdavidxu return (EINVAL); 474105854Sjulian kg = td->td_ksegrp; 475115790Sjulian count = 0; 476105854Sjulian PROC_LOCK(p); 477105854Sjulian mtx_lock_spin(&sched_lock); 478115790Sjulian FOREACH_UPCALL_IN_GROUP(kg, ku2) { 479115790Sjulian if (ku2->ku_flags & KUF_EXITING) 480115790Sjulian count++; 481115790Sjulian } 482115790Sjulian if ((kg->kg_numupcalls - count) == 1 && 483115790Sjulian (kg->kg_numthreads > 1)) { 484105854Sjulian mtx_unlock_spin(&sched_lock); 485105854Sjulian PROC_UNLOCK(p); 486105854Sjulian return (EDEADLK); 487105854Sjulian } 488115790Sjulian ku->ku_flags |= KUF_EXITING; 489115790Sjulian mtx_unlock_spin(&sched_lock); 490115790Sjulian PROC_UNLOCK(p); 491115790Sjulian error = suword(&ku->ku_mailbox->km_flags, ku->ku_mflags|KMF_DONE); 492115790Sjulian PROC_LOCK(p); 493115790Sjulian if (error) 494115790Sjulian psignal(p, SIGSEGV); 495115790Sjulian mtx_lock_spin(&sched_lock); 496115790Sjulian upcall_remove(td); 497108640Sdavidxu ke = td->td_kse; 498108640Sdavidxu if (p->p_numthreads == 1) { 499111028Sjeff kse_purge(p, td); 500116361Sdavidxu p->p_flag &= ~P_SA; 501105854Sjulian mtx_unlock_spin(&sched_lock); 502105854Sjulian PROC_UNLOCK(p); 503105854Sjulian } else { 504111028Sjeff if (kg->kg_numthreads == 1) { /* Shutdown a group */ 505111028Sjeff kse_purge_group(td); 506111028Sjeff ke->ke_flags |= KEF_EXIT; 507111028Sjeff } 508112071Sdavidxu thread_stopped(p); 509105854Sjulian thread_exit(); 510105854Sjulian /* NOTREACHED */ 511105854Sjulian } 512106182Sdavidxu return (0); 513105854Sjulian} 514105854Sjulian 515107719Sjulian/* 516108338Sjulian * Either becomes an upcall or waits for an awakening event and 517111028Sjeff * then becomes an upcall. Only error cases return. 518107719Sjulian */ 519111028Sjeff/* 520111028Sjeffstruct kse_release_args { 521111169Sdavidxu struct timespec *timeout; 522111028Sjeff}; 523111028Sjeff*/ 524105854Sjulianint 525111028Sjeffkse_release(struct thread *td, struct kse_release_args *uap) 526105854Sjulian{ 527105854Sjulian struct proc *p; 528107719Sjulian struct ksegrp *kg; 529116401Sdavidxu struct kse_upcall *ku; 530116401Sdavidxu struct timespec timeout; 531111169Sdavidxu struct timeval tv; 532116963Sdavidxu sigset_t sigset; 533111169Sdavidxu int error; 534105854Sjulian 535105854Sjulian p = td->td_proc; 536107719Sjulian kg = td->td_ksegrp; 537116401Sdavidxu if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 538107719Sjulian return (EINVAL); 539111169Sdavidxu if (uap->timeout != NULL) { 540111169Sdavidxu if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 541111169Sdavidxu return (error); 542111169Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &timeout); 543111169Sdavidxu } 544116401Sdavidxu if (td->td_flags & TDF_SA) 545116401Sdavidxu td->td_pflags |= TDP_UPCALLING; 546116963Sdavidxu else { 547116963Sdavidxu ku->ku_mflags = fuword(&ku->ku_mailbox->km_flags); 548116963Sdavidxu if (ku->ku_mflags == -1) { 549116963Sdavidxu PROC_LOCK(p); 550116963Sdavidxu sigexit(td, SIGSEGV); 551116963Sdavidxu } 552116963Sdavidxu } 553111169Sdavidxu PROC_LOCK(p); 554116963Sdavidxu if (ku->ku_mflags & KMF_WAITSIGEVENT) { 555116963Sdavidxu /* UTS wants to wait for signal event */ 556116963Sdavidxu if (!(p->p_flag & P_SIGEVENT) && !(ku->ku_flags & KUF_DOUPCALL)) 557116963Sdavidxu error = msleep(&p->p_siglist, &p->p_mtx, PPAUSE|PCATCH, 558116963Sdavidxu "ksesigwait", (uap->timeout ? tvtohz(&tv) : 0)); 559116963Sdavidxu p->p_flag &= ~P_SIGEVENT; 560116963Sdavidxu sigset = p->p_siglist; 561116963Sdavidxu PROC_UNLOCK(p); 562116963Sdavidxu error = copyout(&sigset, &ku->ku_mailbox->km_sigscaught, 563116963Sdavidxu sizeof(sigset)); 564116963Sdavidxu } else { 565116963Sdavidxu if (! kg->kg_completed && !(ku->ku_flags & KUF_DOUPCALL)) { 566116963Sdavidxu kg->kg_upsleeps++; 567116963Sdavidxu error = msleep(&kg->kg_completed, &p->p_mtx, 568116963Sdavidxu PPAUSE|PCATCH, "kserel", 569116963Sdavidxu (uap->timeout ? tvtohz(&tv) : 0)); 570116963Sdavidxu kg->kg_upsleeps--; 571116963Sdavidxu } 572116963Sdavidxu PROC_UNLOCK(p); 573105854Sjulian } 574116401Sdavidxu if (ku->ku_flags & KUF_DOUPCALL) { 575116401Sdavidxu mtx_lock_spin(&sched_lock); 576116401Sdavidxu ku->ku_flags &= ~KUF_DOUPCALL; 577116401Sdavidxu mtx_unlock_spin(&sched_lock); 578116401Sdavidxu } 579107719Sjulian return (0); 580105854Sjulian} 581105854Sjulian 582105854Sjulian/* struct kse_wakeup_args { 583105854Sjulian struct kse_mailbox *mbx; 584105854Sjulian}; */ 585105854Sjulianint 586105854Sjuliankse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 587105854Sjulian{ 588105854Sjulian struct proc *p; 589105854Sjulian struct ksegrp *kg; 590111028Sjeff struct kse_upcall *ku; 591108338Sjulian struct thread *td2; 592105854Sjulian 593105854Sjulian p = td->td_proc; 594108338Sjulian td2 = NULL; 595111028Sjeff ku = NULL; 596105854Sjulian /* KSE-enabled processes only, please. */ 597116361Sdavidxu if (!(p->p_flag & P_SA)) 598111028Sjeff return (EINVAL); 599111028Sjeff PROC_LOCK(p); 600108613Sjulian mtx_lock_spin(&sched_lock); 601105854Sjulian if (uap->mbx) { 602105854Sjulian FOREACH_KSEGRP_IN_PROC(p, kg) { 603111028Sjeff FOREACH_UPCALL_IN_GROUP(kg, ku) { 604111207Sdavidxu if (ku->ku_mailbox == uap->mbx) 605111028Sjeff break; 606108613Sjulian } 607111028Sjeff if (ku) 608108338Sjulian break; 609105854Sjulian } 610105854Sjulian } else { 611105854Sjulian kg = td->td_ksegrp; 612111028Sjeff if (kg->kg_upsleeps) { 613111028Sjeff wakeup_one(&kg->kg_completed); 614111028Sjeff mtx_unlock_spin(&sched_lock); 615111028Sjeff PROC_UNLOCK(p); 616111028Sjeff return (0); 617108338Sjulian } 618111028Sjeff ku = TAILQ_FIRST(&kg->kg_upcalls); 619105854Sjulian } 620111028Sjeff if (ku) { 621111028Sjeff if ((td2 = ku->ku_owner) == NULL) { 622111028Sjeff panic("%s: no owner", __func__); 623111028Sjeff } else if (TD_ON_SLEEPQ(td2) && 624116963Sdavidxu ((td2->td_wchan == &kg->kg_completed) || 625116963Sdavidxu (td2->td_wchan == &p->p_siglist && 626116963Sdavidxu (ku->ku_mflags & KMF_WAITSIGEVENT)))) { 627111028Sjeff abortsleep(td2); 628111028Sjeff } else { 629111028Sjeff ku->ku_flags |= KUF_DOUPCALL; 630108613Sjulian } 631105854Sjulian mtx_unlock_spin(&sched_lock); 632111028Sjeff PROC_UNLOCK(p); 633108338Sjulian return (0); 634108613Sjulian } 635105854Sjulian mtx_unlock_spin(&sched_lock); 636111028Sjeff PROC_UNLOCK(p); 637108338Sjulian return (ESRCH); 638105854Sjulian} 639105854Sjulian 640105854Sjulian/* 641105854Sjulian * No new KSEG: first call: use current KSE, don't schedule an upcall 642111028Sjeff * All other situations, do allocate max new KSEs and schedule an upcall. 643105854Sjulian */ 644105854Sjulian/* struct kse_create_args { 645105854Sjulian struct kse_mailbox *mbx; 646105854Sjulian int newgroup; 647105854Sjulian}; */ 648105854Sjulianint 649105854Sjuliankse_create(struct thread *td, struct kse_create_args *uap) 650105854Sjulian{ 651105854Sjulian struct kse *newke; 652105854Sjulian struct ksegrp *newkg; 653105854Sjulian struct ksegrp *kg; 654105854Sjulian struct proc *p; 655105854Sjulian struct kse_mailbox mbx; 656111028Sjeff struct kse_upcall *newku; 657116401Sdavidxu int err, ncpus, sa = 0, first = 0; 658116401Sdavidxu struct thread *newtd; 659105854Sjulian 660105854Sjulian p = td->td_proc; 661105854Sjulian if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 662105854Sjulian return (err); 663105854Sjulian 664111028Sjeff /* Too bad, why hasn't kernel always a cpu counter !? */ 665111028Sjeff#ifdef SMP 666111028Sjeff ncpus = mp_ncpus; 667111028Sjeff#else 668111028Sjeff ncpus = 1; 669111028Sjeff#endif 670116401Sdavidxu if (virtual_cpu != 0) 671111028Sjeff ncpus = virtual_cpu; 672116401Sdavidxu if (!(mbx.km_flags & KMF_BOUND)) 673116401Sdavidxu sa = TDF_SA; 674116440Sdavidxu else 675116440Sdavidxu ncpus = 1; 676112078Sdavidxu PROC_LOCK(p); 677116401Sdavidxu if (!(p->p_flag & P_SA)) { 678116401Sdavidxu first = 1; 679116401Sdavidxu p->p_flag |= P_SA; 680116401Sdavidxu } 681112078Sdavidxu PROC_UNLOCK(p); 682116401Sdavidxu if (!sa && !uap->newgroup && !first) 683116401Sdavidxu return (EINVAL); 684105854Sjulian kg = td->td_ksegrp; 685105854Sjulian if (uap->newgroup) { 686111028Sjeff /* Have race condition but it is cheap */ 687116401Sdavidxu if (p->p_numksegrps >= max_groups_per_proc) 688107006Sdavidxu return (EPROCLIM); 689105854Sjulian /* 690105854Sjulian * If we want a new KSEGRP it doesn't matter whether 691105854Sjulian * we have already fired up KSE mode before or not. 692111028Sjeff * We put the process in KSE mode and create a new KSEGRP. 693105854Sjulian */ 694105854Sjulian newkg = ksegrp_alloc(); 695105854Sjulian bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 696111028Sjeff kg_startzero, kg_endzero)); 697105854Sjulian bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 698105854Sjulian RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 699111028Sjeff mtx_lock_spin(&sched_lock); 700111028Sjeff if (p->p_numksegrps >= max_groups_per_proc) { 701111028Sjeff mtx_unlock_spin(&sched_lock); 702111677Sdavidxu ksegrp_free(newkg); 703111028Sjeff return (EPROCLIM); 704111028Sjeff } 705111677Sdavidxu ksegrp_link(newkg, p); 706111028Sjeff mtx_unlock_spin(&sched_lock); 707105854Sjulian } else { 708116452Sdavidxu if (!first && ((td->td_flags & TDF_SA) ^ sa) != 0) 709116452Sdavidxu return (EINVAL); 710111028Sjeff newkg = kg; 711111028Sjeff } 712111028Sjeff 713111028Sjeff /* 714111028Sjeff * Creating upcalls more than number of physical cpu does 715111028Sjeff * not help performance. 716111028Sjeff */ 717111028Sjeff if (newkg->kg_numupcalls >= ncpus) 718111028Sjeff return (EPROCLIM); 719111028Sjeff 720111028Sjeff if (newkg->kg_numupcalls == 0) { 721111028Sjeff /* 722116401Sdavidxu * Initialize KSE group 723116401Sdavidxu * 724116401Sdavidxu * For multiplxed group, create KSEs as many as physical 725116401Sdavidxu * cpus. This increases concurrent even if userland 726116401Sdavidxu * is not MP safe and can only run on single CPU. 727111028Sjeff * In ideal world, every physical cpu should execute a thread. 728111028Sjeff * If there is enough KSEs, threads in kernel can be 729111028Sjeff * executed parallel on different cpus with full speed, 730111028Sjeff * Concurrent in kernel shouldn't be restricted by number of 731116401Sdavidxu * upcalls userland provides. Adding more upcall structures 732116401Sdavidxu * only increases concurrent in userland. 733116401Sdavidxu * 734116401Sdavidxu * For bound thread group, because there is only thread in the 735116401Sdavidxu * group, we only create one KSE for the group. Thread in this 736116401Sdavidxu * kind of group will never schedule an upcall when blocked, 737116401Sdavidxu * this intends to simulate pthread system scope thread. 738105854Sjulian */ 739111028Sjeff while (newkg->kg_kses < ncpus) { 740105854Sjulian newke = kse_alloc(); 741111028Sjeff bzero(&newke->ke_startzero, RANGEOF(struct kse, 742111028Sjeff ke_startzero, ke_endzero)); 743105854Sjulian#if 0 744111028Sjeff mtx_lock_spin(&sched_lock); 745111028Sjeff bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 746111028Sjeff RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 747111028Sjeff mtx_unlock_spin(&sched_lock); 748105854Sjulian#endif 749111028Sjeff mtx_lock_spin(&sched_lock); 750111028Sjeff kse_link(newke, newkg); 751111028Sjeff /* Add engine */ 752111028Sjeff kse_reassign(newke); 753111028Sjeff mtx_unlock_spin(&sched_lock); 754105854Sjulian } 755111028Sjeff } 756111028Sjeff newku = upcall_alloc(); 757111028Sjeff newku->ku_mailbox = uap->mbx; 758111028Sjeff newku->ku_func = mbx.km_func; 759111028Sjeff bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 760111028Sjeff 761111028Sjeff /* For the first call this may not have been set */ 762111028Sjeff if (td->td_standin == NULL) 763111028Sjeff thread_alloc_spare(td, NULL); 764111028Sjeff 765116963Sdavidxu PROC_LOCK(p); 766111028Sjeff if (newkg->kg_numupcalls >= ncpus) { 767116963Sdavidxu PROC_UNLOCK(p); 768111028Sjeff upcall_free(newku); 769111028Sjeff return (EPROCLIM); 770111028Sjeff } 771117637Sdavidxu if (first && sa) { 772116963Sdavidxu SIGSETOR(p->p_siglist, td->td_siglist); 773116963Sdavidxu SIGEMPTYSET(td->td_siglist); 774116963Sdavidxu SIGFILLSET(td->td_sigmask); 775116963Sdavidxu SIG_CANTMASK(td->td_sigmask); 776116963Sdavidxu } 777116963Sdavidxu mtx_lock_spin(&sched_lock); 778116963Sdavidxu PROC_UNLOCK(p); 779111028Sjeff upcall_link(newku, newkg); 780112397Sdavidxu if (mbx.km_quantum) 781112397Sdavidxu newkg->kg_upquantum = max(1, mbx.km_quantum/tick); 782111028Sjeff 783111028Sjeff /* 784111028Sjeff * Each upcall structure has an owner thread, find which 785111028Sjeff * one owns it. 786111028Sjeff */ 787111028Sjeff if (uap->newgroup) { 788111028Sjeff /* 789111028Sjeff * Because new ksegrp hasn't thread, 790111028Sjeff * create an initial upcall thread to own it. 791111028Sjeff */ 792116401Sdavidxu newtd = thread_schedule_upcall(td, newku); 793105854Sjulian } else { 794105854Sjulian /* 795111028Sjeff * If current thread hasn't an upcall structure, 796111028Sjeff * just assign the upcall to it. 797105854Sjulian */ 798111028Sjeff if (td->td_upcall == NULL) { 799111028Sjeff newku->ku_owner = td; 800111028Sjeff td->td_upcall = newku; 801116401Sdavidxu newtd = td; 802111028Sjeff } else { 803111028Sjeff /* 804111028Sjeff * Create a new upcall thread to own it. 805111028Sjeff */ 806116401Sdavidxu newtd = thread_schedule_upcall(td, newku); 807111028Sjeff } 808105854Sjulian } 809116401Sdavidxu if (!sa) { 810116401Sdavidxu newtd->td_mailbox = mbx.km_curthread; 811116401Sdavidxu newtd->td_flags &= ~TDF_SA; 812116607Sdavidxu if (newtd != td) { 813116607Sdavidxu mtx_unlock_spin(&sched_lock); 814116607Sdavidxu cpu_set_upcall_kse(newtd, newku); 815116607Sdavidxu mtx_lock_spin(&sched_lock); 816116607Sdavidxu } 817116401Sdavidxu } else { 818116401Sdavidxu newtd->td_flags |= TDF_SA; 819116401Sdavidxu } 820116607Sdavidxu if (newtd != td) 821116607Sdavidxu setrunqueue(newtd); 822111028Sjeff mtx_unlock_spin(&sched_lock); 823105854Sjulian return (0); 824105854Sjulian} 825105854Sjulian 826105854Sjulian/* 82799026Sjulian * Initialize global thread allocation resources. 82899026Sjulian */ 82999026Sjulianvoid 83099026Sjulianthreadinit(void) 83199026Sjulian{ 83299026Sjulian 833107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 83499026Sjulian thread_ctor, thread_dtor, thread_init, thread_fini, 83599026Sjulian UMA_ALIGN_CACHE, 0); 836107126Sjeff ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 837107126Sjeff NULL, NULL, ksegrp_init, NULL, 838103367Sjulian UMA_ALIGN_CACHE, 0); 839107126Sjeff kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 840107126Sjeff NULL, NULL, kse_init, NULL, 841103367Sjulian UMA_ALIGN_CACHE, 0); 842111028Sjeff upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 843111028Sjeff NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 84499026Sjulian} 84599026Sjulian 84699026Sjulian/* 847103002Sjulian * Stash an embarasingly extra thread into the zombie thread queue. 84899026Sjulian */ 84999026Sjulianvoid 85099026Sjulianthread_stash(struct thread *td) 85199026Sjulian{ 852111028Sjeff mtx_lock_spin(&kse_zombie_lock); 85399026Sjulian TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 854111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 85599026Sjulian} 85699026Sjulian 857103410Smini/* 858105854Sjulian * Stash an embarasingly extra kse into the zombie kse queue. 859105854Sjulian */ 860105854Sjulianvoid 861105854Sjuliankse_stash(struct kse *ke) 862105854Sjulian{ 863111028Sjeff mtx_lock_spin(&kse_zombie_lock); 864105854Sjulian TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 865111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 866105854Sjulian} 867105854Sjulian 868105854Sjulian/* 869111028Sjeff * Stash an embarasingly extra upcall into the zombie upcall queue. 870111028Sjeff */ 871111028Sjeff 872111028Sjeffvoid 873111028Sjeffupcall_stash(struct kse_upcall *ku) 874111028Sjeff{ 875111028Sjeff mtx_lock_spin(&kse_zombie_lock); 876111028Sjeff TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 877111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 878111028Sjeff} 879111028Sjeff 880111028Sjeff/* 881105854Sjulian * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 882105854Sjulian */ 883105854Sjulianvoid 884105854Sjulianksegrp_stash(struct ksegrp *kg) 885105854Sjulian{ 886111028Sjeff mtx_lock_spin(&kse_zombie_lock); 887105854Sjulian TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 888111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 889105854Sjulian} 890105854Sjulian 891105854Sjulian/* 892111028Sjeff * Reap zombie kse resource. 89399026Sjulian */ 89499026Sjulianvoid 89599026Sjulianthread_reap(void) 89699026Sjulian{ 897105854Sjulian struct thread *td_first, *td_next; 898105854Sjulian struct kse *ke_first, *ke_next; 899105854Sjulian struct ksegrp *kg_first, * kg_next; 900111028Sjeff struct kse_upcall *ku_first, *ku_next; 90199026Sjulian 90299026Sjulian /* 903111028Sjeff * Don't even bother to lock if none at this instant, 904111028Sjeff * we really don't care about the next instant.. 90599026Sjulian */ 906105854Sjulian if ((!TAILQ_EMPTY(&zombie_threads)) 907105854Sjulian || (!TAILQ_EMPTY(&zombie_kses)) 908111028Sjeff || (!TAILQ_EMPTY(&zombie_ksegrps)) 909111028Sjeff || (!TAILQ_EMPTY(&zombie_upcalls))) { 910111028Sjeff mtx_lock_spin(&kse_zombie_lock); 911105854Sjulian td_first = TAILQ_FIRST(&zombie_threads); 912105854Sjulian ke_first = TAILQ_FIRST(&zombie_kses); 913105854Sjulian kg_first = TAILQ_FIRST(&zombie_ksegrps); 914111028Sjeff ku_first = TAILQ_FIRST(&zombie_upcalls); 915105854Sjulian if (td_first) 916105854Sjulian TAILQ_INIT(&zombie_threads); 917105854Sjulian if (ke_first) 918105854Sjulian TAILQ_INIT(&zombie_kses); 919105854Sjulian if (kg_first) 920105854Sjulian TAILQ_INIT(&zombie_ksegrps); 921111028Sjeff if (ku_first) 922111028Sjeff TAILQ_INIT(&zombie_upcalls); 923111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 924105854Sjulian while (td_first) { 925105854Sjulian td_next = TAILQ_NEXT(td_first, td_runq); 926111028Sjeff if (td_first->td_ucred) 927111028Sjeff crfree(td_first->td_ucred); 928105854Sjulian thread_free(td_first); 929105854Sjulian td_first = td_next; 93099026Sjulian } 931105854Sjulian while (ke_first) { 932105854Sjulian ke_next = TAILQ_NEXT(ke_first, ke_procq); 933105854Sjulian kse_free(ke_first); 934105854Sjulian ke_first = ke_next; 935105854Sjulian } 936105854Sjulian while (kg_first) { 937105854Sjulian kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 938105854Sjulian ksegrp_free(kg_first); 939105854Sjulian kg_first = kg_next; 940105854Sjulian } 941111028Sjeff while (ku_first) { 942111028Sjeff ku_next = TAILQ_NEXT(ku_first, ku_link); 943111028Sjeff upcall_free(ku_first); 944111028Sjeff ku_first = ku_next; 945111028Sjeff } 94699026Sjulian } 94799026Sjulian} 94899026Sjulian 94999026Sjulian/* 950103367Sjulian * Allocate a ksegrp. 951103367Sjulian */ 952103367Sjulianstruct ksegrp * 953103367Sjulianksegrp_alloc(void) 954103367Sjulian{ 955111119Simp return (uma_zalloc(ksegrp_zone, M_WAITOK)); 956103367Sjulian} 957103367Sjulian 958103367Sjulian/* 959103367Sjulian * Allocate a kse. 960103367Sjulian */ 961103367Sjulianstruct kse * 962103367Sjuliankse_alloc(void) 963103367Sjulian{ 964111119Simp return (uma_zalloc(kse_zone, M_WAITOK)); 965103367Sjulian} 966103367Sjulian 967103367Sjulian/* 96899026Sjulian * Allocate a thread. 96999026Sjulian */ 97099026Sjulianstruct thread * 97199026Sjulianthread_alloc(void) 97299026Sjulian{ 97399026Sjulian thread_reap(); /* check if any zombies to get */ 974111119Simp return (uma_zalloc(thread_zone, M_WAITOK)); 97599026Sjulian} 97699026Sjulian 97799026Sjulian/* 978103367Sjulian * Deallocate a ksegrp. 979103367Sjulian */ 980103367Sjulianvoid 981103367Sjulianksegrp_free(struct ksegrp *td) 982103367Sjulian{ 983103367Sjulian uma_zfree(ksegrp_zone, td); 984103367Sjulian} 985103367Sjulian 986103367Sjulian/* 987103367Sjulian * Deallocate a kse. 988103367Sjulian */ 989103367Sjulianvoid 990103367Sjuliankse_free(struct kse *td) 991103367Sjulian{ 992103367Sjulian uma_zfree(kse_zone, td); 993103367Sjulian} 994103367Sjulian 995103367Sjulian/* 99699026Sjulian * Deallocate a thread. 99799026Sjulian */ 99899026Sjulianvoid 99999026Sjulianthread_free(struct thread *td) 100099026Sjulian{ 1001107719Sjulian 1002107719Sjulian cpu_thread_clean(td); 100399026Sjulian uma_zfree(thread_zone, td); 100499026Sjulian} 100599026Sjulian 100699026Sjulian/* 100799026Sjulian * Store the thread context in the UTS's mailbox. 1008104031Sjulian * then add the mailbox at the head of a list we are building in user space. 1009104031Sjulian * The list is anchored in the ksegrp structure. 101099026Sjulian */ 101199026Sjulianint 1012117704Sdavidxuthread_export_context(struct thread *td, int willexit) 101399026Sjulian{ 1014104503Sjmallett struct proc *p; 1015104031Sjulian struct ksegrp *kg; 1016104031Sjulian uintptr_t mbx; 1017104031Sjulian void *addr; 1018116963Sdavidxu int error = 0, temp, sig; 1019115790Sjulian mcontext_t mc; 102099026Sjulian 1021104503Sjmallett p = td->td_proc; 1022104503Sjmallett kg = td->td_ksegrp; 1023104503Sjmallett 1024104031Sjulian /* Export the user/machine context. */ 1025115790Sjulian get_mcontext(td, &mc, 0); 1026115790Sjulian addr = (void *)(&td->td_mailbox->tm_context.uc_mcontext); 1027115790Sjulian error = copyout(&mc, addr, sizeof(mcontext_t)); 1028115790Sjulian if (error) 1029108338Sjulian goto bad; 1030104031Sjulian 1031111028Sjeff /* Exports clock ticks in kernel mode */ 1032111028Sjeff addr = (caddr_t)(&td->td_mailbox->tm_sticks); 1033117000Smarcel temp = fuword32(addr) + td->td_usticks; 1034117000Smarcel if (suword32(addr, temp)) { 1035115790Sjulian error = EFAULT; 1036111028Sjeff goto bad; 1037115790Sjulian } 1038111028Sjeff 1039116963Sdavidxu /* 1040116963Sdavidxu * Post sync signal, or process SIGKILL and SIGSTOP. 1041116963Sdavidxu * For sync signal, it is only possible when the signal is not 1042116963Sdavidxu * caught by userland or process is being debugged. 1043116963Sdavidxu */ 1044117704Sdavidxu PROC_LOCK(p); 1045116963Sdavidxu if (td->td_flags & TDF_NEEDSIGCHK) { 1046116963Sdavidxu mtx_lock_spin(&sched_lock); 1047116963Sdavidxu td->td_flags &= ~TDF_NEEDSIGCHK; 1048116963Sdavidxu mtx_unlock_spin(&sched_lock); 1049116963Sdavidxu mtx_lock(&p->p_sigacts->ps_mtx); 1050116963Sdavidxu while ((sig = cursig(td)) != 0) 1051116963Sdavidxu postsig(sig); 1052116963Sdavidxu mtx_unlock(&p->p_sigacts->ps_mtx); 1053116963Sdavidxu } 1054117704Sdavidxu if (willexit) 1055117704Sdavidxu SIGFILLSET(td->td_sigmask); 1056117704Sdavidxu PROC_UNLOCK(p); 1057116963Sdavidxu 1058111028Sjeff /* Get address in latest mbox of list pointer */ 1059104031Sjulian addr = (void *)(&td->td_mailbox->tm_next); 1060104031Sjulian /* 1061104031Sjulian * Put the saved address of the previous first 1062104031Sjulian * entry into this one 1063104031Sjulian */ 1064104031Sjulian for (;;) { 1065104031Sjulian mbx = (uintptr_t)kg->kg_completed; 1066104031Sjulian if (suword(addr, mbx)) { 1067108338Sjulian error = EFAULT; 1068107034Sdavidxu goto bad; 1069104031Sjulian } 1070104126Sjulian PROC_LOCK(p); 1071104031Sjulian if (mbx == (uintptr_t)kg->kg_completed) { 1072104031Sjulian kg->kg_completed = td->td_mailbox; 1073111028Sjeff /* 1074111028Sjeff * The thread context may be taken away by 1075111028Sjeff * other upcall threads when we unlock 1076111028Sjeff * process lock. it's no longer valid to 1077111028Sjeff * use it again in any other places. 1078111028Sjeff */ 1079111028Sjeff td->td_mailbox = NULL; 1080104126Sjulian PROC_UNLOCK(p); 1081104031Sjulian break; 1082104031Sjulian } 1083104126Sjulian PROC_UNLOCK(p); 1084104031Sjulian } 1085111028Sjeff td->td_usticks = 0; 1086104031Sjulian return (0); 1087107034Sdavidxu 1088107034Sdavidxubad: 1089107034Sdavidxu PROC_LOCK(p); 1090117704Sdavidxu sigexit(td, SIGILL); 1091108338Sjulian return (error); 1092104031Sjulian} 109399026Sjulian 1094104031Sjulian/* 1095104031Sjulian * Take the list of completed mailboxes for this KSEGRP and put them on this 1096111028Sjeff * upcall's mailbox as it's the next one going up. 1097104031Sjulian */ 1098104031Sjulianstatic int 1099111028Sjeffthread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1100104031Sjulian{ 1101104126Sjulian struct proc *p = kg->kg_proc; 1102104031Sjulian void *addr; 1103104031Sjulian uintptr_t mbx; 1104104031Sjulian 1105111028Sjeff addr = (void *)(&ku->ku_mailbox->km_completed); 1106104031Sjulian for (;;) { 1107104031Sjulian mbx = (uintptr_t)kg->kg_completed; 1108104031Sjulian if (suword(addr, mbx)) { 1109104126Sjulian PROC_LOCK(p); 1110104126Sjulian psignal(p, SIGSEGV); 1111104126Sjulian PROC_UNLOCK(p); 1112104031Sjulian return (EFAULT); 1113104031Sjulian } 1114104126Sjulian PROC_LOCK(p); 1115104031Sjulian if (mbx == (uintptr_t)kg->kg_completed) { 1116104031Sjulian kg->kg_completed = NULL; 1117104126Sjulian PROC_UNLOCK(p); 1118104031Sjulian break; 1119104031Sjulian } 1120104126Sjulian PROC_UNLOCK(p); 112199026Sjulian } 1122104031Sjulian return (0); 112399026Sjulian} 112499026Sjulian 112599026Sjulian/* 1126107034Sdavidxu * This function should be called at statclock interrupt time 1127107034Sdavidxu */ 1128107034Sdavidxuint 1129111028Sjeffthread_statclock(int user) 1130107034Sdavidxu{ 1131107034Sdavidxu struct thread *td = curthread; 1132116401Sdavidxu struct ksegrp *kg = td->td_ksegrp; 1133107034Sdavidxu 1134116401Sdavidxu if (kg->kg_numupcalls == 0 || !(td->td_flags & TDF_SA)) 1135116401Sdavidxu return (0); 1136107034Sdavidxu if (user) { 1137107034Sdavidxu /* Current always do via ast() */ 1138111976Sdavidxu mtx_lock_spin(&sched_lock); 1139111032Sjulian td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1140111976Sdavidxu mtx_unlock_spin(&sched_lock); 1141111028Sjeff td->td_uuticks++; 1142107034Sdavidxu } else { 1143107034Sdavidxu if (td->td_mailbox != NULL) 1144111028Sjeff td->td_usticks++; 1145111028Sjeff else { 1146111028Sjeff /* XXXKSE 1147111028Sjeff * We will call thread_user_enter() for every 1148111028Sjeff * kernel entry in future, so if the thread mailbox 1149111028Sjeff * is NULL, it must be a UTS kernel, don't account 1150111028Sjeff * clock ticks for it. 1151111028Sjeff */ 1152111028Sjeff } 1153107034Sdavidxu } 1154111028Sjeff return (0); 1155107034Sdavidxu} 1156107034Sdavidxu 1157111028Sjeff/* 1158111515Sdavidxu * Export state clock ticks for userland 1159111028Sjeff */ 1160107034Sdavidxustatic int 1161111515Sdavidxuthread_update_usr_ticks(struct thread *td, int user) 1162107034Sdavidxu{ 1163107034Sdavidxu struct proc *p = td->td_proc; 1164107034Sdavidxu struct kse_thr_mailbox *tmbx; 1165111028Sjeff struct kse_upcall *ku; 1166112397Sdavidxu struct ksegrp *kg; 1167107034Sdavidxu caddr_t addr; 1168118607Sjhb u_int uticks; 1169107034Sdavidxu 1170111028Sjeff if ((ku = td->td_upcall) == NULL) 1171111028Sjeff return (-1); 1172111028Sjeff 1173111028Sjeff tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1174107034Sdavidxu if ((tmbx == NULL) || (tmbx == (void *)-1)) 1175111028Sjeff return (-1); 1176111515Sdavidxu if (user) { 1177111515Sdavidxu uticks = td->td_uuticks; 1178111515Sdavidxu td->td_uuticks = 0; 1179111515Sdavidxu addr = (caddr_t)&tmbx->tm_uticks; 1180111515Sdavidxu } else { 1181111515Sdavidxu uticks = td->td_usticks; 1182111515Sdavidxu td->td_usticks = 0; 1183111515Sdavidxu addr = (caddr_t)&tmbx->tm_sticks; 1184111515Sdavidxu } 1185107034Sdavidxu if (uticks) { 1186117000Smarcel if (suword32(addr, uticks+fuword32(addr))) { 1187111028Sjeff PROC_LOCK(p); 1188111028Sjeff psignal(p, SIGSEGV); 1189111028Sjeff PROC_UNLOCK(p); 1190111028Sjeff return (-2); 1191111028Sjeff } 1192107034Sdavidxu } 1193112397Sdavidxu kg = td->td_ksegrp; 1194112397Sdavidxu if (kg->kg_upquantum && ticks >= kg->kg_nextupcall) { 1195112397Sdavidxu mtx_lock_spin(&sched_lock); 1196112397Sdavidxu td->td_upcall->ku_flags |= KUF_DOUPCALL; 1197112397Sdavidxu mtx_unlock_spin(&sched_lock); 1198112397Sdavidxu } 1199111028Sjeff return (0); 1200111028Sjeff} 1201111028Sjeff 1202111028Sjeff/* 120399026Sjulian * Discard the current thread and exit from its context. 120499026Sjulian * 120599026Sjulian * Because we can't free a thread while we're operating under its context, 1206107719Sjulian * push the current thread into our CPU's deadthread holder. This means 1207107719Sjulian * we needn't worry about someone else grabbing our context before we 1208107719Sjulian * do a cpu_throw(). 120999026Sjulian */ 121099026Sjulianvoid 121199026Sjulianthread_exit(void) 121299026Sjulian{ 121399026Sjulian struct thread *td; 121499026Sjulian struct kse *ke; 121599026Sjulian struct proc *p; 121699026Sjulian struct ksegrp *kg; 121799026Sjulian 121899026Sjulian td = curthread; 121999026Sjulian kg = td->td_ksegrp; 122099026Sjulian p = td->td_proc; 122199026Sjulian ke = td->td_kse; 122299026Sjulian 122399026Sjulian mtx_assert(&sched_lock, MA_OWNED); 1224102581Sjulian KASSERT(p != NULL, ("thread exiting without a process")); 1225102581Sjulian KASSERT(ke != NULL, ("thread exiting without a kse")); 1226102581Sjulian KASSERT(kg != NULL, ("thread exiting without a kse group")); 122799026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 122899026Sjulian CTR1(KTR_PROC, "thread_exit: thread %p", td); 122999026Sjulian KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 123099026Sjulian 1231104695Sjulian if (td->td_standin != NULL) { 1232104695Sjulian thread_stash(td->td_standin); 1233104695Sjulian td->td_standin = NULL; 1234104695Sjulian } 1235104695Sjulian 123699026Sjulian cpu_thread_exit(td); /* XXXSMP */ 123799026Sjulian 1238102581Sjulian /* 1239103002Sjulian * The last thread is left attached to the process 1240103002Sjulian * So that the whole bundle gets recycled. Skip 1241103002Sjulian * all this stuff. 1242102581Sjulian */ 1243103002Sjulian if (p->p_numthreads > 1) { 1244113641Sjulian thread_unlink(td); 1245111115Sdavidxu if (p->p_maxthrwaits) 1246111115Sdavidxu wakeup(&p->p_numthreads); 1247103002Sjulian /* 1248103002Sjulian * The test below is NOT true if we are the 1249103002Sjulian * sole exiting thread. P_STOPPED_SNGL is unset 1250103002Sjulian * in exit1() after it is the only survivor. 1251103002Sjulian */ 1252103002Sjulian if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1253103002Sjulian if (p->p_numthreads == p->p_suspcount) { 1254103216Sjulian thread_unsuspend_one(p->p_singlethread); 1255103002Sjulian } 125699026Sjulian } 1257104695Sjulian 1258111028Sjeff /* 1259111028Sjeff * Because each upcall structure has an owner thread, 1260111028Sjeff * owner thread exits only when process is in exiting 1261111028Sjeff * state, so upcall to userland is no longer needed, 1262111028Sjeff * deleting upcall structure is safe here. 1263111028Sjeff * So when all threads in a group is exited, all upcalls 1264111028Sjeff * in the group should be automatically freed. 1265111028Sjeff */ 1266111028Sjeff if (td->td_upcall) 1267111028Sjeff upcall_remove(td); 1268111028Sjeff 1269104695Sjulian ke->ke_state = KES_UNQUEUED; 1270111028Sjeff ke->ke_thread = NULL; 1271104695Sjulian /* 1272108338Sjulian * Decide what to do with the KSE attached to this thread. 1273104695Sjulian */ 1274111028Sjeff if (ke->ke_flags & KEF_EXIT) 1275105854Sjulian kse_unlink(ke); 1276111028Sjeff else 1277105854Sjulian kse_reassign(ke); 1278105854Sjulian PROC_UNLOCK(p); 1279111028Sjeff td->td_kse = NULL; 1280105854Sjulian td->td_state = TDS_INACTIVE; 1281113244Sdavidxu#if 0 1282105854Sjulian td->td_proc = NULL; 1283113244Sdavidxu#endif 1284105854Sjulian td->td_ksegrp = NULL; 1285105854Sjulian td->td_last_kse = NULL; 1286107719Sjulian PCPU_SET(deadthread, td); 1287103002Sjulian } else { 1288103002Sjulian PROC_UNLOCK(p); 128999026Sjulian } 1290112888Sjeff /* XXX Shouldn't cpu_throw() here. */ 1291112993Speter mtx_assert(&sched_lock, MA_OWNED); 1292112993Speter cpu_throw(td, choosethread()); 1293112993Speter panic("I'm a teapot!"); 129499026Sjulian /* NOTREACHED */ 129599026Sjulian} 129699026Sjulian 1297107719Sjulian/* 1298107719Sjulian * Do any thread specific cleanups that may be needed in wait() 1299107719Sjulian * called with Giant held, proc and schedlock not held. 1300107719Sjulian */ 1301107719Sjulianvoid 1302107719Sjulianthread_wait(struct proc *p) 1303107719Sjulian{ 1304107719Sjulian struct thread *td; 1305107719Sjulian 1306107719Sjulian KASSERT((p->p_numthreads == 1), ("Muliple threads in wait1()")); 1307107719Sjulian KASSERT((p->p_numksegrps == 1), ("Muliple ksegrps in wait1()")); 1308107719Sjulian FOREACH_THREAD_IN_PROC(p, td) { 1309107719Sjulian if (td->td_standin != NULL) { 1310107719Sjulian thread_free(td->td_standin); 1311107719Sjulian td->td_standin = NULL; 1312107719Sjulian } 1313107719Sjulian cpu_thread_clean(td); 1314107719Sjulian } 1315107719Sjulian thread_reap(); /* check for zombie threads etc. */ 1316107719Sjulian} 1317107719Sjulian 131899026Sjulian/* 131999026Sjulian * Link a thread to a process. 1320103002Sjulian * set up anything that needs to be initialized for it to 1321103002Sjulian * be used by the process. 132299026Sjulian * 132399026Sjulian * Note that we do not link to the proc's ucred here. 132499026Sjulian * The thread is linked as if running but no KSE assigned. 132599026Sjulian */ 132699026Sjulianvoid 132799026Sjulianthread_link(struct thread *td, struct ksegrp *kg) 132899026Sjulian{ 132999026Sjulian struct proc *p; 133099026Sjulian 133199026Sjulian p = kg->kg_proc; 1332111028Sjeff td->td_state = TDS_INACTIVE; 1333111028Sjeff td->td_proc = p; 1334111028Sjeff td->td_ksegrp = kg; 1335111028Sjeff td->td_last_kse = NULL; 1336111028Sjeff td->td_flags = 0; 1337111028Sjeff td->td_kse = NULL; 133899026Sjulian 1339103002Sjulian LIST_INIT(&td->td_contested); 1340119137Ssam callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 134199026Sjulian TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 134299026Sjulian TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 134399026Sjulian p->p_numthreads++; 134499026Sjulian kg->kg_numthreads++; 134599026Sjulian} 134699026Sjulian 1347113641Sjulianvoid 1348113641Sjulianthread_unlink(struct thread *td) 1349113641Sjulian{ 1350113641Sjulian struct proc *p = td->td_proc; 1351113641Sjulian struct ksegrp *kg = td->td_ksegrp; 1352113920Sjhb 1353113920Sjhb mtx_assert(&sched_lock, MA_OWNED); 1354113641Sjulian TAILQ_REMOVE(&p->p_threads, td, td_plist); 1355113641Sjulian p->p_numthreads--; 1356113641Sjulian TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1357113641Sjulian kg->kg_numthreads--; 1358113641Sjulian /* could clear a few other things here */ 1359113641Sjulian} 1360113641Sjulian 1361111028Sjeff/* 1362111028Sjeff * Purge a ksegrp resource. When a ksegrp is preparing to 1363111028Sjeff * exit, it calls this function. 1364111028Sjeff */ 1365113864Sjhbstatic void 1366111028Sjeffkse_purge_group(struct thread *td) 1367111028Sjeff{ 1368111028Sjeff struct ksegrp *kg; 1369111028Sjeff struct kse *ke; 1370111028Sjeff 1371111028Sjeff kg = td->td_ksegrp; 1372111028Sjeff KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1373111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1374111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 1375111028Sjeff ("%s: wrong idle KSE state", __func__)); 1376111028Sjeff kse_unlink(ke); 1377111028Sjeff } 1378111028Sjeff KASSERT((kg->kg_kses == 1), 1379111028Sjeff ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1380111028Sjeff KASSERT((kg->kg_numupcalls == 0), 1381111028Sjeff ("%s: ksegrp still has %d upcall datas", 1382111028Sjeff __func__, kg->kg_numupcalls)); 1383111028Sjeff} 1384111028Sjeff 1385111028Sjeff/* 1386111028Sjeff * Purge a process's KSE resource. When a process is preparing to 1387111028Sjeff * exit, it calls kse_purge to release any extra KSE resources in 1388111028Sjeff * the process. 1389111028Sjeff */ 1390113864Sjhbstatic void 1391105854Sjuliankse_purge(struct proc *p, struct thread *td) 1392105854Sjulian{ 1393105854Sjulian struct ksegrp *kg; 1394111028Sjeff struct kse *ke; 1395105854Sjulian 1396105854Sjulian KASSERT(p->p_numthreads == 1, ("bad thread number")); 1397105854Sjulian while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1398105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1399105854Sjulian p->p_numksegrps--; 1400111028Sjeff /* 1401111028Sjeff * There is no ownership for KSE, after all threads 1402111028Sjeff * in the group exited, it is possible that some KSEs 1403111028Sjeff * were left in idle queue, gc them now. 1404111028Sjeff */ 1405111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1406111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 1407111028Sjeff ("%s: wrong idle KSE state", __func__)); 1408111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1409111028Sjeff kg->kg_idle_kses--; 1410111028Sjeff TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1411111028Sjeff kg->kg_kses--; 1412111028Sjeff kse_stash(ke); 1413111028Sjeff } 1414105854Sjulian KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1415111028Sjeff ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1416111028Sjeff ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1417111028Sjeff KASSERT((kg->kg_numupcalls == 0), 1418111028Sjeff ("%s: ksegrp still has %d upcall datas", 1419111028Sjeff __func__, kg->kg_numupcalls)); 1420111028Sjeff 1421111028Sjeff if (kg != td->td_ksegrp) 1422105854Sjulian ksegrp_stash(kg); 1423105854Sjulian } 1424105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1425105854Sjulian p->p_numksegrps++; 1426105854Sjulian} 1427105854Sjulian 1428111028Sjeff/* 1429111028Sjeff * This function is intended to be used to initialize a spare thread 1430111028Sjeff * for upcall. Initialize thread's large data area outside sched_lock 1431111028Sjeff * for thread_schedule_upcall(). 1432111028Sjeff */ 1433111028Sjeffvoid 1434111028Sjeffthread_alloc_spare(struct thread *td, struct thread *spare) 1435111028Sjeff{ 1436111028Sjeff if (td->td_standin) 1437111028Sjeff return; 1438111028Sjeff if (spare == NULL) 1439111028Sjeff spare = thread_alloc(); 1440111028Sjeff td->td_standin = spare; 1441111028Sjeff bzero(&spare->td_startzero, 1442111028Sjeff (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1443111028Sjeff spare->td_proc = td->td_proc; 1444111028Sjeff spare->td_ucred = crhold(td->td_ucred); 1445111028Sjeff} 1446105854Sjulian 144799026Sjulian/* 1448103410Smini * Create a thread and schedule it for upcall on the KSE given. 1449108338Sjulian * Use our thread's standin so that we don't have to allocate one. 145099026Sjulian */ 145199026Sjulianstruct thread * 1452111028Sjeffthread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 145399026Sjulian{ 145499026Sjulian struct thread *td2; 145599026Sjulian 145699026Sjulian mtx_assert(&sched_lock, MA_OWNED); 1457104695Sjulian 1458104695Sjulian /* 1459111028Sjeff * Schedule an upcall thread on specified kse_upcall, 1460111028Sjeff * the kse_upcall must be free. 1461111028Sjeff * td must have a spare thread. 1462104695Sjulian */ 1463111028Sjeff KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1464104695Sjulian if ((td2 = td->td_standin) != NULL) { 1465104695Sjulian td->td_standin = NULL; 146699026Sjulian } else { 1467111028Sjeff panic("no reserve thread when scheduling an upcall"); 1468106182Sdavidxu return (NULL); 146999026Sjulian } 147099026Sjulian CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1471104695Sjulian td2, td->td_proc->p_pid, td->td_proc->p_comm); 1472103002Sjulian bcopy(&td->td_startcopy, &td2->td_startcopy, 1473103002Sjulian (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1474111028Sjeff thread_link(td2, ku->ku_ksegrp); 1475113244Sdavidxu /* inherit blocked thread's context */ 1476115858Smarcel cpu_set_upcall(td2, td); 1477111028Sjeff /* Let the new thread become owner of the upcall */ 1478111028Sjeff ku->ku_owner = td2; 1479111028Sjeff td2->td_upcall = ku; 1480116401Sdavidxu td2->td_flags = TDF_SA; 1481116372Sdavidxu td2->td_pflags = TDP_UPCALLING; 1482111028Sjeff td2->td_kse = NULL; 1483111028Sjeff td2->td_state = TDS_CAN_RUN; 1484104695Sjulian td2->td_inhibitors = 0; 1485116963Sdavidxu SIGFILLSET(td2->td_sigmask); 1486116963Sdavidxu SIG_CANTMASK(td2->td_sigmask); 1487104695Sjulian return (td2); /* bogus.. should be a void function */ 148899026Sjulian} 148999026Sjulian 1490116963Sdavidxu/* 1491116963Sdavidxu * It is only used when thread generated a trap and process is being 1492116963Sdavidxu * debugged. 1493116963Sdavidxu */ 1494111033Sjeffvoid 1495111033Sjeffthread_signal_add(struct thread *td, int sig) 1496103410Smini{ 1497111033Sjeff struct proc *p; 1498116963Sdavidxu siginfo_t siginfo; 1499116963Sdavidxu struct sigacts *ps; 1500103410Smini int error; 1501103410Smini 1502115884Sdavidxu p = td->td_proc; 1503115884Sdavidxu PROC_LOCK_ASSERT(p, MA_OWNED); 1504116963Sdavidxu ps = p->p_sigacts; 1505116963Sdavidxu mtx_assert(&ps->ps_mtx, MA_OWNED); 1506116963Sdavidxu 1507117607Sdavidxu cpu_thread_siginfo(sig, 0, &siginfo); 1508116963Sdavidxu mtx_unlock(&ps->ps_mtx); 1509103410Smini PROC_UNLOCK(p); 1510116963Sdavidxu error = copyout(&siginfo, &td->td_mailbox->tm_syncsig, sizeof(siginfo)); 1511116963Sdavidxu if (error) { 1512116963Sdavidxu PROC_LOCK(p); 1513116963Sdavidxu sigexit(td, SIGILL); 1514116963Sdavidxu } 1515103410Smini PROC_LOCK(p); 1516116963Sdavidxu SIGADDSET(td->td_sigmask, sig); 1517116963Sdavidxu mtx_lock(&ps->ps_mtx); 1518111033Sjeff} 1519111033Sjeff 1520111033Sjeffvoid 1521112397Sdavidxuthread_switchout(struct thread *td) 1522112397Sdavidxu{ 1523112397Sdavidxu struct kse_upcall *ku; 1524116607Sdavidxu struct thread *td2; 1525112397Sdavidxu 1526112397Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 1527112397Sdavidxu 1528112397Sdavidxu /* 1529112397Sdavidxu * If the outgoing thread is in threaded group and has never 1530112397Sdavidxu * scheduled an upcall, decide whether this is a short 1531112397Sdavidxu * or long term event and thus whether or not to schedule 1532112397Sdavidxu * an upcall. 1533112397Sdavidxu * If it is a short term event, just suspend it in 1534112397Sdavidxu * a way that takes its KSE with it. 1535112397Sdavidxu * Select the events for which we want to schedule upcalls. 1536112397Sdavidxu * For now it's just sleep. 1537112397Sdavidxu * XXXKSE eventually almost any inhibition could do. 1538112397Sdavidxu */ 1539112397Sdavidxu if (TD_CAN_UNBIND(td) && (td->td_standin) && TD_ON_SLEEPQ(td)) { 1540112397Sdavidxu /* 1541112397Sdavidxu * Release ownership of upcall, and schedule an upcall 1542112397Sdavidxu * thread, this new upcall thread becomes the owner of 1543112397Sdavidxu * the upcall structure. 1544112397Sdavidxu */ 1545112397Sdavidxu ku = td->td_upcall; 1546112397Sdavidxu ku->ku_owner = NULL; 1547112397Sdavidxu td->td_upcall = NULL; 1548112397Sdavidxu td->td_flags &= ~TDF_CAN_UNBIND; 1549116607Sdavidxu td2 = thread_schedule_upcall(td, ku); 1550116607Sdavidxu setrunqueue(td2); 1551112397Sdavidxu } 1552112397Sdavidxu} 1553112397Sdavidxu 1554103410Smini/* 1555111028Sjeff * Setup done on the thread when it enters the kernel. 1556105900Sjulian * XXXKSE Presently only for syscalls but eventually all kernel entries. 1557105900Sjulian */ 1558105900Sjulianvoid 1559105900Sjulianthread_user_enter(struct proc *p, struct thread *td) 1560105900Sjulian{ 1561111028Sjeff struct ksegrp *kg; 1562111028Sjeff struct kse_upcall *ku; 1563113793Sdavidxu struct kse_thr_mailbox *tmbx; 1564118486Sdavidxu uint32_t tflags; 1565105900Sjulian 1566111028Sjeff kg = td->td_ksegrp; 1567113793Sdavidxu 1568105900Sjulian /* 1569105900Sjulian * First check that we shouldn't just abort. 1570105900Sjulian * But check if we are the single thread first! 1571105900Sjulian */ 1572116401Sdavidxu if (p->p_flag & P_SINGLE_EXIT) { 1573116401Sdavidxu PROC_LOCK(p); 1574105900Sjulian mtx_lock_spin(&sched_lock); 1575112071Sdavidxu thread_stopped(p); 1576105900Sjulian thread_exit(); 1577105900Sjulian /* NOTREACHED */ 1578105900Sjulian } 1579105900Sjulian 1580105900Sjulian /* 1581105900Sjulian * If we are doing a syscall in a KSE environment, 1582105900Sjulian * note where our mailbox is. There is always the 1583108338Sjulian * possibility that we could do this lazily (in kse_reassign()), 1584105900Sjulian * but for now do it every time. 1585105900Sjulian */ 1586111028Sjeff kg = td->td_ksegrp; 1587116401Sdavidxu if (td->td_flags & TDF_SA) { 1588111028Sjeff ku = td->td_upcall; 1589111028Sjeff KASSERT(ku, ("%s: no upcall owned", __func__)); 1590111028Sjeff KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1591113793Sdavidxu KASSERT(!TD_CAN_UNBIND(td), ("%s: can unbind", __func__)); 1592117000Smarcel ku->ku_mflags = fuword32((void *)&ku->ku_mailbox->km_flags); 1593113793Sdavidxu tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1594118486Sdavidxu if ((tmbx == NULL) || (tmbx == (void *)-1L) || 1595118486Sdavidxu (ku->ku_mflags & KMF_NOUPCALL)) { 1596111028Sjeff td->td_mailbox = NULL; 1597105900Sjulian } else { 1598111115Sdavidxu if (td->td_standin == NULL) 1599111115Sdavidxu thread_alloc_spare(td, NULL); 1600118673Sdeischen tflags = fuword32(&tmbx->tm_flags); 1601118486Sdavidxu /* 1602118486Sdavidxu * On some architectures, TP register points to thread 1603118486Sdavidxu * mailbox but not points to kse mailbox, and userland 1604118486Sdavidxu * can not atomically clear km_curthread, but can 1605118486Sdavidxu * use TP register, and set TMF_NOUPCALL in thread 1606118486Sdavidxu * flag to indicate a critical region. 1607118486Sdavidxu */ 1608118486Sdavidxu if (tflags & TMF_NOUPCALL) { 1609118486Sdavidxu td->td_mailbox = NULL; 1610118486Sdavidxu } else { 1611118486Sdavidxu td->td_mailbox = tmbx; 1612118486Sdavidxu mtx_lock_spin(&sched_lock); 1613113793Sdavidxu td->td_flags |= TDF_CAN_UNBIND; 1614118486Sdavidxu mtx_unlock_spin(&sched_lock); 1615118486Sdavidxu } 1616105900Sjulian } 1617105900Sjulian } 1618105900Sjulian} 1619105900Sjulian 1620105900Sjulian/* 1621103410Smini * The extra work we go through if we are a threaded process when we 1622103410Smini * return to userland. 1623103410Smini * 162499026Sjulian * If we are a KSE process and returning to user mode, check for 162599026Sjulian * extra work to do before we return (e.g. for more syscalls 162699026Sjulian * to complete first). If we were in a critical section, we should 162799026Sjulian * just return to let it finish. Same if we were in the UTS (in 1628103410Smini * which case the mailbox's context's busy indicator will be set). 1629103410Smini * The only traps we suport will have set the mailbox. 1630103410Smini * We will clear it here. 163199026Sjulian */ 163299026Sjulianint 1633103838Sjulianthread_userret(struct thread *td, struct trapframe *frame) 163499026Sjulian{ 1635113793Sdavidxu int error = 0, upcalls, uts_crit; 1636111028Sjeff struct kse_upcall *ku; 1637111115Sdavidxu struct ksegrp *kg, *kg2; 1638104695Sjulian struct proc *p; 1639107060Sdavidxu struct timespec ts; 164099026Sjulian 1641111028Sjeff p = td->td_proc; 1642110190Sjulian kg = td->td_ksegrp; 1643116401Sdavidxu ku = td->td_upcall; 1644104695Sjulian 1645116401Sdavidxu /* Nothing to do with bound thread */ 1646116401Sdavidxu if (!(td->td_flags & TDF_SA)) 1647111028Sjeff return (0); 1648108338Sjulian 1649103410Smini /* 1650111028Sjeff * Stat clock interrupt hit in userland, it 1651111028Sjeff * is returning from interrupt, charge thread's 1652111028Sjeff * userland time for UTS. 1653103410Smini */ 1654111028Sjeff if (td->td_flags & TDF_USTATCLOCK) { 1655111515Sdavidxu thread_update_usr_ticks(td, 1); 1656111028Sjeff mtx_lock_spin(&sched_lock); 1657111028Sjeff td->td_flags &= ~TDF_USTATCLOCK; 1658111028Sjeff mtx_unlock_spin(&sched_lock); 1659116401Sdavidxu if (kg->kg_completed || 1660111515Sdavidxu (td->td_upcall->ku_flags & KUF_DOUPCALL)) 1661111515Sdavidxu thread_user_enter(p, td); 1662111028Sjeff } 1663108338Sjulian 1664113793Sdavidxu uts_crit = (td->td_mailbox == NULL); 1665111028Sjeff /* 1666111028Sjeff * Optimisation: 1667111028Sjeff * This thread has not started any upcall. 1668111028Sjeff * If there is no work to report other than ourself, 1669111028Sjeff * then it can return direct to userland. 1670111028Sjeff */ 1671108338Sjulian if (TD_CAN_UNBIND(td)) { 1672111028Sjeff mtx_lock_spin(&sched_lock); 1673111028Sjeff td->td_flags &= ~TDF_CAN_UNBIND; 1674112888Sjeff if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && 1675112077Sdavidxu (kg->kg_completed == NULL) && 1676112397Sdavidxu (ku->ku_flags & KUF_DOUPCALL) == 0 && 1677113708Sdavidxu (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { 1678112888Sjeff mtx_unlock_spin(&sched_lock); 1679111515Sdavidxu thread_update_usr_ticks(td, 0); 1680112222Sdavidxu nanotime(&ts); 1681112397Sdavidxu error = copyout(&ts, 1682112222Sdavidxu (caddr_t)&ku->ku_mailbox->km_timeofday, 1683112222Sdavidxu sizeof(ts)); 1684112077Sdavidxu td->td_mailbox = 0; 1685113793Sdavidxu ku->ku_mflags = 0; 1686112222Sdavidxu if (error) 1687112222Sdavidxu goto out; 1688112077Sdavidxu return (0); 1689108338Sjulian } 1690112888Sjeff mtx_unlock_spin(&sched_lock); 1691117704Sdavidxu thread_export_context(td, 0); 1692104695Sjulian /* 1693111028Sjeff * There is something to report, and we own an upcall 1694111028Sjeff * strucuture, we can go to userland. 1695111028Sjeff * Turn ourself into an upcall thread. 1696104695Sjulian */ 1697116372Sdavidxu td->td_pflags |= TDP_UPCALLING; 1698113793Sdavidxu } else if (td->td_mailbox && (ku == NULL)) { 1699117704Sdavidxu thread_export_context(td, 1); 1700112071Sdavidxu PROC_LOCK(p); 1701112071Sdavidxu /* 1702112071Sdavidxu * There are upcall threads waiting for 1703112071Sdavidxu * work to do, wake one of them up. 1704112071Sdavidxu * XXXKSE Maybe wake all of them up. 1705112071Sdavidxu */ 1706117704Sdavidxu if (kg->kg_upsleeps) 1707112071Sdavidxu wakeup_one(&kg->kg_completed); 1708112071Sdavidxu mtx_lock_spin(&sched_lock); 1709112071Sdavidxu thread_stopped(p); 1710108338Sjulian thread_exit(); 1711111028Sjeff /* NOTREACHED */ 1712104695Sjulian } 1713104695Sjulian 1714116401Sdavidxu KASSERT(ku != NULL, ("upcall is NULL\n")); 1715111154Sdavidxu KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1716111154Sdavidxu 1717111154Sdavidxu if (p->p_numthreads > max_threads_per_proc) { 1718111154Sdavidxu max_threads_hits++; 1719111154Sdavidxu PROC_LOCK(p); 1720113920Sjhb mtx_lock_spin(&sched_lock); 1721116184Sdavidxu p->p_maxthrwaits++; 1722111154Sdavidxu while (p->p_numthreads > max_threads_per_proc) { 1723111154Sdavidxu upcalls = 0; 1724111154Sdavidxu FOREACH_KSEGRP_IN_PROC(p, kg2) { 1725111154Sdavidxu if (kg2->kg_numupcalls == 0) 1726111154Sdavidxu upcalls++; 1727111154Sdavidxu else 1728111154Sdavidxu upcalls += kg2->kg_numupcalls; 1729111154Sdavidxu } 1730111154Sdavidxu if (upcalls >= max_threads_per_proc) 1731111154Sdavidxu break; 1732114106Sdavidxu mtx_unlock_spin(&sched_lock); 1733116138Sdavidxu if (msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1734116184Sdavidxu "maxthreads", NULL)) { 1735116184Sdavidxu mtx_lock_spin(&sched_lock); 1736116184Sdavidxu break; 1737116184Sdavidxu } else { 1738116184Sdavidxu mtx_lock_spin(&sched_lock); 1739116184Sdavidxu } 1740111154Sdavidxu } 1741116184Sdavidxu p->p_maxthrwaits--; 1742113920Sjhb mtx_unlock_spin(&sched_lock); 1743111154Sdavidxu PROC_UNLOCK(p); 1744111154Sdavidxu } 1745111154Sdavidxu 1746116372Sdavidxu if (td->td_pflags & TDP_UPCALLING) { 1747113793Sdavidxu uts_crit = 0; 1748112397Sdavidxu kg->kg_nextupcall = ticks+kg->kg_upquantum; 1749108338Sjulian /* 1750108338Sjulian * There is no more work to do and we are going to ride 1751111028Sjeff * this thread up to userland as an upcall. 1752108338Sjulian * Do the last parts of the setup needed for the upcall. 1753108338Sjulian */ 1754108338Sjulian CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1755108338Sjulian td, td->td_proc->p_pid, td->td_proc->p_comm); 1756104695Sjulian 1757116372Sdavidxu td->td_pflags &= ~TDP_UPCALLING; 1758116401Sdavidxu if (ku->ku_flags & KUF_DOUPCALL) { 1759116401Sdavidxu mtx_lock_spin(&sched_lock); 1760111028Sjeff ku->ku_flags &= ~KUF_DOUPCALL; 1761116401Sdavidxu mtx_unlock_spin(&sched_lock); 1762116401Sdavidxu } 1763111028Sjeff /* 1764113793Sdavidxu * Set user context to the UTS 1765113793Sdavidxu */ 1766113793Sdavidxu if (!(ku->ku_mflags & KMF_NOUPCALL)) { 1767113793Sdavidxu cpu_set_upcall_kse(td, ku); 1768113793Sdavidxu error = suword(&ku->ku_mailbox->km_curthread, 0); 1769113793Sdavidxu if (error) 1770113793Sdavidxu goto out; 1771113793Sdavidxu } 1772113793Sdavidxu 1773113793Sdavidxu /* 1774108338Sjulian * Unhook the list of completed threads. 1775108338Sjulian * anything that completes after this gets to 1776108338Sjulian * come in next time. 1777108338Sjulian * Put the list of completed thread mailboxes on 1778108338Sjulian * this KSE's mailbox. 1779108338Sjulian */ 1780113793Sdavidxu if (!(ku->ku_mflags & KMF_NOCOMPLETED) && 1781113793Sdavidxu (error = thread_link_mboxes(kg, ku)) != 0) 1782111115Sdavidxu goto out; 1783113793Sdavidxu } 1784113793Sdavidxu if (!uts_crit) { 1785107060Sdavidxu nanotime(&ts); 1786113793Sdavidxu error = copyout(&ts, &ku->ku_mailbox->km_timeofday, sizeof(ts)); 1787111115Sdavidxu } 1788111115Sdavidxu 1789111115Sdavidxuout: 1790111115Sdavidxu if (error) { 1791111115Sdavidxu /* 1792111129Sdavidxu * Things are going to be so screwed we should just kill 1793111129Sdavidxu * the process. 1794111115Sdavidxu * how do we do that? 1795111115Sdavidxu */ 1796111115Sdavidxu PROC_LOCK(td->td_proc); 1797111115Sdavidxu psignal(td->td_proc, SIGSEGV); 1798111115Sdavidxu PROC_UNLOCK(td->td_proc); 1799111115Sdavidxu } else { 1800111115Sdavidxu /* 1801111115Sdavidxu * Optimisation: 1802111115Sdavidxu * Ensure that we have a spare thread available, 1803111115Sdavidxu * for when we re-enter the kernel. 1804111115Sdavidxu */ 1805111115Sdavidxu if (td->td_standin == NULL) 1806111115Sdavidxu thread_alloc_spare(td, NULL); 1807111115Sdavidxu } 1808111115Sdavidxu 1809113793Sdavidxu ku->ku_mflags = 0; 1810111028Sjeff /* 1811111028Sjeff * Clear thread mailbox first, then clear system tick count. 1812111028Sjeff * The order is important because thread_statclock() use 1813111028Sjeff * mailbox pointer to see if it is an userland thread or 1814111028Sjeff * an UTS kernel thread. 1815111028Sjeff */ 1816108338Sjulian td->td_mailbox = NULL; 1817111028Sjeff td->td_usticks = 0; 1818104695Sjulian return (error); /* go sync */ 181999026Sjulian} 182099026Sjulian 182199026Sjulian/* 182299026Sjulian * Enforce single-threading. 182399026Sjulian * 182499026Sjulian * Returns 1 if the caller must abort (another thread is waiting to 182599026Sjulian * exit the process or similar). Process is locked! 182699026Sjulian * Returns 0 when you are successfully the only thread running. 182799026Sjulian * A process has successfully single threaded in the suspend mode when 182899026Sjulian * There are no threads in user mode. Threads in the kernel must be 182999026Sjulian * allowed to continue until they get to the user boundary. They may even 183099026Sjulian * copy out their return values and data before suspending. They may however be 183199026Sjulian * accellerated in reaching the user boundary as we will wake up 183299026Sjulian * any sleeping threads that are interruptable. (PCATCH). 183399026Sjulian */ 183499026Sjulianint 183599026Sjulianthread_single(int force_exit) 183699026Sjulian{ 183799026Sjulian struct thread *td; 183899026Sjulian struct thread *td2; 183999026Sjulian struct proc *p; 184099026Sjulian 184199026Sjulian td = curthread; 184299026Sjulian p = td->td_proc; 1843107719Sjulian mtx_assert(&Giant, MA_OWNED); 184499026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 184599026Sjulian KASSERT((td != NULL), ("curthread is NULL")); 184699026Sjulian 1847116361Sdavidxu if ((p->p_flag & P_SA) == 0 && p->p_numthreads == 1) 184899026Sjulian return (0); 184999026Sjulian 1850100648Sjulian /* Is someone already single threading? */ 1851100648Sjulian if (p->p_singlethread) 185299026Sjulian return (1); 185399026Sjulian 1854108338Sjulian if (force_exit == SINGLE_EXIT) { 185599026Sjulian p->p_flag |= P_SINGLE_EXIT; 1856108338Sjulian } else 185799026Sjulian p->p_flag &= ~P_SINGLE_EXIT; 1858102950Sdavidxu p->p_flag |= P_STOPPED_SINGLE; 1859113920Sjhb mtx_lock_spin(&sched_lock); 186099026Sjulian p->p_singlethread = td; 186199026Sjulian while ((p->p_numthreads - p->p_suspcount) != 1) { 186299026Sjulian FOREACH_THREAD_IN_PROC(p, td2) { 186399026Sjulian if (td2 == td) 186499026Sjulian continue; 1865113705Sdavidxu td2->td_flags |= TDF_ASTPENDING; 1866103216Sjulian if (TD_IS_INHIBITED(td2)) { 1867105911Sjulian if (force_exit == SINGLE_EXIT) { 1868105911Sjulian if (TD_IS_SUSPENDED(td2)) { 1869103216Sjulian thread_unsuspend_one(td2); 1870105911Sjulian } 1871105911Sjulian if (TD_ON_SLEEPQ(td2) && 1872105911Sjulian (td2->td_flags & TDF_SINTR)) { 1873105911Sjulian if (td2->td_flags & TDF_CVWAITQ) 1874105911Sjulian cv_abort(td2); 1875105911Sjulian else 1876105911Sjulian abortsleep(td2); 1877105911Sjulian } 1878105911Sjulian } else { 1879105911Sjulian if (TD_IS_SUSPENDED(td2)) 1880105874Sdavidxu continue; 1881111028Sjeff /* 1882111028Sjeff * maybe other inhibitted states too? 1883111028Sjeff * XXXKSE Is it totally safe to 1884111028Sjeff * suspend a non-interruptable thread? 1885111028Sjeff */ 1886108338Sjulian if (td2->td_inhibitors & 1887111028Sjeff (TDI_SLEEPING | TDI_SWAPPED)) 1888105911Sjulian thread_suspend_one(td2); 188999026Sjulian } 189099026Sjulian } 189199026Sjulian } 1892105911Sjulian /* 1893105911Sjulian * Maybe we suspended some threads.. was it enough? 1894105911Sjulian */ 1895113920Sjhb if ((p->p_numthreads - p->p_suspcount) == 1) 1896105911Sjulian break; 1897105911Sjulian 189899026Sjulian /* 189999026Sjulian * Wake us up when everyone else has suspended. 1900100648Sjulian * In the mean time we suspend as well. 190199026Sjulian */ 1902103216Sjulian thread_suspend_one(td); 1903113795Sdavidxu DROP_GIANT(); 190499026Sjulian PROC_UNLOCK(p); 1905107719Sjulian p->p_stats->p_ru.ru_nvcsw++; 190699026Sjulian mi_switch(); 190799026Sjulian mtx_unlock_spin(&sched_lock); 1908113795Sdavidxu PICKUP_GIANT(); 190999026Sjulian PROC_LOCK(p); 1910113920Sjhb mtx_lock_spin(&sched_lock); 191199026Sjulian } 1912111028Sjeff if (force_exit == SINGLE_EXIT) { 1913113920Sjhb if (td->td_upcall) 1914111028Sjeff upcall_remove(td); 1915105854Sjulian kse_purge(p, td); 1916111028Sjeff } 1917113920Sjhb mtx_unlock_spin(&sched_lock); 191899026Sjulian return (0); 191999026Sjulian} 192099026Sjulian 192199026Sjulian/* 192299026Sjulian * Called in from locations that can safely check to see 192399026Sjulian * whether we have to suspend or at least throttle for a 192499026Sjulian * single-thread event (e.g. fork). 192599026Sjulian * 192699026Sjulian * Such locations include userret(). 192799026Sjulian * If the "return_instead" argument is non zero, the thread must be able to 192899026Sjulian * accept 0 (caller may continue), or 1 (caller must abort) as a result. 192999026Sjulian * 193099026Sjulian * The 'return_instead' argument tells the function if it may do a 193199026Sjulian * thread_exit() or suspend, or whether the caller must abort and back 193299026Sjulian * out instead. 193399026Sjulian * 193499026Sjulian * If the thread that set the single_threading request has set the 193599026Sjulian * P_SINGLE_EXIT bit in the process flags then this call will never return 193699026Sjulian * if 'return_instead' is false, but will exit. 193799026Sjulian * 193899026Sjulian * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 193999026Sjulian *---------------+--------------------+--------------------- 194099026Sjulian * 0 | returns 0 | returns 0 or 1 194199026Sjulian * | when ST ends | immediatly 194299026Sjulian *---------------+--------------------+--------------------- 194399026Sjulian * 1 | thread exits | returns 1 194499026Sjulian * | | immediatly 194599026Sjulian * 0 = thread_exit() or suspension ok, 194699026Sjulian * other = return error instead of stopping the thread. 194799026Sjulian * 194899026Sjulian * While a full suspension is under effect, even a single threading 194999026Sjulian * thread would be suspended if it made this call (but it shouldn't). 195099026Sjulian * This call should only be made from places where 195199026Sjulian * thread_exit() would be safe as that may be the outcome unless 195299026Sjulian * return_instead is set. 195399026Sjulian */ 195499026Sjulianint 195599026Sjulianthread_suspend_check(int return_instead) 195699026Sjulian{ 1957104502Sjmallett struct thread *td; 1958104502Sjmallett struct proc *p; 195999026Sjulian 196099026Sjulian td = curthread; 196199026Sjulian p = td->td_proc; 196299026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 196399026Sjulian while (P_SHOULDSTOP(p)) { 1964102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 196599026Sjulian KASSERT(p->p_singlethread != NULL, 196699026Sjulian ("singlethread not set")); 196799026Sjulian /* 1968100648Sjulian * The only suspension in action is a 1969100648Sjulian * single-threading. Single threader need not stop. 1970100646Sjulian * XXX Should be safe to access unlocked 1971100646Sjulian * as it can only be set to be true by us. 197299026Sjulian */ 1973100648Sjulian if (p->p_singlethread == td) 197499026Sjulian return (0); /* Exempt from stopping. */ 197599026Sjulian } 1976100648Sjulian if (return_instead) 197799026Sjulian return (1); 197899026Sjulian 1979112071Sdavidxu mtx_lock_spin(&sched_lock); 1980112071Sdavidxu thread_stopped(p); 198199026Sjulian /* 198299026Sjulian * If the process is waiting for us to exit, 198399026Sjulian * this thread should just suicide. 1984102950Sdavidxu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 198599026Sjulian */ 198699026Sjulian if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 198799026Sjulian while (mtx_owned(&Giant)) 198899026Sjulian mtx_unlock(&Giant); 1989116361Sdavidxu if (p->p_flag & P_SA) 1990112910Sjeff thread_exit(); 1991112910Sjeff else 1992112910Sjeff thr_exit1(); 199399026Sjulian } 199499026Sjulian 199599026Sjulian /* 199699026Sjulian * When a thread suspends, it just 199799026Sjulian * moves to the processes's suspend queue 199899026Sjulian * and stays there. 199999026Sjulian */ 2000103216Sjulian thread_suspend_one(td); 2001102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 2002100632Sjulian if (p->p_numthreads == p->p_suspcount) { 2003103216Sjulian thread_unsuspend_one(p->p_singlethread); 2004100632Sjulian } 2005100632Sjulian } 2006114398Sdavidxu DROP_GIANT(); 2007113864Sjhb PROC_UNLOCK(p); 2008100594Sjulian p->p_stats->p_ru.ru_nivcsw++; 200999026Sjulian mi_switch(); 201099026Sjulian mtx_unlock_spin(&sched_lock); 2011114398Sdavidxu PICKUP_GIANT(); 201299026Sjulian PROC_LOCK(p); 201399026Sjulian } 201499026Sjulian return (0); 201599026Sjulian} 201699026Sjulian 2017102898Sdavidxuvoid 2018102898Sdavidxuthread_suspend_one(struct thread *td) 2019102898Sdavidxu{ 2020102898Sdavidxu struct proc *p = td->td_proc; 2021102898Sdavidxu 2022102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 2023113920Sjhb PROC_LOCK_ASSERT(p, MA_OWNED); 2024112071Sdavidxu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 2025102898Sdavidxu p->p_suspcount++; 2026103216Sjulian TD_SET_SUSPENDED(td); 2027102898Sdavidxu TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 2028103216Sjulian /* 2029103216Sjulian * Hack: If we are suspending but are on the sleep queue 2030103216Sjulian * then we are in msleep or the cv equivalent. We 2031103216Sjulian * want to look like we have two Inhibitors. 2032105911Sjulian * May already be set.. doesn't matter. 2033103216Sjulian */ 2034103216Sjulian if (TD_ON_SLEEPQ(td)) 2035103216Sjulian TD_SET_SLEEPING(td); 2036102898Sdavidxu} 2037102898Sdavidxu 2038102898Sdavidxuvoid 2039102898Sdavidxuthread_unsuspend_one(struct thread *td) 2040102898Sdavidxu{ 2041102898Sdavidxu struct proc *p = td->td_proc; 2042102898Sdavidxu 2043102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 2044113920Sjhb PROC_LOCK_ASSERT(p, MA_OWNED); 2045102898Sdavidxu TAILQ_REMOVE(&p->p_suspended, td, td_runq); 2046103216Sjulian TD_CLR_SUSPENDED(td); 2047102898Sdavidxu p->p_suspcount--; 2048103216Sjulian setrunnable(td); 2049102898Sdavidxu} 2050102898Sdavidxu 205199026Sjulian/* 205299026Sjulian * Allow all threads blocked by single threading to continue running. 205399026Sjulian */ 205499026Sjulianvoid 205599026Sjulianthread_unsuspend(struct proc *p) 205699026Sjulian{ 205799026Sjulian struct thread *td; 205899026Sjulian 2059100646Sjulian mtx_assert(&sched_lock, MA_OWNED); 206099026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 206199026Sjulian if (!P_SHOULDSTOP(p)) { 206299026Sjulian while (( td = TAILQ_FIRST(&p->p_suspended))) { 2063102898Sdavidxu thread_unsuspend_one(td); 206499026Sjulian } 2065102950Sdavidxu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 206699026Sjulian (p->p_numthreads == p->p_suspcount)) { 206799026Sjulian /* 206899026Sjulian * Stopping everything also did the job for the single 206999026Sjulian * threading request. Now we've downgraded to single-threaded, 207099026Sjulian * let it continue. 207199026Sjulian */ 2072102898Sdavidxu thread_unsuspend_one(p->p_singlethread); 207399026Sjulian } 207499026Sjulian} 207599026Sjulian 207699026Sjulianvoid 207799026Sjulianthread_single_end(void) 207899026Sjulian{ 207999026Sjulian struct thread *td; 208099026Sjulian struct proc *p; 208199026Sjulian 208299026Sjulian td = curthread; 208399026Sjulian p = td->td_proc; 208499026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 2085102950Sdavidxu p->p_flag &= ~P_STOPPED_SINGLE; 2086113920Sjhb mtx_lock_spin(&sched_lock); 208799026Sjulian p->p_singlethread = NULL; 2088102292Sjulian /* 2089102292Sjulian * If there are other threads they mey now run, 2090102292Sjulian * unless of course there is a blanket 'stop order' 2091102292Sjulian * on the process. The single threader must be allowed 2092102292Sjulian * to continue however as this is a bad place to stop. 2093102292Sjulian */ 2094102292Sjulian if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2095102292Sjulian while (( td = TAILQ_FIRST(&p->p_suspended))) { 2096103216Sjulian thread_unsuspend_one(td); 2097102292Sjulian } 2098102292Sjulian } 2099113920Sjhb mtx_unlock_spin(&sched_lock); 210099026Sjulian} 210199026Sjulian 2102102292Sjulian 2103