kern_thread.c revision 111119
199026Sjulian/* 299026Sjulian * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 399026Sjulian * All rights reserved. 499026Sjulian * 599026Sjulian * Redistribution and use in source and binary forms, with or without 699026Sjulian * modification, are permitted provided that the following conditions 799026Sjulian * are met: 899026Sjulian * 1. Redistributions of source code must retain the above copyright 999026Sjulian * notice(s), this list of conditions and the following disclaimer as 1099026Sjulian * the first lines of this file unmodified other than the possible 1199026Sjulian * addition of one or more copyright notices. 1299026Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1399026Sjulian * notice(s), this list of conditions and the following disclaimer in the 1499026Sjulian * documentation and/or other materials provided with the distribution. 1599026Sjulian * 1699026Sjulian * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1799026Sjulian * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1899026Sjulian * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1999026Sjulian * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2099026Sjulian * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2199026Sjulian * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2299026Sjulian * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2399026Sjulian * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2499026Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2599026Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2699026Sjulian * DAMAGE. 2799026Sjulian * 2899026Sjulian * $FreeBSD: head/sys/kern/kern_thread.c 111119 2003-02-19 05:47:46Z imp $ 2999026Sjulian */ 3099026Sjulian 3199026Sjulian#include <sys/param.h> 3299026Sjulian#include <sys/systm.h> 3399026Sjulian#include <sys/kernel.h> 3499026Sjulian#include <sys/lock.h> 3599026Sjulian#include <sys/malloc.h> 3699026Sjulian#include <sys/mutex.h> 3799026Sjulian#include <sys/proc.h> 38107029Sjulian#include <sys/smp.h> 3999026Sjulian#include <sys/sysctl.h> 40105854Sjulian#include <sys/sysproto.h> 4199026Sjulian#include <sys/filedesc.h> 42107126Sjeff#include <sys/sched.h> 4399026Sjulian#include <sys/signalvar.h> 4499026Sjulian#include <sys/sx.h> 45107126Sjeff#include <sys/tty.h> 4699026Sjulian#include <sys/user.h> 4799026Sjulian#include <sys/jail.h> 4899026Sjulian#include <sys/kse.h> 4999026Sjulian#include <sys/ktr.h> 50103410Smini#include <sys/ucontext.h> 5199026Sjulian 5299026Sjulian#include <vm/vm.h> 5399026Sjulian#include <vm/vm_object.h> 5499026Sjulian#include <vm/pmap.h> 5599026Sjulian#include <vm/uma.h> 5699026Sjulian#include <vm/vm_map.h> 5799026Sjulian 58100273Speter#include <machine/frame.h> 59100273Speter 6099026Sjulian/* 61103367Sjulian * KSEGRP related storage. 6299026Sjulian */ 63103367Sjulianstatic uma_zone_t ksegrp_zone; 64103367Sjulianstatic uma_zone_t kse_zone; 6599026Sjulianstatic uma_zone_t thread_zone; 66111028Sjeffstatic uma_zone_t upcall_zone; 6799026Sjulian 68103367Sjulian/* DEBUG ONLY */ 6999026SjulianSYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 70107719Sjulianstatic int thread_debug = 0; 71107719SjulianSYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 72107719Sjulian &thread_debug, 0, "thread debug"); 7399026Sjulian 74107006Sdavidxustatic int max_threads_per_proc = 30; 75107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 76103367Sjulian &max_threads_per_proc, 0, "Limit on threads per proc"); 77103367Sjulian 78107006Sdavidxustatic int max_groups_per_proc = 5; 79107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 80107006Sdavidxu &max_groups_per_proc, 0, "Limit on thread groups per proc"); 81107006Sdavidxu 82111115Sdavidxustatic int max_threads_hits; 83111115SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 84111115Sdavidxu &max_threads_hits, 0, ""); 85111115Sdavidxu 86111028Sjeffstatic int virtual_cpu; 87111028Sjeff 8899026Sjulian#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 8999026Sjulian 90111028SjeffTAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 91105854SjulianTAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 92105854SjulianTAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 93111028SjeffTAILQ_HEAD(, kse_upcall) zombie_upcalls = 94111028Sjeff TAILQ_HEAD_INITIALIZER(zombie_upcalls); 95111028Sjeffstruct mtx kse_zombie_lock; 96111028SjeffMTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 9799026Sjulian 98107719Sjulianstatic void kse_purge(struct proc *p, struct thread *td); 99111028Sjeffstatic void kse_purge_group(struct thread *td); 100111028Sjeffstatic int thread_update_usr_ticks(struct thread *td); 101111028Sjeffstatic int thread_update_sys_ticks(struct thread *td); 102111028Sjeffstatic void thread_alloc_spare(struct thread *td, struct thread *spare); 103105854Sjulian 104111028Sjeffstatic int 105111028Sjeffsysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 106111028Sjeff{ 107111028Sjeff int error, new_val; 108111028Sjeff int def_val; 109111028Sjeff 110111028Sjeff#ifdef SMP 111111028Sjeff def_val = mp_ncpus; 112111028Sjeff#else 113111028Sjeff def_val = 1; 114111028Sjeff#endif 115111028Sjeff if (virtual_cpu == 0) 116111028Sjeff new_val = def_val; 117111028Sjeff else 118111028Sjeff new_val = virtual_cpu; 119111028Sjeff error = sysctl_handle_int(oidp, &new_val, 0, req); 120111028Sjeff if (error != 0 || req->newptr == NULL) 121111028Sjeff return (error); 122111028Sjeff if (new_val < 0) 123111028Sjeff return (EINVAL); 124111028Sjeff virtual_cpu = new_val; 125111028Sjeff return (0); 126111028Sjeff} 127111028Sjeff 128111028Sjeff/* DEBUG ONLY */ 129111028SjeffSYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 130111028Sjeff 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 131111028Sjeff "debug virtual cpus"); 132111028Sjeff 13399026Sjulian/* 134107719Sjulian * Prepare a thread for use. 13599026Sjulian */ 13699026Sjulianstatic void 13799026Sjulianthread_ctor(void *mem, int size, void *arg) 13899026Sjulian{ 13999026Sjulian struct thread *td; 14099026Sjulian 14199026Sjulian td = (struct thread *)mem; 142103216Sjulian td->td_state = TDS_INACTIVE; 14399026Sjulian} 14499026Sjulian 14599026Sjulian/* 14699026Sjulian * Reclaim a thread after use. 14799026Sjulian */ 14899026Sjulianstatic void 14999026Sjulianthread_dtor(void *mem, int size, void *arg) 15099026Sjulian{ 15199026Sjulian struct thread *td; 15299026Sjulian 15399026Sjulian td = (struct thread *)mem; 15499026Sjulian 15599026Sjulian#ifdef INVARIANTS 15699026Sjulian /* Verify that this thread is in a safe state to free. */ 15799026Sjulian switch (td->td_state) { 158103216Sjulian case TDS_INHIBITED: 159103216Sjulian case TDS_RUNNING: 160103216Sjulian case TDS_CAN_RUN: 16199026Sjulian case TDS_RUNQ: 16299026Sjulian /* 16399026Sjulian * We must never unlink a thread that is in one of 16499026Sjulian * these states, because it is currently active. 16599026Sjulian */ 16699026Sjulian panic("bad state for thread unlinking"); 16799026Sjulian /* NOTREACHED */ 168103216Sjulian case TDS_INACTIVE: 16999026Sjulian break; 17099026Sjulian default: 17199026Sjulian panic("bad thread state"); 17299026Sjulian /* NOTREACHED */ 17399026Sjulian } 17499026Sjulian#endif 17599026Sjulian} 17699026Sjulian 17799026Sjulian/* 17899026Sjulian * Initialize type-stable parts of a thread (when newly created). 17999026Sjulian */ 18099026Sjulianstatic void 18199026Sjulianthread_init(void *mem, int size) 18299026Sjulian{ 18399026Sjulian struct thread *td; 18499026Sjulian 18599026Sjulian td = (struct thread *)mem; 186103312Sjulian mtx_lock(&Giant); 187104354Sscottl pmap_new_thread(td, 0); 188103312Sjulian mtx_unlock(&Giant); 18999026Sjulian cpu_thread_setup(td); 190107126Sjeff td->td_sched = (struct td_sched *)&td[1]; 19199026Sjulian} 19299026Sjulian 19399026Sjulian/* 19499026Sjulian * Tear down type-stable parts of a thread (just before being discarded). 19599026Sjulian */ 19699026Sjulianstatic void 19799026Sjulianthread_fini(void *mem, int size) 19899026Sjulian{ 19999026Sjulian struct thread *td; 20099026Sjulian 20199026Sjulian td = (struct thread *)mem; 20299026Sjulian pmap_dispose_thread(td); 20399026Sjulian} 204111028Sjeff 205107126Sjeff/* 206107126Sjeff * Initialize type-stable parts of a kse (when newly created). 207107126Sjeff */ 208107126Sjeffstatic void 209107126Sjeffkse_init(void *mem, int size) 210107126Sjeff{ 211107126Sjeff struct kse *ke; 21299026Sjulian 213107126Sjeff ke = (struct kse *)mem; 214107126Sjeff ke->ke_sched = (struct ke_sched *)&ke[1]; 215107126Sjeff} 216111028Sjeff 217107126Sjeff/* 218107126Sjeff * Initialize type-stable parts of a ksegrp (when newly created). 219107126Sjeff */ 220107126Sjeffstatic void 221107126Sjeffksegrp_init(void *mem, int size) 222107126Sjeff{ 223107126Sjeff struct ksegrp *kg; 224107126Sjeff 225107126Sjeff kg = (struct ksegrp *)mem; 226107126Sjeff kg->kg_sched = (struct kg_sched *)&kg[1]; 227107126Sjeff} 228107126Sjeff 229105854Sjulian/* 230111028Sjeff * KSE is linked into kse group. 231105854Sjulian */ 232105854Sjulianvoid 233105854Sjuliankse_link(struct kse *ke, struct ksegrp *kg) 234105854Sjulian{ 235105854Sjulian struct proc *p = kg->kg_proc; 236105854Sjulian 237105854Sjulian TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 238105854Sjulian kg->kg_kses++; 239111028Sjeff ke->ke_state = KES_UNQUEUED; 240105854Sjulian ke->ke_proc = p; 241105854Sjulian ke->ke_ksegrp = kg; 242105854Sjulian ke->ke_thread = NULL; 243111028Sjeff ke->ke_oncpu = NOCPU; 244111028Sjeff ke->ke_flags = 0; 245105854Sjulian} 246105854Sjulian 247105854Sjulianvoid 248105854Sjuliankse_unlink(struct kse *ke) 249105854Sjulian{ 250105854Sjulian struct ksegrp *kg; 251105854Sjulian 252105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 253105854Sjulian kg = ke->ke_ksegrp; 254105854Sjulian TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 255111028Sjeff if (ke->ke_state == KES_IDLE) { 256111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 257111028Sjeff kg->kg_idle_kses--; 258105854Sjulian } 259111028Sjeff if (--kg->kg_kses == 0) 260111028Sjeff ksegrp_unlink(kg); 261105854Sjulian /* 262105854Sjulian * Aggregate stats from the KSE 263105854Sjulian */ 264105854Sjulian kse_stash(ke); 265105854Sjulian} 266105854Sjulian 267105854Sjulianvoid 268105854Sjulianksegrp_link(struct ksegrp *kg, struct proc *p) 269105854Sjulian{ 270105854Sjulian 271105854Sjulian TAILQ_INIT(&kg->kg_threads); 272105854Sjulian TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 273105854Sjulian TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 274105854Sjulian TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 275111028Sjeff TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 276111028Sjeff TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 277111028Sjeff kg->kg_proc = p; 278111028Sjeff /* 279111028Sjeff * the following counters are in the -zero- section 280111028Sjeff * and may not need clearing 281111028Sjeff */ 282105854Sjulian kg->kg_numthreads = 0; 283111028Sjeff kg->kg_runnable = 0; 284111028Sjeff kg->kg_kses = 0; 285111028Sjeff kg->kg_runq_kses = 0; /* XXXKSE change name */ 286111028Sjeff kg->kg_idle_kses = 0; 287111028Sjeff kg->kg_numupcalls = 0; 288111028Sjeff /* link it in now that it's consistent */ 289105854Sjulian p->p_numksegrps++; 290105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 291105854Sjulian} 292105854Sjulian 293105854Sjulianvoid 294105854Sjulianksegrp_unlink(struct ksegrp *kg) 295105854Sjulian{ 296105854Sjulian struct proc *p; 297105854Sjulian 298105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 299111028Sjeff KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 300111028Sjeff KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 301111028Sjeff KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 302111028Sjeff 303105854Sjulian p = kg->kg_proc; 304105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 305105854Sjulian p->p_numksegrps--; 306105854Sjulian /* 307105854Sjulian * Aggregate stats from the KSE 308105854Sjulian */ 309105854Sjulian ksegrp_stash(kg); 310105854Sjulian} 311105854Sjulian 312111028Sjeffstruct kse_upcall * 313111028Sjeffupcall_alloc(void) 314111028Sjeff{ 315111028Sjeff struct kse_upcall *ku; 316111028Sjeff 317111028Sjeff ku = uma_zalloc(upcall_zone, 0); 318111028Sjeff bzero(ku, sizeof(*ku)); 319111028Sjeff return (ku); 320111028Sjeff} 321111028Sjeff 322111028Sjeffvoid 323111028Sjeffupcall_free(struct kse_upcall *ku) 324111028Sjeff{ 325111028Sjeff 326111028Sjeff uma_zfree(upcall_zone, ku); 327111028Sjeff} 328111028Sjeff 329111028Sjeffvoid 330111028Sjeffupcall_link(struct kse_upcall *ku, struct ksegrp *kg) 331111028Sjeff{ 332111028Sjeff 333111028Sjeff mtx_assert(&sched_lock, MA_OWNED); 334111028Sjeff TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 335111028Sjeff ku->ku_ksegrp = kg; 336111028Sjeff kg->kg_numupcalls++; 337111028Sjeff} 338111028Sjeff 339111028Sjeffvoid 340111028Sjeffupcall_unlink(struct kse_upcall *ku) 341111028Sjeff{ 342111028Sjeff struct ksegrp *kg = ku->ku_ksegrp; 343111028Sjeff 344111028Sjeff mtx_assert(&sched_lock, MA_OWNED); 345111028Sjeff KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 346111028Sjeff TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 347111028Sjeff kg->kg_numupcalls--; 348111028Sjeff upcall_stash(ku); 349111028Sjeff} 350111028Sjeff 351111028Sjeffvoid 352111028Sjeffupcall_remove(struct thread *td) 353111028Sjeff{ 354111028Sjeff 355111028Sjeff if (td->td_upcall) { 356111028Sjeff td->td_upcall->ku_owner = NULL; 357111028Sjeff upcall_unlink(td->td_upcall); 358111028Sjeff td->td_upcall = 0; 359111028Sjeff } 360111028Sjeff} 361111028Sjeff 36299026Sjulian/* 363111028Sjeff * For a newly created process, 364111028Sjeff * link up all the structures and its initial threads etc. 365105854Sjulian */ 366105854Sjulianvoid 367105854Sjulianproc_linkup(struct proc *p, struct ksegrp *kg, 368111028Sjeff struct kse *ke, struct thread *td) 369105854Sjulian{ 370105854Sjulian 371105854Sjulian TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 372105854Sjulian TAILQ_INIT(&p->p_threads); /* all threads in proc */ 373105854Sjulian TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 374105854Sjulian p->p_numksegrps = 0; 375105854Sjulian p->p_numthreads = 0; 376105854Sjulian 377105854Sjulian ksegrp_link(kg, p); 378105854Sjulian kse_link(ke, kg); 379105854Sjulian thread_link(td, kg); 380105854Sjulian} 381105854Sjulian 382111028Sjeff/* 383111028Sjeffstruct kse_thr_interrupt_args { 384111028Sjeff struct kse_thr_mailbox * tmbx; 385111028Sjeff}; 386111028Sjeff*/ 387105854Sjulianint 388105854Sjuliankse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 389105854Sjulian{ 390106180Sdavidxu struct proc *p; 391106180Sdavidxu struct thread *td2; 392105854Sjulian 393106242Sdavidxu p = td->td_proc; 394111028Sjeff if (!(p->p_flag & P_KSES) || (uap->tmbx == NULL)) 395106242Sdavidxu return (EINVAL); 396106180Sdavidxu mtx_lock_spin(&sched_lock); 397106180Sdavidxu FOREACH_THREAD_IN_PROC(p, td2) { 398106180Sdavidxu if (td2->td_mailbox == uap->tmbx) { 399106180Sdavidxu td2->td_flags |= TDF_INTERRUPT; 400106180Sdavidxu if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { 401106180Sdavidxu if (td2->td_flags & TDF_CVWAITQ) 402106180Sdavidxu cv_abort(td2); 403106180Sdavidxu else 404106180Sdavidxu abortsleep(td2); 405111028Sjeff } 406106180Sdavidxu mtx_unlock_spin(&sched_lock); 407106182Sdavidxu return (0); 408106180Sdavidxu } 409106180Sdavidxu } 410106180Sdavidxu mtx_unlock_spin(&sched_lock); 411106182Sdavidxu return (ESRCH); 412105854Sjulian} 413105854Sjulian 414111028Sjeff/* 415111028Sjeffstruct kse_exit_args { 416111028Sjeff register_t dummy; 417111028Sjeff}; 418111028Sjeff*/ 419105854Sjulianint 420105854Sjuliankse_exit(struct thread *td, struct kse_exit_args *uap) 421105854Sjulian{ 422105854Sjulian struct proc *p; 423105854Sjulian struct ksegrp *kg; 424108640Sdavidxu struct kse *ke; 425105854Sjulian 426105854Sjulian p = td->td_proc; 427111028Sjeff /* 428111028Sjeff * Only UTS can call the syscall and current group 429111028Sjeff * should be a threaded group. 430111028Sjeff */ 431111028Sjeff if ((td->td_mailbox != NULL) || (td->td_ksegrp->kg_numupcalls == 0)) 432106182Sdavidxu return (EINVAL); 433111028Sjeff KASSERT((td->td_upcall != NULL), ("%s: not own an upcall", __func__)); 434111028Sjeff 435105854Sjulian kg = td->td_ksegrp; 436111028Sjeff /* Serialize removing upcall */ 437105854Sjulian PROC_LOCK(p); 438105854Sjulian mtx_lock_spin(&sched_lock); 439111028Sjeff if ((kg->kg_numupcalls == 1) && (kg->kg_numthreads > 1)) { 440105854Sjulian mtx_unlock_spin(&sched_lock); 441105854Sjulian PROC_UNLOCK(p); 442105854Sjulian return (EDEADLK); 443105854Sjulian } 444108640Sdavidxu ke = td->td_kse; 445111028Sjeff upcall_remove(td); 446108640Sdavidxu if (p->p_numthreads == 1) { 447111028Sjeff kse_purge(p, td); 448105854Sjulian p->p_flag &= ~P_KSES; 449105854Sjulian mtx_unlock_spin(&sched_lock); 450105854Sjulian PROC_UNLOCK(p); 451105854Sjulian } else { 452111028Sjeff if (kg->kg_numthreads == 1) { /* Shutdown a group */ 453111028Sjeff kse_purge_group(td); 454111028Sjeff ke->ke_flags |= KEF_EXIT; 455111028Sjeff } 456105854Sjulian thread_exit(); 457105854Sjulian /* NOTREACHED */ 458105854Sjulian } 459106182Sdavidxu return (0); 460105854Sjulian} 461105854Sjulian 462107719Sjulian/* 463108338Sjulian * Either becomes an upcall or waits for an awakening event and 464111028Sjeff * then becomes an upcall. Only error cases return. 465107719Sjulian */ 466111028Sjeff/* 467111028Sjeffstruct kse_release_args { 468111028Sjeff register_t dummy; 469111028Sjeff}; 470111028Sjeff*/ 471105854Sjulianint 472111028Sjeffkse_release(struct thread *td, struct kse_release_args *uap) 473105854Sjulian{ 474105854Sjulian struct proc *p; 475107719Sjulian struct ksegrp *kg; 476105854Sjulian 477105854Sjulian p = td->td_proc; 478107719Sjulian kg = td->td_ksegrp; 479106903Sdavidxu /* 480111028Sjeff * Only UTS can call the syscall and current group 481111028Sjeff * should be a threaded group. 482111028Sjeff */ 483111028Sjeff if ((td->td_mailbox != NULL) || (td->td_ksegrp->kg_numupcalls == 0)) 484107719Sjulian return (EINVAL); 485111028Sjeff KASSERT((td->td_upcall != NULL), ("%s: not own an upcall", __func__)); 486108338Sjulian 487107719Sjulian PROC_LOCK(p); 488108613Sjulian mtx_lock_spin(&sched_lock); 489108338Sjulian /* Change OURSELF to become an upcall. */ 490111028Sjeff td->td_flags = TDF_UPCALLING; 491111042Sdavidxu if (p->p_sflag & PS_NEEDSIGCHK) 492111042Sdavidxu td->td_flags |= TDF_ASTPENDING; 493111028Sjeff if ((td->td_upcall->ku_flags & KUF_DOUPCALL) == 0 && 494108613Sjulian (kg->kg_completed == NULL)) { 495111028Sjeff kg->kg_upsleeps++; 496111028Sjeff mtx_unlock_spin(&sched_lock); 497111028Sjeff msleep(&kg->kg_completed, &p->p_mtx, PPAUSE|PCATCH, "ksepause", 498111028Sjeff NULL); 499111028Sjeff kg->kg_upsleeps--; 500110190Sjulian PROC_UNLOCK(p); 501108338Sjulian } else { 502108613Sjulian mtx_unlock_spin(&sched_lock); 503108338Sjulian PROC_UNLOCK(p); 504105854Sjulian } 505107719Sjulian return (0); 506105854Sjulian} 507105854Sjulian 508105854Sjulian/* struct kse_wakeup_args { 509105854Sjulian struct kse_mailbox *mbx; 510105854Sjulian}; */ 511105854Sjulianint 512105854Sjuliankse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 513105854Sjulian{ 514105854Sjulian struct proc *p; 515105854Sjulian struct ksegrp *kg; 516111028Sjeff struct kse_upcall *ku; 517108338Sjulian struct thread *td2; 518105854Sjulian 519105854Sjulian p = td->td_proc; 520108338Sjulian td2 = NULL; 521111028Sjeff ku = NULL; 522105854Sjulian /* KSE-enabled processes only, please. */ 523105854Sjulian if (!(p->p_flag & P_KSES)) 524111028Sjeff return (EINVAL); 525108613Sjulian 526111028Sjeff PROC_LOCK(p); 527108613Sjulian mtx_lock_spin(&sched_lock); 528105854Sjulian if (uap->mbx) { 529105854Sjulian FOREACH_KSEGRP_IN_PROC(p, kg) { 530111028Sjeff FOREACH_UPCALL_IN_GROUP(kg, ku) { 531111028Sjeff if (ku->ku_mailbox == uap->mbx) 532111028Sjeff break; 533108613Sjulian } 534111028Sjeff if (ku) 535108338Sjulian break; 536105854Sjulian } 537105854Sjulian } else { 538105854Sjulian kg = td->td_ksegrp; 539111028Sjeff if (kg->kg_upsleeps) { 540111028Sjeff wakeup_one(&kg->kg_completed); 541111028Sjeff mtx_unlock_spin(&sched_lock); 542111028Sjeff PROC_UNLOCK(p); 543111028Sjeff return (0); 544108338Sjulian } 545111028Sjeff ku = TAILQ_FIRST(&kg->kg_upcalls); 546105854Sjulian } 547111028Sjeff if (ku) { 548111028Sjeff if ((td2 = ku->ku_owner) == NULL) { 549111028Sjeff panic("%s: no owner", __func__); 550111028Sjeff } else if (TD_ON_SLEEPQ(td2) && 551111028Sjeff (td2->td_wchan == &kg->kg_completed)) { 552111028Sjeff abortsleep(td2); 553111028Sjeff } else { 554111028Sjeff ku->ku_flags |= KUF_DOUPCALL; 555108613Sjulian } 556105854Sjulian mtx_unlock_spin(&sched_lock); 557111028Sjeff PROC_UNLOCK(p); 558108338Sjulian return (0); 559108613Sjulian } 560105854Sjulian mtx_unlock_spin(&sched_lock); 561111028Sjeff PROC_UNLOCK(p); 562108338Sjulian return (ESRCH); 563105854Sjulian} 564105854Sjulian 565105854Sjulian/* 566105854Sjulian * No new KSEG: first call: use current KSE, don't schedule an upcall 567111028Sjeff * All other situations, do allocate max new KSEs and schedule an upcall. 568105854Sjulian */ 569105854Sjulian/* struct kse_create_args { 570105854Sjulian struct kse_mailbox *mbx; 571105854Sjulian int newgroup; 572105854Sjulian}; */ 573105854Sjulianint 574105854Sjuliankse_create(struct thread *td, struct kse_create_args *uap) 575105854Sjulian{ 576105854Sjulian struct kse *newke; 577105854Sjulian struct ksegrp *newkg; 578105854Sjulian struct ksegrp *kg; 579105854Sjulian struct proc *p; 580105854Sjulian struct kse_mailbox mbx; 581111028Sjeff struct kse_upcall *newku; 582111028Sjeff int err, ncpus; 583105854Sjulian 584105854Sjulian p = td->td_proc; 585105854Sjulian if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 586105854Sjulian return (err); 587105854Sjulian 588111028Sjeff /* Too bad, why hasn't kernel always a cpu counter !? */ 589111028Sjeff#ifdef SMP 590111028Sjeff ncpus = mp_ncpus; 591111028Sjeff#else 592111028Sjeff ncpus = 1; 593111028Sjeff#endif 594111028Sjeff if (thread_debug && virtual_cpu != 0) 595111028Sjeff ncpus = virtual_cpu; 596111028Sjeff 597111028Sjeff /* Easier to just set it than to test and set */ 598111028Sjeff p->p_flag |= P_KSES; 599105854Sjulian kg = td->td_ksegrp; 600105854Sjulian if (uap->newgroup) { 601111028Sjeff /* Have race condition but it is cheap */ 602107006Sdavidxu if (p->p_numksegrps >= max_groups_per_proc) 603107006Sdavidxu return (EPROCLIM); 604105854Sjulian /* 605105854Sjulian * If we want a new KSEGRP it doesn't matter whether 606105854Sjulian * we have already fired up KSE mode before or not. 607111028Sjeff * We put the process in KSE mode and create a new KSEGRP. 608105854Sjulian */ 609105854Sjulian newkg = ksegrp_alloc(); 610105854Sjulian bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 611111028Sjeff kg_startzero, kg_endzero)); 612105854Sjulian bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 613105854Sjulian RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 614111028Sjeff mtx_lock_spin(&sched_lock); 615111028Sjeff ksegrp_link(newkg, p); 616111028Sjeff if (p->p_numksegrps >= max_groups_per_proc) { 617111028Sjeff ksegrp_unlink(newkg); 618111028Sjeff mtx_unlock_spin(&sched_lock); 619111028Sjeff return (EPROCLIM); 620111028Sjeff } 621111028Sjeff mtx_unlock_spin(&sched_lock); 622105854Sjulian } else { 623111028Sjeff newkg = kg; 624111028Sjeff } 625111028Sjeff 626111028Sjeff /* 627111028Sjeff * Creating upcalls more than number of physical cpu does 628111028Sjeff * not help performance. 629111028Sjeff */ 630111028Sjeff if (newkg->kg_numupcalls >= ncpus) 631111028Sjeff return (EPROCLIM); 632111028Sjeff 633111028Sjeff if (newkg->kg_numupcalls == 0) { 634111028Sjeff /* 635111028Sjeff * Initialize KSE group, optimized for MP. 636111028Sjeff * Create KSEs as many as physical cpus, this increases 637111028Sjeff * concurrent even if userland is not MP safe and can only run 638111028Sjeff * on single CPU (for early version of libpthread, it is true). 639111028Sjeff * In ideal world, every physical cpu should execute a thread. 640111028Sjeff * If there is enough KSEs, threads in kernel can be 641111028Sjeff * executed parallel on different cpus with full speed, 642111028Sjeff * Concurrent in kernel shouldn't be restricted by number of 643111028Sjeff * upcalls userland provides. 644111028Sjeff * Adding more upcall structures only increases concurrent 645111028Sjeff * in userland. 646111028Sjeff * Highest performance configuration is: 647111028Sjeff * N kses = N upcalls = N phyiscal cpus 648105854Sjulian */ 649111028Sjeff while (newkg->kg_kses < ncpus) { 650105854Sjulian newke = kse_alloc(); 651111028Sjeff bzero(&newke->ke_startzero, RANGEOF(struct kse, 652111028Sjeff ke_startzero, ke_endzero)); 653105854Sjulian#if 0 654111028Sjeff mtx_lock_spin(&sched_lock); 655111028Sjeff bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 656111028Sjeff RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 657111028Sjeff mtx_unlock_spin(&sched_lock); 658105854Sjulian#endif 659111028Sjeff mtx_lock_spin(&sched_lock); 660111028Sjeff kse_link(newke, newkg); 661111028Sjeff /* Add engine */ 662111028Sjeff kse_reassign(newke); 663111028Sjeff mtx_unlock_spin(&sched_lock); 664105854Sjulian } 665111028Sjeff } 666111028Sjeff newku = upcall_alloc(); 667111028Sjeff newku->ku_mailbox = uap->mbx; 668111028Sjeff newku->ku_func = mbx.km_func; 669111028Sjeff bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 670111028Sjeff 671111028Sjeff /* For the first call this may not have been set */ 672111028Sjeff if (td->td_standin == NULL) 673111028Sjeff thread_alloc_spare(td, NULL); 674111028Sjeff 675111028Sjeff mtx_lock_spin(&sched_lock); 676111028Sjeff if (newkg->kg_numupcalls >= ncpus) { 677111028Sjeff upcall_free(newku); 678105854Sjulian mtx_unlock_spin(&sched_lock); 679111028Sjeff return (EPROCLIM); 680111028Sjeff } 681111028Sjeff upcall_link(newku, newkg); 682111028Sjeff 683111028Sjeff /* 684111028Sjeff * Each upcall structure has an owner thread, find which 685111028Sjeff * one owns it. 686111028Sjeff */ 687111028Sjeff if (uap->newgroup) { 688111028Sjeff /* 689111028Sjeff * Because new ksegrp hasn't thread, 690111028Sjeff * create an initial upcall thread to own it. 691111028Sjeff */ 692111028Sjeff thread_schedule_upcall(td, newku); 693105854Sjulian } else { 694105854Sjulian /* 695111028Sjeff * If current thread hasn't an upcall structure, 696111028Sjeff * just assign the upcall to it. 697105854Sjulian */ 698111028Sjeff if (td->td_upcall == NULL) { 699111028Sjeff newku->ku_owner = td; 700111028Sjeff td->td_upcall = newku; 701111028Sjeff } else { 702111028Sjeff /* 703111028Sjeff * Create a new upcall thread to own it. 704111028Sjeff */ 705111028Sjeff thread_schedule_upcall(td, newku); 706111028Sjeff } 707105854Sjulian } 708111028Sjeff mtx_unlock_spin(&sched_lock); 709105854Sjulian return (0); 710105854Sjulian} 711105854Sjulian 712105854Sjulian/* 713103410Smini * Fill a ucontext_t with a thread's context information. 714103410Smini * 715103410Smini * This is an analogue to getcontext(3). 716103410Smini */ 717103410Sminivoid 718103410Sminithread_getcontext(struct thread *td, ucontext_t *uc) 719103410Smini{ 720103410Smini 721103464Speter/* 722103464Speter * XXX this is declared in a MD include file, i386/include/ucontext.h but 723103464Speter * is used in MI code. 724103464Speter */ 725103463Speter#ifdef __i386__ 726103410Smini get_mcontext(td, &uc->uc_mcontext); 727103463Speter#endif 728103410Smini uc->uc_sigmask = td->td_proc->p_sigmask; 729103410Smini} 730103410Smini 731103410Smini/* 732103410Smini * Set a thread's context from a ucontext_t. 733103410Smini * 734103410Smini * This is an analogue to setcontext(3). 735103410Smini */ 736103410Sminiint 737103410Sminithread_setcontext(struct thread *td, ucontext_t *uc) 738103410Smini{ 739103410Smini int ret; 740103410Smini 741103464Speter/* 742103464Speter * XXX this is declared in a MD include file, i386/include/ucontext.h but 743103464Speter * is used in MI code. 744103464Speter */ 745103463Speter#ifdef __i386__ 746103410Smini ret = set_mcontext(td, &uc->uc_mcontext); 747103463Speter#else 748103463Speter ret = ENOSYS; 749103463Speter#endif 750103410Smini if (ret == 0) { 751103410Smini SIG_CANTMASK(uc->uc_sigmask); 752103410Smini PROC_LOCK(td->td_proc); 753103410Smini td->td_proc->p_sigmask = uc->uc_sigmask; 754103410Smini PROC_UNLOCK(td->td_proc); 755103410Smini } 756103410Smini return (ret); 757103410Smini} 758103410Smini 759103410Smini/* 76099026Sjulian * Initialize global thread allocation resources. 76199026Sjulian */ 76299026Sjulianvoid 76399026Sjulianthreadinit(void) 76499026Sjulian{ 76599026Sjulian 766104437Speter#ifndef __ia64__ 767107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 76899026Sjulian thread_ctor, thread_dtor, thread_init, thread_fini, 76999026Sjulian UMA_ALIGN_CACHE, 0); 770104437Speter#else 771104437Speter /* 772104437Speter * XXX the ia64 kstack allocator is really lame and is at the mercy 773104437Speter * of contigmallloc(). This hackery is to pre-construct a whole 774104437Speter * pile of thread structures with associated kernel stacks early 775104437Speter * in the system startup while contigmalloc() still works. Once we 776104437Speter * have them, keep them. Sigh. 777104437Speter */ 778107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 779104437Speter thread_ctor, thread_dtor, thread_init, thread_fini, 780104437Speter UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); 781104437Speter uma_prealloc(thread_zone, 512); /* XXX arbitary */ 782104437Speter#endif 783107126Sjeff ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 784107126Sjeff NULL, NULL, ksegrp_init, NULL, 785103367Sjulian UMA_ALIGN_CACHE, 0); 786107126Sjeff kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 787107126Sjeff NULL, NULL, kse_init, NULL, 788103367Sjulian UMA_ALIGN_CACHE, 0); 789111028Sjeff upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 790111028Sjeff NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 79199026Sjulian} 79299026Sjulian 79399026Sjulian/* 794103002Sjulian * Stash an embarasingly extra thread into the zombie thread queue. 79599026Sjulian */ 79699026Sjulianvoid 79799026Sjulianthread_stash(struct thread *td) 79899026Sjulian{ 799111028Sjeff mtx_lock_spin(&kse_zombie_lock); 80099026Sjulian TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 801111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 80299026Sjulian} 80399026Sjulian 804103410Smini/* 805105854Sjulian * Stash an embarasingly extra kse into the zombie kse queue. 806105854Sjulian */ 807105854Sjulianvoid 808105854Sjuliankse_stash(struct kse *ke) 809105854Sjulian{ 810111028Sjeff mtx_lock_spin(&kse_zombie_lock); 811105854Sjulian TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 812111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 813105854Sjulian} 814105854Sjulian 815105854Sjulian/* 816111028Sjeff * Stash an embarasingly extra upcall into the zombie upcall queue. 817111028Sjeff */ 818111028Sjeff 819111028Sjeffvoid 820111028Sjeffupcall_stash(struct kse_upcall *ku) 821111028Sjeff{ 822111028Sjeff mtx_lock_spin(&kse_zombie_lock); 823111028Sjeff TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 824111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 825111028Sjeff} 826111028Sjeff 827111028Sjeff/* 828105854Sjulian * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 829105854Sjulian */ 830105854Sjulianvoid 831105854Sjulianksegrp_stash(struct ksegrp *kg) 832105854Sjulian{ 833111028Sjeff mtx_lock_spin(&kse_zombie_lock); 834105854Sjulian TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 835111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 836105854Sjulian} 837105854Sjulian 838105854Sjulian/* 839111028Sjeff * Reap zombie kse resource. 84099026Sjulian */ 84199026Sjulianvoid 84299026Sjulianthread_reap(void) 84399026Sjulian{ 844105854Sjulian struct thread *td_first, *td_next; 845105854Sjulian struct kse *ke_first, *ke_next; 846105854Sjulian struct ksegrp *kg_first, * kg_next; 847111028Sjeff struct kse_upcall *ku_first, *ku_next; 84899026Sjulian 84999026Sjulian /* 850111028Sjeff * Don't even bother to lock if none at this instant, 851111028Sjeff * we really don't care about the next instant.. 85299026Sjulian */ 853105854Sjulian if ((!TAILQ_EMPTY(&zombie_threads)) 854105854Sjulian || (!TAILQ_EMPTY(&zombie_kses)) 855111028Sjeff || (!TAILQ_EMPTY(&zombie_ksegrps)) 856111028Sjeff || (!TAILQ_EMPTY(&zombie_upcalls))) { 857111028Sjeff mtx_lock_spin(&kse_zombie_lock); 858105854Sjulian td_first = TAILQ_FIRST(&zombie_threads); 859105854Sjulian ke_first = TAILQ_FIRST(&zombie_kses); 860105854Sjulian kg_first = TAILQ_FIRST(&zombie_ksegrps); 861111028Sjeff ku_first = TAILQ_FIRST(&zombie_upcalls); 862105854Sjulian if (td_first) 863105854Sjulian TAILQ_INIT(&zombie_threads); 864105854Sjulian if (ke_first) 865105854Sjulian TAILQ_INIT(&zombie_kses); 866105854Sjulian if (kg_first) 867105854Sjulian TAILQ_INIT(&zombie_ksegrps); 868111028Sjeff if (ku_first) 869111028Sjeff TAILQ_INIT(&zombie_upcalls); 870111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 871105854Sjulian while (td_first) { 872105854Sjulian td_next = TAILQ_NEXT(td_first, td_runq); 873111028Sjeff if (td_first->td_ucred) 874111028Sjeff crfree(td_first->td_ucred); 875105854Sjulian thread_free(td_first); 876105854Sjulian td_first = td_next; 87799026Sjulian } 878105854Sjulian while (ke_first) { 879105854Sjulian ke_next = TAILQ_NEXT(ke_first, ke_procq); 880105854Sjulian kse_free(ke_first); 881105854Sjulian ke_first = ke_next; 882105854Sjulian } 883105854Sjulian while (kg_first) { 884105854Sjulian kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 885105854Sjulian ksegrp_free(kg_first); 886105854Sjulian kg_first = kg_next; 887105854Sjulian } 888111028Sjeff while (ku_first) { 889111028Sjeff ku_next = TAILQ_NEXT(ku_first, ku_link); 890111028Sjeff upcall_free(ku_first); 891111028Sjeff ku_first = ku_next; 892111028Sjeff } 89399026Sjulian } 89499026Sjulian} 89599026Sjulian 89699026Sjulian/* 897103367Sjulian * Allocate a ksegrp. 898103367Sjulian */ 899103367Sjulianstruct ksegrp * 900103367Sjulianksegrp_alloc(void) 901103367Sjulian{ 902111119Simp return (uma_zalloc(ksegrp_zone, M_WAITOK)); 903103367Sjulian} 904103367Sjulian 905103367Sjulian/* 906103367Sjulian * Allocate a kse. 907103367Sjulian */ 908103367Sjulianstruct kse * 909103367Sjuliankse_alloc(void) 910103367Sjulian{ 911111119Simp return (uma_zalloc(kse_zone, M_WAITOK)); 912103367Sjulian} 913103367Sjulian 914103367Sjulian/* 91599026Sjulian * Allocate a thread. 91699026Sjulian */ 91799026Sjulianstruct thread * 91899026Sjulianthread_alloc(void) 91999026Sjulian{ 92099026Sjulian thread_reap(); /* check if any zombies to get */ 921111119Simp return (uma_zalloc(thread_zone, M_WAITOK)); 92299026Sjulian} 92399026Sjulian 92499026Sjulian/* 925103367Sjulian * Deallocate a ksegrp. 926103367Sjulian */ 927103367Sjulianvoid 928103367Sjulianksegrp_free(struct ksegrp *td) 929103367Sjulian{ 930103367Sjulian uma_zfree(ksegrp_zone, td); 931103367Sjulian} 932103367Sjulian 933103367Sjulian/* 934103367Sjulian * Deallocate a kse. 935103367Sjulian */ 936103367Sjulianvoid 937103367Sjuliankse_free(struct kse *td) 938103367Sjulian{ 939103367Sjulian uma_zfree(kse_zone, td); 940103367Sjulian} 941103367Sjulian 942103367Sjulian/* 94399026Sjulian * Deallocate a thread. 94499026Sjulian */ 94599026Sjulianvoid 94699026Sjulianthread_free(struct thread *td) 94799026Sjulian{ 948107719Sjulian 949107719Sjulian cpu_thread_clean(td); 95099026Sjulian uma_zfree(thread_zone, td); 95199026Sjulian} 95299026Sjulian 95399026Sjulian/* 95499026Sjulian * Store the thread context in the UTS's mailbox. 955104031Sjulian * then add the mailbox at the head of a list we are building in user space. 956104031Sjulian * The list is anchored in the ksegrp structure. 95799026Sjulian */ 95899026Sjulianint 95999026Sjulianthread_export_context(struct thread *td) 96099026Sjulian{ 961104503Sjmallett struct proc *p; 962104031Sjulian struct ksegrp *kg; 963104031Sjulian uintptr_t mbx; 964104031Sjulian void *addr; 965111028Sjeff int error,temp; 966103410Smini ucontext_t uc; 96799026Sjulian 968104503Sjmallett p = td->td_proc; 969104503Sjmallett kg = td->td_ksegrp; 970104503Sjmallett 971104031Sjulian /* Export the user/machine context. */ 972111028Sjeff addr = (void *)(&td->td_mailbox->tm_context); 973104031Sjulian error = copyin(addr, &uc, sizeof(ucontext_t)); 974108338Sjulian if (error) 975108338Sjulian goto bad; 976104031Sjulian 977108338Sjulian thread_getcontext(td, &uc); 978108338Sjulian error = copyout(&uc, addr, sizeof(ucontext_t)); 979108338Sjulian if (error) 980108338Sjulian goto bad; 981108338Sjulian 982111028Sjeff /* Exports clock ticks in kernel mode */ 983111028Sjeff addr = (caddr_t)(&td->td_mailbox->tm_sticks); 984111028Sjeff temp = fuword(addr) + td->td_usticks; 985111028Sjeff if (suword(addr, temp)) 986111028Sjeff goto bad; 987111028Sjeff 988111028Sjeff /* Get address in latest mbox of list pointer */ 989104031Sjulian addr = (void *)(&td->td_mailbox->tm_next); 990104031Sjulian /* 991104031Sjulian * Put the saved address of the previous first 992104031Sjulian * entry into this one 993104031Sjulian */ 994104031Sjulian for (;;) { 995104031Sjulian mbx = (uintptr_t)kg->kg_completed; 996104031Sjulian if (suword(addr, mbx)) { 997108338Sjulian error = EFAULT; 998107034Sdavidxu goto bad; 999104031Sjulian } 1000104126Sjulian PROC_LOCK(p); 1001104031Sjulian if (mbx == (uintptr_t)kg->kg_completed) { 1002104031Sjulian kg->kg_completed = td->td_mailbox; 1003111028Sjeff /* 1004111028Sjeff * The thread context may be taken away by 1005111028Sjeff * other upcall threads when we unlock 1006111028Sjeff * process lock. it's no longer valid to 1007111028Sjeff * use it again in any other places. 1008111028Sjeff */ 1009111028Sjeff td->td_mailbox = NULL; 1010104126Sjulian PROC_UNLOCK(p); 1011104031Sjulian break; 1012104031Sjulian } 1013104126Sjulian PROC_UNLOCK(p); 1014104031Sjulian } 1015111028Sjeff td->td_usticks = 0; 1016104031Sjulian return (0); 1017107034Sdavidxu 1018107034Sdavidxubad: 1019107034Sdavidxu PROC_LOCK(p); 1020107034Sdavidxu psignal(p, SIGSEGV); 1021107034Sdavidxu PROC_UNLOCK(p); 1022111028Sjeff /* The mailbox is bad, don't use it */ 1023111028Sjeff td->td_mailbox = NULL; 1024111028Sjeff td->td_usticks = 0; 1025108338Sjulian return (error); 1026104031Sjulian} 102799026Sjulian 1028104031Sjulian/* 1029104031Sjulian * Take the list of completed mailboxes for this KSEGRP and put them on this 1030111028Sjeff * upcall's mailbox as it's the next one going up. 1031104031Sjulian */ 1032104031Sjulianstatic int 1033111028Sjeffthread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1034104031Sjulian{ 1035104126Sjulian struct proc *p = kg->kg_proc; 1036104031Sjulian void *addr; 1037104031Sjulian uintptr_t mbx; 1038104031Sjulian 1039111028Sjeff addr = (void *)(&ku->ku_mailbox->km_completed); 1040104031Sjulian for (;;) { 1041104031Sjulian mbx = (uintptr_t)kg->kg_completed; 1042104031Sjulian if (suword(addr, mbx)) { 1043104126Sjulian PROC_LOCK(p); 1044104126Sjulian psignal(p, SIGSEGV); 1045104126Sjulian PROC_UNLOCK(p); 1046104031Sjulian return (EFAULT); 1047104031Sjulian } 1048104031Sjulian /* XXXKSE could use atomic CMPXCH here */ 1049104126Sjulian PROC_LOCK(p); 1050104031Sjulian if (mbx == (uintptr_t)kg->kg_completed) { 1051104031Sjulian kg->kg_completed = NULL; 1052104126Sjulian PROC_UNLOCK(p); 1053104031Sjulian break; 1054104031Sjulian } 1055104126Sjulian PROC_UNLOCK(p); 105699026Sjulian } 1057104031Sjulian return (0); 105899026Sjulian} 105999026Sjulian 106099026Sjulian/* 1061107034Sdavidxu * This function should be called at statclock interrupt time 1062107034Sdavidxu */ 1063107034Sdavidxuint 1064111028Sjeffthread_statclock(int user) 1065107034Sdavidxu{ 1066107034Sdavidxu struct thread *td = curthread; 1067107034Sdavidxu 1068111028Sjeff if (td->td_ksegrp->kg_numupcalls == 0) 1069111028Sjeff return (-1); 1070107034Sdavidxu if (user) { 1071107034Sdavidxu /* Current always do via ast() */ 1072111032Sjulian td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1073111028Sjeff td->td_uuticks++; 1074107034Sdavidxu } else { 1075107034Sdavidxu if (td->td_mailbox != NULL) 1076111028Sjeff td->td_usticks++; 1077111028Sjeff else { 1078111028Sjeff /* XXXKSE 1079111028Sjeff * We will call thread_user_enter() for every 1080111028Sjeff * kernel entry in future, so if the thread mailbox 1081111028Sjeff * is NULL, it must be a UTS kernel, don't account 1082111028Sjeff * clock ticks for it. 1083111028Sjeff */ 1084111028Sjeff } 1085107034Sdavidxu } 1086111028Sjeff return (0); 1087107034Sdavidxu} 1088107034Sdavidxu 1089111028Sjeff/* 1090111028Sjeff * Export user mode state clock ticks 1091111028Sjeff */ 1092107034Sdavidxustatic int 1093111028Sjeffthread_update_usr_ticks(struct thread *td) 1094107034Sdavidxu{ 1095107034Sdavidxu struct proc *p = td->td_proc; 1096107034Sdavidxu struct kse_thr_mailbox *tmbx; 1097111028Sjeff struct kse_upcall *ku; 1098107034Sdavidxu caddr_t addr; 1099111028Sjeff uint uticks; 1100107034Sdavidxu 1101111028Sjeff if ((ku = td->td_upcall) == NULL) 1102111028Sjeff return (-1); 1103111028Sjeff 1104111028Sjeff tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1105107034Sdavidxu if ((tmbx == NULL) || (tmbx == (void *)-1)) 1106111028Sjeff return (-1); 1107111028Sjeff uticks = td->td_uuticks; 1108111028Sjeff td->td_uuticks = 0; 1109107034Sdavidxu if (uticks) { 1110111028Sjeff addr = (caddr_t)&tmbx->tm_uticks; 1111107034Sdavidxu uticks += fuword(addr); 1112111028Sjeff if (suword(addr, uticks)) { 1113111028Sjeff PROC_LOCK(p); 1114111028Sjeff psignal(p, SIGSEGV); 1115111028Sjeff PROC_UNLOCK(p); 1116111028Sjeff return (-2); 1117111028Sjeff } 1118107034Sdavidxu } 1119111028Sjeff return (0); 1120111028Sjeff} 1121111028Sjeff 1122111028Sjeff/* 1123111028Sjeff * Export kernel mode state clock ticks 1124111028Sjeff */ 1125111028Sjeff 1126111028Sjeffstatic int 1127111028Sjeffthread_update_sys_ticks(struct thread *td) 1128111028Sjeff{ 1129111028Sjeff struct proc *p = td->td_proc; 1130111028Sjeff caddr_t addr; 1131111028Sjeff int sticks; 1132111028Sjeff 1133111028Sjeff if (td->td_mailbox == NULL) 1134111028Sjeff return (-1); 1135111028Sjeff if (td->td_usticks == 0) 1136111028Sjeff return (0); 1137111028Sjeff addr = (caddr_t)&td->td_mailbox->tm_sticks; 1138111028Sjeff sticks = fuword(addr); 1139111028Sjeff /* XXXKSE use XCHG instead */ 1140111028Sjeff sticks += td->td_usticks; 1141111028Sjeff td->td_usticks = 0; 1142111028Sjeff if (suword(addr, sticks)) { 1143111028Sjeff PROC_LOCK(p); 1144111028Sjeff psignal(p, SIGSEGV); 1145111028Sjeff PROC_UNLOCK(p); 1146111028Sjeff return (-2); 1147107034Sdavidxu } 1148111028Sjeff return (0); 1149107034Sdavidxu} 1150107034Sdavidxu 1151107034Sdavidxu/* 115299026Sjulian * Discard the current thread and exit from its context. 115399026Sjulian * 115499026Sjulian * Because we can't free a thread while we're operating under its context, 1155107719Sjulian * push the current thread into our CPU's deadthread holder. This means 1156107719Sjulian * we needn't worry about someone else grabbing our context before we 1157107719Sjulian * do a cpu_throw(). 115899026Sjulian */ 115999026Sjulianvoid 116099026Sjulianthread_exit(void) 116199026Sjulian{ 116299026Sjulian struct thread *td; 116399026Sjulian struct kse *ke; 116499026Sjulian struct proc *p; 116599026Sjulian struct ksegrp *kg; 116699026Sjulian 116799026Sjulian td = curthread; 116899026Sjulian kg = td->td_ksegrp; 116999026Sjulian p = td->td_proc; 117099026Sjulian ke = td->td_kse; 117199026Sjulian 117299026Sjulian mtx_assert(&sched_lock, MA_OWNED); 1173102581Sjulian KASSERT(p != NULL, ("thread exiting without a process")); 1174102581Sjulian KASSERT(ke != NULL, ("thread exiting without a kse")); 1175102581Sjulian KASSERT(kg != NULL, ("thread exiting without a kse group")); 117699026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 117799026Sjulian CTR1(KTR_PROC, "thread_exit: thread %p", td); 117899026Sjulian KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 117999026Sjulian 1180104695Sjulian if (td->td_standin != NULL) { 1181104695Sjulian thread_stash(td->td_standin); 1182104695Sjulian td->td_standin = NULL; 1183104695Sjulian } 1184104695Sjulian 118599026Sjulian cpu_thread_exit(td); /* XXXSMP */ 118699026Sjulian 1187102581Sjulian /* 1188103002Sjulian * The last thread is left attached to the process 1189103002Sjulian * So that the whole bundle gets recycled. Skip 1190103002Sjulian * all this stuff. 1191102581Sjulian */ 1192103002Sjulian if (p->p_numthreads > 1) { 1193105854Sjulian /* 1194105854Sjulian * Unlink this thread from its proc and the kseg. 1195105854Sjulian * In keeping with the other structs we probably should 1196105854Sjulian * have a thread_unlink() that does some of this but it 1197105854Sjulian * would only be called from here (I think) so it would 1198105854Sjulian * be a waste. (might be useful for proc_fini() as well.) 1199105854Sjulian */ 1200103002Sjulian TAILQ_REMOVE(&p->p_threads, td, td_plist); 1201103002Sjulian p->p_numthreads--; 1202103002Sjulian TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1203103002Sjulian kg->kg_numthreads--; 1204111115Sdavidxu if (p->p_maxthrwaits) 1205111115Sdavidxu wakeup(&p->p_numthreads); 1206103002Sjulian /* 1207103002Sjulian * The test below is NOT true if we are the 1208103002Sjulian * sole exiting thread. P_STOPPED_SNGL is unset 1209103002Sjulian * in exit1() after it is the only survivor. 1210103002Sjulian */ 1211103002Sjulian if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1212103002Sjulian if (p->p_numthreads == p->p_suspcount) { 1213103216Sjulian thread_unsuspend_one(p->p_singlethread); 1214103002Sjulian } 121599026Sjulian } 1216104695Sjulian 1217111028Sjeff /* 1218111028Sjeff * Because each upcall structure has an owner thread, 1219111028Sjeff * owner thread exits only when process is in exiting 1220111028Sjeff * state, so upcall to userland is no longer needed, 1221111028Sjeff * deleting upcall structure is safe here. 1222111028Sjeff * So when all threads in a group is exited, all upcalls 1223111028Sjeff * in the group should be automatically freed. 1224111028Sjeff */ 1225111028Sjeff if (td->td_upcall) 1226111028Sjeff upcall_remove(td); 1227111028Sjeff 1228104695Sjulian ke->ke_state = KES_UNQUEUED; 1229111028Sjeff ke->ke_thread = NULL; 1230104695Sjulian /* 1231108338Sjulian * Decide what to do with the KSE attached to this thread. 1232104695Sjulian */ 1233111028Sjeff if (ke->ke_flags & KEF_EXIT) 1234105854Sjulian kse_unlink(ke); 1235111028Sjeff else 1236105854Sjulian kse_reassign(ke); 1237105854Sjulian PROC_UNLOCK(p); 1238111028Sjeff td->td_kse = NULL; 1239105854Sjulian td->td_state = TDS_INACTIVE; 1240105854Sjulian td->td_proc = NULL; 1241105854Sjulian td->td_ksegrp = NULL; 1242105854Sjulian td->td_last_kse = NULL; 1243107719Sjulian PCPU_SET(deadthread, td); 1244103002Sjulian } else { 1245103002Sjulian PROC_UNLOCK(p); 124699026Sjulian } 124799026Sjulian cpu_throw(); 124899026Sjulian /* NOTREACHED */ 124999026Sjulian} 125099026Sjulian 1251107719Sjulian/* 1252107719Sjulian * Do any thread specific cleanups that may be needed in wait() 1253107719Sjulian * called with Giant held, proc and schedlock not held. 1254107719Sjulian */ 1255107719Sjulianvoid 1256107719Sjulianthread_wait(struct proc *p) 1257107719Sjulian{ 1258107719Sjulian struct thread *td; 1259107719Sjulian 1260107719Sjulian KASSERT((p->p_numthreads == 1), ("Muliple threads in wait1()")); 1261107719Sjulian KASSERT((p->p_numksegrps == 1), ("Muliple ksegrps in wait1()")); 1262107719Sjulian FOREACH_THREAD_IN_PROC(p, td) { 1263107719Sjulian if (td->td_standin != NULL) { 1264107719Sjulian thread_free(td->td_standin); 1265107719Sjulian td->td_standin = NULL; 1266107719Sjulian } 1267107719Sjulian cpu_thread_clean(td); 1268107719Sjulian } 1269107719Sjulian thread_reap(); /* check for zombie threads etc. */ 1270107719Sjulian} 1271107719Sjulian 127299026Sjulian/* 127399026Sjulian * Link a thread to a process. 1274103002Sjulian * set up anything that needs to be initialized for it to 1275103002Sjulian * be used by the process. 127699026Sjulian * 127799026Sjulian * Note that we do not link to the proc's ucred here. 127899026Sjulian * The thread is linked as if running but no KSE assigned. 127999026Sjulian */ 128099026Sjulianvoid 128199026Sjulianthread_link(struct thread *td, struct ksegrp *kg) 128299026Sjulian{ 128399026Sjulian struct proc *p; 128499026Sjulian 128599026Sjulian p = kg->kg_proc; 1286111028Sjeff td->td_state = TDS_INACTIVE; 1287111028Sjeff td->td_proc = p; 1288111028Sjeff td->td_ksegrp = kg; 1289111028Sjeff td->td_last_kse = NULL; 1290111028Sjeff td->td_flags = 0; 1291111028Sjeff td->td_kse = NULL; 129299026Sjulian 1293103002Sjulian LIST_INIT(&td->td_contested); 1294103002Sjulian callout_init(&td->td_slpcallout, 1); 129599026Sjulian TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 129699026Sjulian TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 129799026Sjulian p->p_numthreads++; 129899026Sjulian kg->kg_numthreads++; 129999026Sjulian} 130099026Sjulian 1301111028Sjeff/* 1302111028Sjeff * Purge a ksegrp resource. When a ksegrp is preparing to 1303111028Sjeff * exit, it calls this function. 1304111028Sjeff */ 1305105854Sjulianvoid 1306111028Sjeffkse_purge_group(struct thread *td) 1307111028Sjeff{ 1308111028Sjeff struct ksegrp *kg; 1309111028Sjeff struct kse *ke; 1310111028Sjeff 1311111028Sjeff kg = td->td_ksegrp; 1312111028Sjeff KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1313111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1314111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 1315111028Sjeff ("%s: wrong idle KSE state", __func__)); 1316111028Sjeff kse_unlink(ke); 1317111028Sjeff } 1318111028Sjeff KASSERT((kg->kg_kses == 1), 1319111028Sjeff ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1320111028Sjeff KASSERT((kg->kg_numupcalls == 0), 1321111028Sjeff ("%s: ksegrp still has %d upcall datas", 1322111028Sjeff __func__, kg->kg_numupcalls)); 1323111028Sjeff} 1324111028Sjeff 1325111028Sjeff/* 1326111028Sjeff * Purge a process's KSE resource. When a process is preparing to 1327111028Sjeff * exit, it calls kse_purge to release any extra KSE resources in 1328111028Sjeff * the process. 1329111028Sjeff */ 1330111028Sjeffvoid 1331105854Sjuliankse_purge(struct proc *p, struct thread *td) 1332105854Sjulian{ 1333105854Sjulian struct ksegrp *kg; 1334111028Sjeff struct kse *ke; 1335105854Sjulian 1336105854Sjulian KASSERT(p->p_numthreads == 1, ("bad thread number")); 1337105854Sjulian mtx_lock_spin(&sched_lock); 1338105854Sjulian while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1339105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1340105854Sjulian p->p_numksegrps--; 1341111028Sjeff /* 1342111028Sjeff * There is no ownership for KSE, after all threads 1343111028Sjeff * in the group exited, it is possible that some KSEs 1344111028Sjeff * were left in idle queue, gc them now. 1345111028Sjeff */ 1346111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1347111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 1348111028Sjeff ("%s: wrong idle KSE state", __func__)); 1349111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1350111028Sjeff kg->kg_idle_kses--; 1351111028Sjeff TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1352111028Sjeff kg->kg_kses--; 1353111028Sjeff kse_stash(ke); 1354111028Sjeff } 1355105854Sjulian KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1356111028Sjeff ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1357111028Sjeff ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1358111028Sjeff KASSERT((kg->kg_numupcalls == 0), 1359111028Sjeff ("%s: ksegrp still has %d upcall datas", 1360111028Sjeff __func__, kg->kg_numupcalls)); 1361111028Sjeff 1362111028Sjeff if (kg != td->td_ksegrp) 1363105854Sjulian ksegrp_stash(kg); 1364105854Sjulian } 1365105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1366105854Sjulian p->p_numksegrps++; 1367105854Sjulian mtx_unlock_spin(&sched_lock); 1368105854Sjulian} 1369105854Sjulian 1370111028Sjeff/* 1371111028Sjeff * This function is intended to be used to initialize a spare thread 1372111028Sjeff * for upcall. Initialize thread's large data area outside sched_lock 1373111028Sjeff * for thread_schedule_upcall(). 1374111028Sjeff */ 1375111028Sjeffvoid 1376111028Sjeffthread_alloc_spare(struct thread *td, struct thread *spare) 1377111028Sjeff{ 1378111028Sjeff if (td->td_standin) 1379111028Sjeff return; 1380111028Sjeff if (spare == NULL) 1381111028Sjeff spare = thread_alloc(); 1382111028Sjeff td->td_standin = spare; 1383111028Sjeff bzero(&spare->td_startzero, 1384111028Sjeff (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1385111028Sjeff spare->td_proc = td->td_proc; 1386111028Sjeff /* Setup PCB and fork address */ 1387111028Sjeff cpu_set_upcall(spare, td->td_pcb); 1388111028Sjeff /* 1389111028Sjeff * XXXKSE do we really need this? (default values for the 1390111028Sjeff * frame). 1391111028Sjeff */ 1392111028Sjeff bcopy(td->td_frame, spare->td_frame, sizeof(struct trapframe)); 1393111028Sjeff spare->td_ucred = crhold(td->td_ucred); 1394111028Sjeff} 1395105854Sjulian 139699026Sjulian/* 1397103410Smini * Create a thread and schedule it for upcall on the KSE given. 1398108338Sjulian * Use our thread's standin so that we don't have to allocate one. 139999026Sjulian */ 140099026Sjulianstruct thread * 1401111028Sjeffthread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 140299026Sjulian{ 140399026Sjulian struct thread *td2; 140499026Sjulian 140599026Sjulian mtx_assert(&sched_lock, MA_OWNED); 1406104695Sjulian 1407104695Sjulian /* 1408111028Sjeff * Schedule an upcall thread on specified kse_upcall, 1409111028Sjeff * the kse_upcall must be free. 1410111028Sjeff * td must have a spare thread. 1411104695Sjulian */ 1412111028Sjeff KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1413104695Sjulian if ((td2 = td->td_standin) != NULL) { 1414104695Sjulian td->td_standin = NULL; 141599026Sjulian } else { 1416111028Sjeff panic("no reserve thread when scheduling an upcall"); 1417106182Sdavidxu return (NULL); 141899026Sjulian } 141999026Sjulian CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1420104695Sjulian td2, td->td_proc->p_pid, td->td_proc->p_comm); 1421103002Sjulian bcopy(&td->td_startcopy, &td2->td_startcopy, 1422103002Sjulian (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1423111028Sjeff thread_link(td2, ku->ku_ksegrp); 1424111028Sjeff /* Let the new thread become owner of the upcall */ 1425111028Sjeff ku->ku_owner = td2; 1426111028Sjeff td2->td_upcall = ku; 1427111028Sjeff td2->td_flags = TDF_UPCALLING; 1428111041Sdavidxu if (td->td_proc->p_sflag & PS_NEEDSIGCHK) 1429111041Sdavidxu td2->td_flags |= TDF_ASTPENDING; 1430111028Sjeff td2->td_kse = NULL; 1431111028Sjeff td2->td_state = TDS_CAN_RUN; 1432104695Sjulian td2->td_inhibitors = 0; 1433111028Sjeff setrunqueue(td2); 1434104695Sjulian return (td2); /* bogus.. should be a void function */ 143599026Sjulian} 143699026Sjulian 1437111033Sjeffvoid 1438111033Sjeffthread_signal_add(struct thread *td, int sig) 1439103410Smini{ 1440111033Sjeff struct kse_upcall *ku; 1441111033Sjeff struct proc *p; 1442103410Smini sigset_t ss; 1443103410Smini int error; 1444103410Smini 1445111033Sjeff PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); 1446111033Sjeff td = curthread; 1447111033Sjeff ku = td->td_upcall; 1448111033Sjeff p = td->td_proc; 1449111033Sjeff 1450103410Smini PROC_UNLOCK(p); 1451111033Sjeff error = copyin(&ku->ku_mailbox->km_sigscaught, &ss, sizeof(sigset_t)); 1452103410Smini if (error) 1453111033Sjeff goto error; 1454111033Sjeff 1455103410Smini SIGADDSET(ss, sig); 1456111033Sjeff 1457111033Sjeff error = copyout(&ss, &ku->ku_mailbox->km_sigscaught, sizeof(sigset_t)); 1458111033Sjeff if (error) 1459111033Sjeff goto error; 1460111033Sjeff 1461103410Smini PROC_LOCK(p); 1462111033Sjeff return; 1463111033Sjefferror: 1464111033Sjeff PROC_LOCK(p); 1465111033Sjeff sigexit(td, SIGILL); 1466111033Sjeff} 1467111033Sjeff 1468111033Sjeff 1469111033Sjeff/* 1470111033Sjeff * Schedule an upcall to notify a KSE process recieved signals. 1471111033Sjeff * 1472111033Sjeff */ 1473111033Sjeffvoid 1474111033Sjeffthread_signal_upcall(struct thread *td) 1475111033Sjeff{ 1476103410Smini mtx_lock_spin(&sched_lock); 1477111033Sjeff td->td_flags |= TDF_UPCALLING; 1478103410Smini mtx_unlock_spin(&sched_lock); 1479111033Sjeff 1480111033Sjeff return; 1481103410Smini} 1482103410Smini 1483103410Smini/* 1484111028Sjeff * Setup done on the thread when it enters the kernel. 1485105900Sjulian * XXXKSE Presently only for syscalls but eventually all kernel entries. 1486105900Sjulian */ 1487105900Sjulianvoid 1488105900Sjulianthread_user_enter(struct proc *p, struct thread *td) 1489105900Sjulian{ 1490111028Sjeff struct ksegrp *kg; 1491111028Sjeff struct kse_upcall *ku; 1492105900Sjulian 1493111028Sjeff kg = td->td_ksegrp; 1494105900Sjulian /* 1495105900Sjulian * First check that we shouldn't just abort. 1496105900Sjulian * But check if we are the single thread first! 1497105900Sjulian * XXX p_singlethread not locked, but should be safe. 1498105900Sjulian */ 1499111028Sjeff if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1500105900Sjulian PROC_LOCK(p); 1501105900Sjulian mtx_lock_spin(&sched_lock); 1502105900Sjulian thread_exit(); 1503105900Sjulian /* NOTREACHED */ 1504105900Sjulian } 1505105900Sjulian 1506105900Sjulian /* 1507105900Sjulian * If we are doing a syscall in a KSE environment, 1508105900Sjulian * note where our mailbox is. There is always the 1509108338Sjulian * possibility that we could do this lazily (in kse_reassign()), 1510105900Sjulian * but for now do it every time. 1511105900Sjulian */ 1512111028Sjeff kg = td->td_ksegrp; 1513111028Sjeff if (kg->kg_numupcalls) { 1514111028Sjeff ku = td->td_upcall; 1515111028Sjeff KASSERT(ku, ("%s: no upcall owned", __func__)); 1516111028Sjeff KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1517105900Sjulian td->td_mailbox = 1518111028Sjeff (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1519105900Sjulian if ((td->td_mailbox == NULL) || 1520107034Sdavidxu (td->td_mailbox == (void *)-1)) { 1521111028Sjeff /* Don't schedule upcall when blocked */ 1522111028Sjeff td->td_mailbox = NULL; 1523107034Sdavidxu mtx_lock_spin(&sched_lock); 1524111028Sjeff td->td_flags &= ~TDF_CAN_UNBIND; 1525107034Sdavidxu mtx_unlock_spin(&sched_lock); 1526105900Sjulian } else { 1527111115Sdavidxu if (td->td_standin == NULL) 1528111115Sdavidxu thread_alloc_spare(td, NULL); 1529111115Sdavidxu mtx_lock_spin(&sched_lock); 1530111115Sdavidxu td->td_flags |= TDF_CAN_UNBIND; 1531111115Sdavidxu mtx_unlock_spin(&sched_lock); 1532105900Sjulian } 1533105900Sjulian } 1534105900Sjulian} 1535105900Sjulian 1536105900Sjulian/* 1537103410Smini * The extra work we go through if we are a threaded process when we 1538103410Smini * return to userland. 1539103410Smini * 154099026Sjulian * If we are a KSE process and returning to user mode, check for 154199026Sjulian * extra work to do before we return (e.g. for more syscalls 154299026Sjulian * to complete first). If we were in a critical section, we should 154399026Sjulian * just return to let it finish. Same if we were in the UTS (in 1544103410Smini * which case the mailbox's context's busy indicator will be set). 1545103410Smini * The only traps we suport will have set the mailbox. 1546103410Smini * We will clear it here. 154799026Sjulian */ 154899026Sjulianint 1549103838Sjulianthread_userret(struct thread *td, struct trapframe *frame) 155099026Sjulian{ 1551111115Sdavidxu int error = 0, upcalls; 1552111028Sjeff struct kse_upcall *ku; 1553111115Sdavidxu struct ksegrp *kg, *kg2; 1554104695Sjulian struct proc *p; 1555107060Sdavidxu struct timespec ts; 155699026Sjulian 1557111028Sjeff p = td->td_proc; 1558110190Sjulian kg = td->td_ksegrp; 1559104695Sjulian 1560111028Sjeff /* Nothing to do with non-threaded group/process */ 1561111028Sjeff if (td->td_ksegrp->kg_numupcalls == 0) 1562111028Sjeff return (0); 1563108338Sjulian 1564103410Smini /* 1565111028Sjeff * Stat clock interrupt hit in userland, it 1566111028Sjeff * is returning from interrupt, charge thread's 1567111028Sjeff * userland time for UTS. 1568103410Smini */ 1569111028Sjeff if (td->td_flags & TDF_USTATCLOCK) { 1570111028Sjeff thread_update_usr_ticks(td); 1571111028Sjeff mtx_lock_spin(&sched_lock); 1572111028Sjeff td->td_flags &= ~TDF_USTATCLOCK; 1573111028Sjeff mtx_unlock_spin(&sched_lock); 1574111028Sjeff } 1575108338Sjulian 1576111028Sjeff /* 1577111028Sjeff * Optimisation: 1578111028Sjeff * This thread has not started any upcall. 1579111028Sjeff * If there is no work to report other than ourself, 1580111028Sjeff * then it can return direct to userland. 1581111028Sjeff */ 1582108338Sjulian if (TD_CAN_UNBIND(td)) { 1583111028Sjeff mtx_lock_spin(&sched_lock); 1584111028Sjeff td->td_flags &= ~TDF_CAN_UNBIND; 1585111028Sjeff mtx_unlock_spin(&sched_lock); 1586111028Sjeff if ((kg->kg_completed == NULL) && 1587111028Sjeff (td->td_upcall->ku_flags & KUF_DOUPCALL) == 0) { 1588111028Sjeff thread_update_sys_ticks(td); 1589108338Sjulian td->td_mailbox = NULL; 1590108338Sjulian return (0); 1591108338Sjulian } 1592104695Sjulian error = thread_export_context(td); 1593104695Sjulian if (error) { 1594104695Sjulian /* 1595111028Sjeff * Failing to do the KSE operation just defaults 1596104695Sjulian * back to synchonous operation, so just return from 1597108338Sjulian * the syscall. 1598104695Sjulian */ 1599111028Sjeff return (0); 1600104695Sjulian } 1601104695Sjulian /* 1602111028Sjeff * There is something to report, and we own an upcall 1603111028Sjeff * strucuture, we can go to userland. 1604111028Sjeff * Turn ourself into an upcall thread. 1605104695Sjulian */ 1606111028Sjeff mtx_lock_spin(&sched_lock); 1607104695Sjulian td->td_flags |= TDF_UPCALLING; 1608108338Sjulian mtx_unlock_spin(&sched_lock); 1609111028Sjeff } else if (td->td_mailbox) { 1610108338Sjulian error = thread_export_context(td); 1611108338Sjulian if (error) { 1612108338Sjulian PROC_LOCK(td->td_proc); 1613108338Sjulian mtx_lock_spin(&sched_lock); 1614108338Sjulian /* possibly upcall with error? */ 1615108338Sjulian } else { 1616111028Sjeff PROC_LOCK(td->td_proc); 1617111028Sjeff mtx_lock_spin(&sched_lock); 1618108338Sjulian /* 1619111028Sjeff * There are upcall threads waiting for 1620111028Sjeff * work to do, wake one of them up. 1621111028Sjeff * XXXKSE Maybe wake all of them up. 1622108338Sjulian */ 1623111028Sjeff if (kg->kg_upsleeps) 1624111028Sjeff wakeup_one(&kg->kg_completed); 1625108338Sjulian } 1626108338Sjulian thread_exit(); 1627111028Sjeff /* NOTREACHED */ 1628104695Sjulian } 1629104695Sjulian 1630108338Sjulian if (td->td_flags & TDF_UPCALLING) { 1631111028Sjeff KASSERT(TD_CAN_UNBIND(td) == 0, ("upcall thread can unbind")); 1632111028Sjeff ku = td->td_upcall; 1633108338Sjulian /* 1634108338Sjulian * There is no more work to do and we are going to ride 1635111028Sjeff * this thread up to userland as an upcall. 1636108338Sjulian * Do the last parts of the setup needed for the upcall. 1637108338Sjulian */ 1638108338Sjulian CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1639108338Sjulian td, td->td_proc->p_pid, td->td_proc->p_comm); 1640104695Sjulian 1641108338Sjulian /* 1642108338Sjulian * Set user context to the UTS. 1643108338Sjulian * Will use Giant in cpu_thread_clean() because it uses 1644108338Sjulian * kmem_free(kernel_map, ...) 1645108338Sjulian */ 1646111028Sjeff cpu_set_upcall_kse(td, ku); 1647104695Sjulian 1648111028Sjeff /* 1649111028Sjeff * Clear TDF_UPCALLING after set upcall context, 1650111028Sjeff * profiling code looks TDF_UPCALLING to avoid account 1651111028Sjeff * a wrong user %EIP 1652111028Sjeff */ 1653111028Sjeff mtx_lock_spin(&sched_lock); 1654111028Sjeff td->td_flags &= ~TDF_UPCALLING; 1655111028Sjeff if (ku->ku_flags & KUF_DOUPCALL) 1656111028Sjeff ku->ku_flags &= ~KUF_DOUPCALL; 1657111028Sjeff mtx_unlock_spin(&sched_lock); 1658111028Sjeff 1659111028Sjeff /* 1660108338Sjulian * Unhook the list of completed threads. 1661108338Sjulian * anything that completes after this gets to 1662108338Sjulian * come in next time. 1663108338Sjulian * Put the list of completed thread mailboxes on 1664108338Sjulian * this KSE's mailbox. 1665108338Sjulian */ 1666111028Sjeff error = thread_link_mboxes(kg, ku); 1667108338Sjulian if (error) 1668111115Sdavidxu goto out; 166999026Sjulian 1670108338Sjulian /* 1671108338Sjulian * Set state and clear the thread mailbox pointer. 1672108338Sjulian * From now on we are just a bound outgoing process. 1673108338Sjulian * **Problem** userret is often called several times. 1674108338Sjulian * it would be nice if this all happenned only on the first 1675108338Sjulian * time through. (the scan for extra work etc.) 1676108338Sjulian */ 1677111028Sjeff error = suword((caddr_t)&ku->ku_mailbox->km_curthread, 0); 1678108338Sjulian if (error) 1679111115Sdavidxu goto out; 1680111028Sjeff 1681111028Sjeff /* Export current system time */ 1682107060Sdavidxu nanotime(&ts); 1683111115Sdavidxu error = copyout(&ts, (caddr_t)&ku->ku_mailbox->km_timeofday, 1684111115Sdavidxu sizeof(ts)); 1685111115Sdavidxu } 1686111115Sdavidxu 1687111115Sdavidxuout: 1688111115Sdavidxu if (p->p_numthreads > max_threads_per_proc) { 1689111115Sdavidxu max_threads_hits++; 1690111115Sdavidxu PROC_LOCK(p); 1691111115Sdavidxu while (p->p_numthreads > max_threads_per_proc) { 1692111115Sdavidxu if (P_SHOULDSTOP(p)) 1693111115Sdavidxu break; 1694111115Sdavidxu upcalls = 0; 1695111115Sdavidxu mtx_lock_spin(&sched_lock); 1696111115Sdavidxu FOREACH_KSEGRP_IN_PROC(p, kg2) 1697111115Sdavidxu upcalls += kg2->kg_numupcalls; 1698111115Sdavidxu mtx_unlock_spin(&sched_lock); 1699111115Sdavidxu if (upcalls >= max_threads_per_proc) 1700111115Sdavidxu break; 1701111115Sdavidxu p->p_maxthrwaits++; 1702111115Sdavidxu msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, "maxthreads", 1703111115Sdavidxu NULL); 1704111115Sdavidxu p->p_maxthrwaits--; 1705107060Sdavidxu } 1706111115Sdavidxu PROC_UNLOCK(p); 1707107060Sdavidxu } 1708108338Sjulian 1709111115Sdavidxu if (error) { 1710111115Sdavidxu /* 1711111115Sdavidxu * Things are going to be so screwed we should just kill the process. 1712111115Sdavidxu * how do we do that? 1713111115Sdavidxu */ 1714111115Sdavidxu PROC_LOCK(td->td_proc); 1715111115Sdavidxu psignal(td->td_proc, SIGSEGV); 1716111115Sdavidxu PROC_UNLOCK(td->td_proc); 1717111115Sdavidxu } else { 1718111115Sdavidxu /* 1719111115Sdavidxu * Optimisation: 1720111115Sdavidxu * Ensure that we have a spare thread available, 1721111115Sdavidxu * for when we re-enter the kernel. 1722111115Sdavidxu */ 1723111115Sdavidxu if (td->td_standin == NULL) 1724111115Sdavidxu thread_alloc_spare(td, NULL); 1725111115Sdavidxu } 1726111115Sdavidxu 1727111028Sjeff /* 1728111028Sjeff * Clear thread mailbox first, then clear system tick count. 1729111028Sjeff * The order is important because thread_statclock() use 1730111028Sjeff * mailbox pointer to see if it is an userland thread or 1731111028Sjeff * an UTS kernel thread. 1732111028Sjeff */ 1733108338Sjulian td->td_mailbox = NULL; 1734111028Sjeff td->td_usticks = 0; 1735104695Sjulian return (error); /* go sync */ 173699026Sjulian} 173799026Sjulian 173899026Sjulian/* 173999026Sjulian * Enforce single-threading. 174099026Sjulian * 174199026Sjulian * Returns 1 if the caller must abort (another thread is waiting to 174299026Sjulian * exit the process or similar). Process is locked! 174399026Sjulian * Returns 0 when you are successfully the only thread running. 174499026Sjulian * A process has successfully single threaded in the suspend mode when 174599026Sjulian * There are no threads in user mode. Threads in the kernel must be 174699026Sjulian * allowed to continue until they get to the user boundary. They may even 174799026Sjulian * copy out their return values and data before suspending. They may however be 174899026Sjulian * accellerated in reaching the user boundary as we will wake up 174999026Sjulian * any sleeping threads that are interruptable. (PCATCH). 175099026Sjulian */ 175199026Sjulianint 175299026Sjulianthread_single(int force_exit) 175399026Sjulian{ 175499026Sjulian struct thread *td; 175599026Sjulian struct thread *td2; 175699026Sjulian struct proc *p; 175799026Sjulian 175899026Sjulian td = curthread; 175999026Sjulian p = td->td_proc; 1760107719Sjulian mtx_assert(&Giant, MA_OWNED); 176199026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 176299026Sjulian KASSERT((td != NULL), ("curthread is NULL")); 176399026Sjulian 176499026Sjulian if ((p->p_flag & P_KSES) == 0) 176599026Sjulian return (0); 176699026Sjulian 1767100648Sjulian /* Is someone already single threading? */ 1768100648Sjulian if (p->p_singlethread) 176999026Sjulian return (1); 177099026Sjulian 1771108338Sjulian if (force_exit == SINGLE_EXIT) { 177299026Sjulian p->p_flag |= P_SINGLE_EXIT; 1773108338Sjulian } else 177499026Sjulian p->p_flag &= ~P_SINGLE_EXIT; 1775102950Sdavidxu p->p_flag |= P_STOPPED_SINGLE; 177699026Sjulian p->p_singlethread = td; 1777105911Sjulian /* XXXKSE Which lock protects the below values? */ 177899026Sjulian while ((p->p_numthreads - p->p_suspcount) != 1) { 1779103216Sjulian mtx_lock_spin(&sched_lock); 178099026Sjulian FOREACH_THREAD_IN_PROC(p, td2) { 178199026Sjulian if (td2 == td) 178299026Sjulian continue; 1783111115Sdavidxu td->td_flags |= TDF_ASTPENDING; 1784103216Sjulian if (TD_IS_INHIBITED(td2)) { 1785105911Sjulian if (force_exit == SINGLE_EXIT) { 1786105911Sjulian if (TD_IS_SUSPENDED(td2)) { 1787103216Sjulian thread_unsuspend_one(td2); 1788105911Sjulian } 1789105911Sjulian if (TD_ON_SLEEPQ(td2) && 1790105911Sjulian (td2->td_flags & TDF_SINTR)) { 1791105911Sjulian if (td2->td_flags & TDF_CVWAITQ) 1792105911Sjulian cv_abort(td2); 1793105911Sjulian else 1794105911Sjulian abortsleep(td2); 1795105911Sjulian } 1796105911Sjulian } else { 1797105911Sjulian if (TD_IS_SUSPENDED(td2)) 1798105874Sdavidxu continue; 1799111028Sjeff /* 1800111028Sjeff * maybe other inhibitted states too? 1801111028Sjeff * XXXKSE Is it totally safe to 1802111028Sjeff * suspend a non-interruptable thread? 1803111028Sjeff */ 1804108338Sjulian if (td2->td_inhibitors & 1805111028Sjeff (TDI_SLEEPING | TDI_SWAPPED)) 1806105911Sjulian thread_suspend_one(td2); 180799026Sjulian } 180899026Sjulian } 180999026Sjulian } 1810105911Sjulian /* 1811105911Sjulian * Maybe we suspended some threads.. was it enough? 1812105911Sjulian */ 1813105911Sjulian if ((p->p_numthreads - p->p_suspcount) == 1) { 1814105911Sjulian mtx_unlock_spin(&sched_lock); 1815105911Sjulian break; 1816105911Sjulian } 1817105911Sjulian 181899026Sjulian /* 181999026Sjulian * Wake us up when everyone else has suspended. 1820100648Sjulian * In the mean time we suspend as well. 182199026Sjulian */ 1822103216Sjulian thread_suspend_one(td); 182399026Sjulian mtx_unlock(&Giant); 182499026Sjulian PROC_UNLOCK(p); 1825107719Sjulian p->p_stats->p_ru.ru_nvcsw++; 182699026Sjulian mi_switch(); 182799026Sjulian mtx_unlock_spin(&sched_lock); 182899026Sjulian mtx_lock(&Giant); 182999026Sjulian PROC_LOCK(p); 183099026Sjulian } 1831111028Sjeff if (force_exit == SINGLE_EXIT) { 1832111028Sjeff if (td->td_upcall) { 1833111028Sjeff mtx_lock_spin(&sched_lock); 1834111028Sjeff upcall_remove(td); 1835111028Sjeff mtx_unlock_spin(&sched_lock); 1836111028Sjeff } 1837105854Sjulian kse_purge(p, td); 1838111028Sjeff } 183999026Sjulian return (0); 184099026Sjulian} 184199026Sjulian 184299026Sjulian/* 184399026Sjulian * Called in from locations that can safely check to see 184499026Sjulian * whether we have to suspend or at least throttle for a 184599026Sjulian * single-thread event (e.g. fork). 184699026Sjulian * 184799026Sjulian * Such locations include userret(). 184899026Sjulian * If the "return_instead" argument is non zero, the thread must be able to 184999026Sjulian * accept 0 (caller may continue), or 1 (caller must abort) as a result. 185099026Sjulian * 185199026Sjulian * The 'return_instead' argument tells the function if it may do a 185299026Sjulian * thread_exit() or suspend, or whether the caller must abort and back 185399026Sjulian * out instead. 185499026Sjulian * 185599026Sjulian * If the thread that set the single_threading request has set the 185699026Sjulian * P_SINGLE_EXIT bit in the process flags then this call will never return 185799026Sjulian * if 'return_instead' is false, but will exit. 185899026Sjulian * 185999026Sjulian * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 186099026Sjulian *---------------+--------------------+--------------------- 186199026Sjulian * 0 | returns 0 | returns 0 or 1 186299026Sjulian * | when ST ends | immediatly 186399026Sjulian *---------------+--------------------+--------------------- 186499026Sjulian * 1 | thread exits | returns 1 186599026Sjulian * | | immediatly 186699026Sjulian * 0 = thread_exit() or suspension ok, 186799026Sjulian * other = return error instead of stopping the thread. 186899026Sjulian * 186999026Sjulian * While a full suspension is under effect, even a single threading 187099026Sjulian * thread would be suspended if it made this call (but it shouldn't). 187199026Sjulian * This call should only be made from places where 187299026Sjulian * thread_exit() would be safe as that may be the outcome unless 187399026Sjulian * return_instead is set. 187499026Sjulian */ 187599026Sjulianint 187699026Sjulianthread_suspend_check(int return_instead) 187799026Sjulian{ 1878104502Sjmallett struct thread *td; 1879104502Sjmallett struct proc *p; 1880105854Sjulian struct ksegrp *kg; 188199026Sjulian 188299026Sjulian td = curthread; 188399026Sjulian p = td->td_proc; 1884105854Sjulian kg = td->td_ksegrp; 188599026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 188699026Sjulian while (P_SHOULDSTOP(p)) { 1887102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 188899026Sjulian KASSERT(p->p_singlethread != NULL, 188999026Sjulian ("singlethread not set")); 189099026Sjulian /* 1891100648Sjulian * The only suspension in action is a 1892100648Sjulian * single-threading. Single threader need not stop. 1893100646Sjulian * XXX Should be safe to access unlocked 1894100646Sjulian * as it can only be set to be true by us. 189599026Sjulian */ 1896100648Sjulian if (p->p_singlethread == td) 189799026Sjulian return (0); /* Exempt from stopping. */ 189899026Sjulian } 1899100648Sjulian if (return_instead) 190099026Sjulian return (1); 190199026Sjulian 190299026Sjulian /* 190399026Sjulian * If the process is waiting for us to exit, 190499026Sjulian * this thread should just suicide. 1905102950Sdavidxu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 190699026Sjulian */ 190799026Sjulian if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 190899026Sjulian mtx_lock_spin(&sched_lock); 190999026Sjulian while (mtx_owned(&Giant)) 191099026Sjulian mtx_unlock(&Giant); 191199026Sjulian thread_exit(); 191299026Sjulian } 191399026Sjulian 191499026Sjulian /* 191599026Sjulian * When a thread suspends, it just 191699026Sjulian * moves to the processes's suspend queue 191799026Sjulian * and stays there. 191899026Sjulian */ 1919102238Sjulian mtx_lock_spin(&sched_lock); 1920102950Sdavidxu if ((p->p_flag & P_STOPPED_SIG) && 1921102238Sjulian (p->p_suspcount+1 == p->p_numthreads)) { 1922102238Sjulian mtx_unlock_spin(&sched_lock); 1923102238Sjulian PROC_LOCK(p->p_pptr); 1924102238Sjulian if ((p->p_pptr->p_procsig->ps_flag & 1925102238Sjulian PS_NOCLDSTOP) == 0) { 1926102238Sjulian psignal(p->p_pptr, SIGCHLD); 1927102238Sjulian } 1928102238Sjulian PROC_UNLOCK(p->p_pptr); 1929103055Sjulian mtx_lock_spin(&sched_lock); 1930102238Sjulian } 193199026Sjulian mtx_assert(&Giant, MA_NOTOWNED); 1932103216Sjulian thread_suspend_one(td); 193399026Sjulian PROC_UNLOCK(p); 1934102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1935100632Sjulian if (p->p_numthreads == p->p_suspcount) { 1936103216Sjulian thread_unsuspend_one(p->p_singlethread); 1937100632Sjulian } 1938100632Sjulian } 1939100594Sjulian p->p_stats->p_ru.ru_nivcsw++; 194099026Sjulian mi_switch(); 194199026Sjulian mtx_unlock_spin(&sched_lock); 194299026Sjulian PROC_LOCK(p); 194399026Sjulian } 194499026Sjulian return (0); 194599026Sjulian} 194699026Sjulian 1947102898Sdavidxuvoid 1948102898Sdavidxuthread_suspend_one(struct thread *td) 1949102898Sdavidxu{ 1950102898Sdavidxu struct proc *p = td->td_proc; 1951102898Sdavidxu 1952102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 1953102898Sdavidxu p->p_suspcount++; 1954103216Sjulian TD_SET_SUSPENDED(td); 1955102898Sdavidxu TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 1956103216Sjulian /* 1957103216Sjulian * Hack: If we are suspending but are on the sleep queue 1958103216Sjulian * then we are in msleep or the cv equivalent. We 1959103216Sjulian * want to look like we have two Inhibitors. 1960105911Sjulian * May already be set.. doesn't matter. 1961103216Sjulian */ 1962103216Sjulian if (TD_ON_SLEEPQ(td)) 1963103216Sjulian TD_SET_SLEEPING(td); 1964102898Sdavidxu} 1965102898Sdavidxu 1966102898Sdavidxuvoid 1967102898Sdavidxuthread_unsuspend_one(struct thread *td) 1968102898Sdavidxu{ 1969102898Sdavidxu struct proc *p = td->td_proc; 1970102898Sdavidxu 1971102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 1972102898Sdavidxu TAILQ_REMOVE(&p->p_suspended, td, td_runq); 1973103216Sjulian TD_CLR_SUSPENDED(td); 1974102898Sdavidxu p->p_suspcount--; 1975103216Sjulian setrunnable(td); 1976102898Sdavidxu} 1977102898Sdavidxu 197899026Sjulian/* 197999026Sjulian * Allow all threads blocked by single threading to continue running. 198099026Sjulian */ 198199026Sjulianvoid 198299026Sjulianthread_unsuspend(struct proc *p) 198399026Sjulian{ 198499026Sjulian struct thread *td; 198599026Sjulian 1986100646Sjulian mtx_assert(&sched_lock, MA_OWNED); 198799026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 198899026Sjulian if (!P_SHOULDSTOP(p)) { 198999026Sjulian while (( td = TAILQ_FIRST(&p->p_suspended))) { 1990102898Sdavidxu thread_unsuspend_one(td); 199199026Sjulian } 1992102950Sdavidxu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 199399026Sjulian (p->p_numthreads == p->p_suspcount)) { 199499026Sjulian /* 199599026Sjulian * Stopping everything also did the job for the single 199699026Sjulian * threading request. Now we've downgraded to single-threaded, 199799026Sjulian * let it continue. 199899026Sjulian */ 1999102898Sdavidxu thread_unsuspend_one(p->p_singlethread); 200099026Sjulian } 200199026Sjulian} 200299026Sjulian 200399026Sjulianvoid 200499026Sjulianthread_single_end(void) 200599026Sjulian{ 200699026Sjulian struct thread *td; 200799026Sjulian struct proc *p; 200899026Sjulian 200999026Sjulian td = curthread; 201099026Sjulian p = td->td_proc; 201199026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 2012102950Sdavidxu p->p_flag &= ~P_STOPPED_SINGLE; 201399026Sjulian p->p_singlethread = NULL; 2014102292Sjulian /* 2015102292Sjulian * If there are other threads they mey now run, 2016102292Sjulian * unless of course there is a blanket 'stop order' 2017102292Sjulian * on the process. The single threader must be allowed 2018102292Sjulian * to continue however as this is a bad place to stop. 2019102292Sjulian */ 2020102292Sjulian if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2021102292Sjulian mtx_lock_spin(&sched_lock); 2022102292Sjulian while (( td = TAILQ_FIRST(&p->p_suspended))) { 2023103216Sjulian thread_unsuspend_one(td); 2024102292Sjulian } 2025102292Sjulian mtx_unlock_spin(&sched_lock); 2026102292Sjulian } 202799026Sjulian} 202899026Sjulian 2029102292Sjulian 2030