kern_thread.c revision 113795
199026Sjulian/* 299026Sjulian * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 399026Sjulian * All rights reserved. 499026Sjulian * 599026Sjulian * Redistribution and use in source and binary forms, with or without 699026Sjulian * modification, are permitted provided that the following conditions 799026Sjulian * are met: 899026Sjulian * 1. Redistributions of source code must retain the above copyright 999026Sjulian * notice(s), this list of conditions and the following disclaimer as 1099026Sjulian * the first lines of this file unmodified other than the possible 1199026Sjulian * addition of one or more copyright notices. 1299026Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1399026Sjulian * notice(s), this list of conditions and the following disclaimer in the 1499026Sjulian * documentation and/or other materials provided with the distribution. 1599026Sjulian * 1699026Sjulian * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1799026Sjulian * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1899026Sjulian * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1999026Sjulian * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 2099026Sjulian * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2199026Sjulian * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2299026Sjulian * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2399026Sjulian * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2499026Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2599026Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2699026Sjulian * DAMAGE. 2799026Sjulian * 2899026Sjulian * $FreeBSD: head/sys/kern/kern_thread.c 113795 2003-04-21 14:42:04Z davidxu $ 2999026Sjulian */ 3099026Sjulian 3199026Sjulian#include <sys/param.h> 3299026Sjulian#include <sys/systm.h> 3399026Sjulian#include <sys/kernel.h> 3499026Sjulian#include <sys/lock.h> 3599026Sjulian#include <sys/malloc.h> 3699026Sjulian#include <sys/mutex.h> 3799026Sjulian#include <sys/proc.h> 38107029Sjulian#include <sys/smp.h> 3999026Sjulian#include <sys/sysctl.h> 40105854Sjulian#include <sys/sysproto.h> 4199026Sjulian#include <sys/filedesc.h> 42107126Sjeff#include <sys/sched.h> 4399026Sjulian#include <sys/signalvar.h> 4499026Sjulian#include <sys/sx.h> 45107126Sjeff#include <sys/tty.h> 4699026Sjulian#include <sys/user.h> 4799026Sjulian#include <sys/jail.h> 4899026Sjulian#include <sys/kse.h> 4999026Sjulian#include <sys/ktr.h> 50103410Smini#include <sys/ucontext.h> 5199026Sjulian 5299026Sjulian#include <vm/vm.h> 5399026Sjulian#include <vm/vm_object.h> 5499026Sjulian#include <vm/pmap.h> 5599026Sjulian#include <vm/uma.h> 5699026Sjulian#include <vm/vm_map.h> 5799026Sjulian 58100273Speter#include <machine/frame.h> 59100273Speter 6099026Sjulian/* 61103367Sjulian * KSEGRP related storage. 6299026Sjulian */ 63103367Sjulianstatic uma_zone_t ksegrp_zone; 64103367Sjulianstatic uma_zone_t kse_zone; 6599026Sjulianstatic uma_zone_t thread_zone; 66111028Sjeffstatic uma_zone_t upcall_zone; 6799026Sjulian 68103367Sjulian/* DEBUG ONLY */ 6999026SjulianSYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 70107719Sjulianstatic int thread_debug = 0; 71107719SjulianSYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 72107719Sjulian &thread_debug, 0, "thread debug"); 7399026Sjulian 74107006Sdavidxustatic int max_threads_per_proc = 30; 75107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 76103367Sjulian &max_threads_per_proc, 0, "Limit on threads per proc"); 77103367Sjulian 78107006Sdavidxustatic int max_groups_per_proc = 5; 79107006SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 80107006Sdavidxu &max_groups_per_proc, 0, "Limit on thread groups per proc"); 81107006Sdavidxu 82111115Sdavidxustatic int max_threads_hits; 83111115SdavidxuSYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 84111115Sdavidxu &max_threads_hits, 0, ""); 85111115Sdavidxu 86111028Sjeffstatic int virtual_cpu; 87111028Sjeff 8899026Sjulian#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 8999026Sjulian 90111028SjeffTAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 91105854SjulianTAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 92105854SjulianTAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 93111028SjeffTAILQ_HEAD(, kse_upcall) zombie_upcalls = 94111028Sjeff TAILQ_HEAD_INITIALIZER(zombie_upcalls); 95111028Sjeffstruct mtx kse_zombie_lock; 96111028SjeffMTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 9799026Sjulian 98107719Sjulianstatic void kse_purge(struct proc *p, struct thread *td); 99111028Sjeffstatic void kse_purge_group(struct thread *td); 100111515Sdavidxustatic int thread_update_usr_ticks(struct thread *td, int user); 101111028Sjeffstatic void thread_alloc_spare(struct thread *td, struct thread *spare); 102105854Sjulian 103111028Sjeffstatic int 104111028Sjeffsysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 105111028Sjeff{ 106111028Sjeff int error, new_val; 107111028Sjeff int def_val; 108111028Sjeff 109111028Sjeff#ifdef SMP 110111028Sjeff def_val = mp_ncpus; 111111028Sjeff#else 112111028Sjeff def_val = 1; 113111028Sjeff#endif 114111028Sjeff if (virtual_cpu == 0) 115111028Sjeff new_val = def_val; 116111028Sjeff else 117111028Sjeff new_val = virtual_cpu; 118111028Sjeff error = sysctl_handle_int(oidp, &new_val, 0, req); 119111028Sjeff if (error != 0 || req->newptr == NULL) 120111028Sjeff return (error); 121111028Sjeff if (new_val < 0) 122111028Sjeff return (EINVAL); 123111028Sjeff virtual_cpu = new_val; 124111028Sjeff return (0); 125111028Sjeff} 126111028Sjeff 127111028Sjeff/* DEBUG ONLY */ 128111028SjeffSYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 129111028Sjeff 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 130111028Sjeff "debug virtual cpus"); 131111028Sjeff 13299026Sjulian/* 133107719Sjulian * Prepare a thread for use. 13499026Sjulian */ 13599026Sjulianstatic void 13699026Sjulianthread_ctor(void *mem, int size, void *arg) 13799026Sjulian{ 13899026Sjulian struct thread *td; 13999026Sjulian 14099026Sjulian td = (struct thread *)mem; 141103216Sjulian td->td_state = TDS_INACTIVE; 142113339Sjulian td->td_oncpu = NOCPU; 14399026Sjulian} 14499026Sjulian 14599026Sjulian/* 14699026Sjulian * Reclaim a thread after use. 14799026Sjulian */ 14899026Sjulianstatic void 14999026Sjulianthread_dtor(void *mem, int size, void *arg) 15099026Sjulian{ 15199026Sjulian struct thread *td; 15299026Sjulian 15399026Sjulian td = (struct thread *)mem; 15499026Sjulian 15599026Sjulian#ifdef INVARIANTS 15699026Sjulian /* Verify that this thread is in a safe state to free. */ 15799026Sjulian switch (td->td_state) { 158103216Sjulian case TDS_INHIBITED: 159103216Sjulian case TDS_RUNNING: 160103216Sjulian case TDS_CAN_RUN: 16199026Sjulian case TDS_RUNQ: 16299026Sjulian /* 16399026Sjulian * We must never unlink a thread that is in one of 16499026Sjulian * these states, because it is currently active. 16599026Sjulian */ 16699026Sjulian panic("bad state for thread unlinking"); 16799026Sjulian /* NOTREACHED */ 168103216Sjulian case TDS_INACTIVE: 16999026Sjulian break; 17099026Sjulian default: 17199026Sjulian panic("bad thread state"); 17299026Sjulian /* NOTREACHED */ 17399026Sjulian } 17499026Sjulian#endif 17599026Sjulian} 17699026Sjulian 17799026Sjulian/* 17899026Sjulian * Initialize type-stable parts of a thread (when newly created). 17999026Sjulian */ 18099026Sjulianstatic void 18199026Sjulianthread_init(void *mem, int size) 18299026Sjulian{ 18399026Sjulian struct thread *td; 18499026Sjulian 18599026Sjulian td = (struct thread *)mem; 186103312Sjulian mtx_lock(&Giant); 187104354Sscottl pmap_new_thread(td, 0); 188103312Sjulian mtx_unlock(&Giant); 18999026Sjulian cpu_thread_setup(td); 190107126Sjeff td->td_sched = (struct td_sched *)&td[1]; 19199026Sjulian} 19299026Sjulian 19399026Sjulian/* 19499026Sjulian * Tear down type-stable parts of a thread (just before being discarded). 19599026Sjulian */ 19699026Sjulianstatic void 19799026Sjulianthread_fini(void *mem, int size) 19899026Sjulian{ 19999026Sjulian struct thread *td; 20099026Sjulian 20199026Sjulian td = (struct thread *)mem; 20299026Sjulian pmap_dispose_thread(td); 20399026Sjulian} 204111028Sjeff 205107126Sjeff/* 206107126Sjeff * Initialize type-stable parts of a kse (when newly created). 207107126Sjeff */ 208107126Sjeffstatic void 209107126Sjeffkse_init(void *mem, int size) 210107126Sjeff{ 211107126Sjeff struct kse *ke; 21299026Sjulian 213107126Sjeff ke = (struct kse *)mem; 214107126Sjeff ke->ke_sched = (struct ke_sched *)&ke[1]; 215107126Sjeff} 216111028Sjeff 217107126Sjeff/* 218107126Sjeff * Initialize type-stable parts of a ksegrp (when newly created). 219107126Sjeff */ 220107126Sjeffstatic void 221107126Sjeffksegrp_init(void *mem, int size) 222107126Sjeff{ 223107126Sjeff struct ksegrp *kg; 224107126Sjeff 225107126Sjeff kg = (struct ksegrp *)mem; 226107126Sjeff kg->kg_sched = (struct kg_sched *)&kg[1]; 227107126Sjeff} 228107126Sjeff 229105854Sjulian/* 230111028Sjeff * KSE is linked into kse group. 231105854Sjulian */ 232105854Sjulianvoid 233105854Sjuliankse_link(struct kse *ke, struct ksegrp *kg) 234105854Sjulian{ 235105854Sjulian struct proc *p = kg->kg_proc; 236105854Sjulian 237105854Sjulian TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 238105854Sjulian kg->kg_kses++; 239111028Sjeff ke->ke_state = KES_UNQUEUED; 240105854Sjulian ke->ke_proc = p; 241105854Sjulian ke->ke_ksegrp = kg; 242105854Sjulian ke->ke_thread = NULL; 243111028Sjeff ke->ke_oncpu = NOCPU; 244111028Sjeff ke->ke_flags = 0; 245105854Sjulian} 246105854Sjulian 247105854Sjulianvoid 248105854Sjuliankse_unlink(struct kse *ke) 249105854Sjulian{ 250105854Sjulian struct ksegrp *kg; 251105854Sjulian 252105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 253105854Sjulian kg = ke->ke_ksegrp; 254105854Sjulian TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 255111028Sjeff if (ke->ke_state == KES_IDLE) { 256111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 257111028Sjeff kg->kg_idle_kses--; 258105854Sjulian } 259111028Sjeff if (--kg->kg_kses == 0) 260111028Sjeff ksegrp_unlink(kg); 261105854Sjulian /* 262105854Sjulian * Aggregate stats from the KSE 263105854Sjulian */ 264105854Sjulian kse_stash(ke); 265105854Sjulian} 266105854Sjulian 267105854Sjulianvoid 268105854Sjulianksegrp_link(struct ksegrp *kg, struct proc *p) 269105854Sjulian{ 270105854Sjulian 271105854Sjulian TAILQ_INIT(&kg->kg_threads); 272105854Sjulian TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 273105854Sjulian TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 274105854Sjulian TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 275111028Sjeff TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 276111028Sjeff TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 277111028Sjeff kg->kg_proc = p; 278111028Sjeff /* 279111028Sjeff * the following counters are in the -zero- section 280111028Sjeff * and may not need clearing 281111028Sjeff */ 282105854Sjulian kg->kg_numthreads = 0; 283111028Sjeff kg->kg_runnable = 0; 284111028Sjeff kg->kg_kses = 0; 285111028Sjeff kg->kg_runq_kses = 0; /* XXXKSE change name */ 286111028Sjeff kg->kg_idle_kses = 0; 287111028Sjeff kg->kg_numupcalls = 0; 288111028Sjeff /* link it in now that it's consistent */ 289105854Sjulian p->p_numksegrps++; 290105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 291105854Sjulian} 292105854Sjulian 293105854Sjulianvoid 294105854Sjulianksegrp_unlink(struct ksegrp *kg) 295105854Sjulian{ 296105854Sjulian struct proc *p; 297105854Sjulian 298105854Sjulian mtx_assert(&sched_lock, MA_OWNED); 299111028Sjeff KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 300111028Sjeff KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 301111028Sjeff KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 302111028Sjeff 303105854Sjulian p = kg->kg_proc; 304105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 305105854Sjulian p->p_numksegrps--; 306105854Sjulian /* 307105854Sjulian * Aggregate stats from the KSE 308105854Sjulian */ 309105854Sjulian ksegrp_stash(kg); 310105854Sjulian} 311105854Sjulian 312111028Sjeffstruct kse_upcall * 313111028Sjeffupcall_alloc(void) 314111028Sjeff{ 315111028Sjeff struct kse_upcall *ku; 316111028Sjeff 317111125Sdavidxu ku = uma_zalloc(upcall_zone, M_WAITOK); 318111028Sjeff bzero(ku, sizeof(*ku)); 319111028Sjeff return (ku); 320111028Sjeff} 321111028Sjeff 322111028Sjeffvoid 323111028Sjeffupcall_free(struct kse_upcall *ku) 324111028Sjeff{ 325111028Sjeff 326111028Sjeff uma_zfree(upcall_zone, ku); 327111028Sjeff} 328111028Sjeff 329111028Sjeffvoid 330111028Sjeffupcall_link(struct kse_upcall *ku, struct ksegrp *kg) 331111028Sjeff{ 332111028Sjeff 333111028Sjeff mtx_assert(&sched_lock, MA_OWNED); 334111028Sjeff TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 335111028Sjeff ku->ku_ksegrp = kg; 336111028Sjeff kg->kg_numupcalls++; 337111028Sjeff} 338111028Sjeff 339111028Sjeffvoid 340111028Sjeffupcall_unlink(struct kse_upcall *ku) 341111028Sjeff{ 342111028Sjeff struct ksegrp *kg = ku->ku_ksegrp; 343111028Sjeff 344111028Sjeff mtx_assert(&sched_lock, MA_OWNED); 345111028Sjeff KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 346111028Sjeff TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 347111028Sjeff kg->kg_numupcalls--; 348111028Sjeff upcall_stash(ku); 349111028Sjeff} 350111028Sjeff 351111028Sjeffvoid 352111028Sjeffupcall_remove(struct thread *td) 353111028Sjeff{ 354111028Sjeff 355111028Sjeff if (td->td_upcall) { 356111028Sjeff td->td_upcall->ku_owner = NULL; 357111028Sjeff upcall_unlink(td->td_upcall); 358111028Sjeff td->td_upcall = 0; 359111028Sjeff } 360111028Sjeff} 361111028Sjeff 36299026Sjulian/* 363111028Sjeff * For a newly created process, 364111028Sjeff * link up all the structures and its initial threads etc. 365105854Sjulian */ 366105854Sjulianvoid 367105854Sjulianproc_linkup(struct proc *p, struct ksegrp *kg, 368111028Sjeff struct kse *ke, struct thread *td) 369105854Sjulian{ 370105854Sjulian 371105854Sjulian TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 372105854Sjulian TAILQ_INIT(&p->p_threads); /* all threads in proc */ 373105854Sjulian TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 374105854Sjulian p->p_numksegrps = 0; 375105854Sjulian p->p_numthreads = 0; 376105854Sjulian 377105854Sjulian ksegrp_link(kg, p); 378105854Sjulian kse_link(ke, kg); 379105854Sjulian thread_link(td, kg); 380105854Sjulian} 381105854Sjulian 382111028Sjeff/* 383111028Sjeffstruct kse_thr_interrupt_args { 384111028Sjeff struct kse_thr_mailbox * tmbx; 385111028Sjeff}; 386111028Sjeff*/ 387105854Sjulianint 388105854Sjuliankse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 389105854Sjulian{ 390106180Sdavidxu struct proc *p; 391106180Sdavidxu struct thread *td2; 392105854Sjulian 393106242Sdavidxu p = td->td_proc; 394111585Sjulian if (!(p->p_flag & P_THREADED) || (uap->tmbx == NULL)) 395106242Sdavidxu return (EINVAL); 396106180Sdavidxu mtx_lock_spin(&sched_lock); 397106180Sdavidxu FOREACH_THREAD_IN_PROC(p, td2) { 398106180Sdavidxu if (td2->td_mailbox == uap->tmbx) { 399106180Sdavidxu td2->td_flags |= TDF_INTERRUPT; 400106180Sdavidxu if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { 401106180Sdavidxu if (td2->td_flags & TDF_CVWAITQ) 402106180Sdavidxu cv_abort(td2); 403106180Sdavidxu else 404106180Sdavidxu abortsleep(td2); 405111028Sjeff } 406106180Sdavidxu mtx_unlock_spin(&sched_lock); 407106182Sdavidxu return (0); 408106180Sdavidxu } 409106180Sdavidxu } 410106180Sdavidxu mtx_unlock_spin(&sched_lock); 411106182Sdavidxu return (ESRCH); 412105854Sjulian} 413105854Sjulian 414111028Sjeff/* 415111028Sjeffstruct kse_exit_args { 416111028Sjeff register_t dummy; 417111028Sjeff}; 418111028Sjeff*/ 419105854Sjulianint 420105854Sjuliankse_exit(struct thread *td, struct kse_exit_args *uap) 421105854Sjulian{ 422105854Sjulian struct proc *p; 423105854Sjulian struct ksegrp *kg; 424108640Sdavidxu struct kse *ke; 425105854Sjulian 426105854Sjulian p = td->td_proc; 427113793Sdavidxu if (td->td_upcall == NULL || TD_CAN_UNBIND(td)) 428106182Sdavidxu return (EINVAL); 429105854Sjulian kg = td->td_ksegrp; 430111028Sjeff /* Serialize removing upcall */ 431105854Sjulian PROC_LOCK(p); 432105854Sjulian mtx_lock_spin(&sched_lock); 433111028Sjeff if ((kg->kg_numupcalls == 1) && (kg->kg_numthreads > 1)) { 434105854Sjulian mtx_unlock_spin(&sched_lock); 435105854Sjulian PROC_UNLOCK(p); 436105854Sjulian return (EDEADLK); 437105854Sjulian } 438108640Sdavidxu ke = td->td_kse; 439111028Sjeff upcall_remove(td); 440108640Sdavidxu if (p->p_numthreads == 1) { 441111028Sjeff kse_purge(p, td); 442111585Sjulian p->p_flag &= ~P_THREADED; 443105854Sjulian mtx_unlock_spin(&sched_lock); 444105854Sjulian PROC_UNLOCK(p); 445105854Sjulian } else { 446111028Sjeff if (kg->kg_numthreads == 1) { /* Shutdown a group */ 447111028Sjeff kse_purge_group(td); 448111028Sjeff ke->ke_flags |= KEF_EXIT; 449111028Sjeff } 450112071Sdavidxu thread_stopped(p); 451105854Sjulian thread_exit(); 452105854Sjulian /* NOTREACHED */ 453105854Sjulian } 454106182Sdavidxu return (0); 455105854Sjulian} 456105854Sjulian 457107719Sjulian/* 458108338Sjulian * Either becomes an upcall or waits for an awakening event and 459111028Sjeff * then becomes an upcall. Only error cases return. 460107719Sjulian */ 461111028Sjeff/* 462111028Sjeffstruct kse_release_args { 463111169Sdavidxu struct timespec *timeout; 464111028Sjeff}; 465111028Sjeff*/ 466105854Sjulianint 467111028Sjeffkse_release(struct thread *td, struct kse_release_args *uap) 468105854Sjulian{ 469105854Sjulian struct proc *p; 470107719Sjulian struct ksegrp *kg; 471111169Sdavidxu struct timespec ts, ts2, ts3, timeout; 472111169Sdavidxu struct timeval tv; 473111169Sdavidxu int error; 474105854Sjulian 475105854Sjulian p = td->td_proc; 476107719Sjulian kg = td->td_ksegrp; 477113793Sdavidxu if (td->td_upcall == NULL || TD_CAN_UNBIND(td)) 478107719Sjulian return (EINVAL); 479111169Sdavidxu if (uap->timeout != NULL) { 480111169Sdavidxu if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 481111169Sdavidxu return (error); 482111169Sdavidxu getnanouptime(&ts); 483111169Sdavidxu timespecadd(&ts, &timeout); 484111169Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &timeout); 485111169Sdavidxu } 486108613Sjulian mtx_lock_spin(&sched_lock); 487108338Sjulian /* Change OURSELF to become an upcall. */ 488111028Sjeff td->td_flags = TDF_UPCALLING; 489112888Sjeff#if 0 /* XXX This shouldn't be necessary */ 490111042Sdavidxu if (p->p_sflag & PS_NEEDSIGCHK) 491111042Sdavidxu td->td_flags |= TDF_ASTPENDING; 492112888Sjeff#endif 493111169Sdavidxu mtx_unlock_spin(&sched_lock); 494111169Sdavidxu PROC_LOCK(p); 495111169Sdavidxu while ((td->td_upcall->ku_flags & KUF_DOUPCALL) == 0 && 496111169Sdavidxu (kg->kg_completed == NULL)) { 497111028Sjeff kg->kg_upsleeps++; 498111169Sdavidxu error = msleep(&kg->kg_completed, &p->p_mtx, PPAUSE|PCATCH, 499111169Sdavidxu "kse_rel", (uap->timeout ? tvtohz(&tv) : 0)); 500111028Sjeff kg->kg_upsleeps--; 501110190Sjulian PROC_UNLOCK(p); 502111169Sdavidxu if (uap->timeout == NULL || error != EWOULDBLOCK) 503111169Sdavidxu return (0); 504111169Sdavidxu getnanouptime(&ts2); 505111169Sdavidxu if (timespeccmp(&ts2, &ts, >=)) 506111169Sdavidxu return (0); 507111169Sdavidxu ts3 = ts; 508111169Sdavidxu timespecsub(&ts3, &ts2); 509111169Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 510111169Sdavidxu PROC_LOCK(p); 511105854Sjulian } 512111169Sdavidxu PROC_UNLOCK(p); 513107719Sjulian return (0); 514105854Sjulian} 515105854Sjulian 516105854Sjulian/* struct kse_wakeup_args { 517105854Sjulian struct kse_mailbox *mbx; 518105854Sjulian}; */ 519105854Sjulianint 520105854Sjuliankse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 521105854Sjulian{ 522105854Sjulian struct proc *p; 523105854Sjulian struct ksegrp *kg; 524111028Sjeff struct kse_upcall *ku; 525108338Sjulian struct thread *td2; 526105854Sjulian 527105854Sjulian p = td->td_proc; 528108338Sjulian td2 = NULL; 529111028Sjeff ku = NULL; 530105854Sjulian /* KSE-enabled processes only, please. */ 531111585Sjulian if (!(p->p_flag & P_THREADED)) 532111028Sjeff return (EINVAL); 533111028Sjeff PROC_LOCK(p); 534108613Sjulian mtx_lock_spin(&sched_lock); 535105854Sjulian if (uap->mbx) { 536105854Sjulian FOREACH_KSEGRP_IN_PROC(p, kg) { 537111028Sjeff FOREACH_UPCALL_IN_GROUP(kg, ku) { 538111207Sdavidxu if (ku->ku_mailbox == uap->mbx) 539111028Sjeff break; 540108613Sjulian } 541111028Sjeff if (ku) 542108338Sjulian break; 543105854Sjulian } 544105854Sjulian } else { 545105854Sjulian kg = td->td_ksegrp; 546111028Sjeff if (kg->kg_upsleeps) { 547111028Sjeff wakeup_one(&kg->kg_completed); 548111028Sjeff mtx_unlock_spin(&sched_lock); 549111028Sjeff PROC_UNLOCK(p); 550111028Sjeff return (0); 551108338Sjulian } 552111028Sjeff ku = TAILQ_FIRST(&kg->kg_upcalls); 553105854Sjulian } 554111028Sjeff if (ku) { 555111028Sjeff if ((td2 = ku->ku_owner) == NULL) { 556111028Sjeff panic("%s: no owner", __func__); 557111028Sjeff } else if (TD_ON_SLEEPQ(td2) && 558111028Sjeff (td2->td_wchan == &kg->kg_completed)) { 559111028Sjeff abortsleep(td2); 560111028Sjeff } else { 561111028Sjeff ku->ku_flags |= KUF_DOUPCALL; 562108613Sjulian } 563105854Sjulian mtx_unlock_spin(&sched_lock); 564111028Sjeff PROC_UNLOCK(p); 565108338Sjulian return (0); 566108613Sjulian } 567105854Sjulian mtx_unlock_spin(&sched_lock); 568111028Sjeff PROC_UNLOCK(p); 569108338Sjulian return (ESRCH); 570105854Sjulian} 571105854Sjulian 572105854Sjulian/* 573105854Sjulian * No new KSEG: first call: use current KSE, don't schedule an upcall 574111028Sjeff * All other situations, do allocate max new KSEs and schedule an upcall. 575105854Sjulian */ 576105854Sjulian/* struct kse_create_args { 577105854Sjulian struct kse_mailbox *mbx; 578105854Sjulian int newgroup; 579105854Sjulian}; */ 580105854Sjulianint 581105854Sjuliankse_create(struct thread *td, struct kse_create_args *uap) 582105854Sjulian{ 583105854Sjulian struct kse *newke; 584105854Sjulian struct ksegrp *newkg; 585105854Sjulian struct ksegrp *kg; 586105854Sjulian struct proc *p; 587105854Sjulian struct kse_mailbox mbx; 588111028Sjeff struct kse_upcall *newku; 589111028Sjeff int err, ncpus; 590105854Sjulian 591105854Sjulian p = td->td_proc; 592105854Sjulian if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 593105854Sjulian return (err); 594105854Sjulian 595111028Sjeff /* Too bad, why hasn't kernel always a cpu counter !? */ 596111028Sjeff#ifdef SMP 597111028Sjeff ncpus = mp_ncpus; 598111028Sjeff#else 599111028Sjeff ncpus = 1; 600111028Sjeff#endif 601111028Sjeff if (thread_debug && virtual_cpu != 0) 602111028Sjeff ncpus = virtual_cpu; 603111028Sjeff 604111028Sjeff /* Easier to just set it than to test and set */ 605112078Sdavidxu PROC_LOCK(p); 606111585Sjulian p->p_flag |= P_THREADED; 607112078Sdavidxu PROC_UNLOCK(p); 608105854Sjulian kg = td->td_ksegrp; 609105854Sjulian if (uap->newgroup) { 610111028Sjeff /* Have race condition but it is cheap */ 611107006Sdavidxu if (p->p_numksegrps >= max_groups_per_proc) 612107006Sdavidxu return (EPROCLIM); 613105854Sjulian /* 614105854Sjulian * If we want a new KSEGRP it doesn't matter whether 615105854Sjulian * we have already fired up KSE mode before or not. 616111028Sjeff * We put the process in KSE mode and create a new KSEGRP. 617105854Sjulian */ 618105854Sjulian newkg = ksegrp_alloc(); 619105854Sjulian bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 620111028Sjeff kg_startzero, kg_endzero)); 621105854Sjulian bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 622105854Sjulian RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 623111028Sjeff mtx_lock_spin(&sched_lock); 624111028Sjeff if (p->p_numksegrps >= max_groups_per_proc) { 625111028Sjeff mtx_unlock_spin(&sched_lock); 626111677Sdavidxu ksegrp_free(newkg); 627111028Sjeff return (EPROCLIM); 628111028Sjeff } 629111677Sdavidxu ksegrp_link(newkg, p); 630111028Sjeff mtx_unlock_spin(&sched_lock); 631105854Sjulian } else { 632111028Sjeff newkg = kg; 633111028Sjeff } 634111028Sjeff 635111028Sjeff /* 636111028Sjeff * Creating upcalls more than number of physical cpu does 637111028Sjeff * not help performance. 638111028Sjeff */ 639111028Sjeff if (newkg->kg_numupcalls >= ncpus) 640111028Sjeff return (EPROCLIM); 641111028Sjeff 642111028Sjeff if (newkg->kg_numupcalls == 0) { 643111028Sjeff /* 644111028Sjeff * Initialize KSE group, optimized for MP. 645111028Sjeff * Create KSEs as many as physical cpus, this increases 646111028Sjeff * concurrent even if userland is not MP safe and can only run 647111028Sjeff * on single CPU (for early version of libpthread, it is true). 648111028Sjeff * In ideal world, every physical cpu should execute a thread. 649111028Sjeff * If there is enough KSEs, threads in kernel can be 650111028Sjeff * executed parallel on different cpus with full speed, 651111028Sjeff * Concurrent in kernel shouldn't be restricted by number of 652111028Sjeff * upcalls userland provides. 653111028Sjeff * Adding more upcall structures only increases concurrent 654111028Sjeff * in userland. 655111028Sjeff * Highest performance configuration is: 656111028Sjeff * N kses = N upcalls = N phyiscal cpus 657105854Sjulian */ 658111028Sjeff while (newkg->kg_kses < ncpus) { 659105854Sjulian newke = kse_alloc(); 660111028Sjeff bzero(&newke->ke_startzero, RANGEOF(struct kse, 661111028Sjeff ke_startzero, ke_endzero)); 662105854Sjulian#if 0 663111028Sjeff mtx_lock_spin(&sched_lock); 664111028Sjeff bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 665111028Sjeff RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 666111028Sjeff mtx_unlock_spin(&sched_lock); 667105854Sjulian#endif 668111028Sjeff mtx_lock_spin(&sched_lock); 669111028Sjeff kse_link(newke, newkg); 670111028Sjeff /* Add engine */ 671111028Sjeff kse_reassign(newke); 672111028Sjeff mtx_unlock_spin(&sched_lock); 673105854Sjulian } 674111028Sjeff } 675111028Sjeff newku = upcall_alloc(); 676111028Sjeff newku->ku_mailbox = uap->mbx; 677111028Sjeff newku->ku_func = mbx.km_func; 678111028Sjeff bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 679111028Sjeff 680111028Sjeff /* For the first call this may not have been set */ 681111028Sjeff if (td->td_standin == NULL) 682111028Sjeff thread_alloc_spare(td, NULL); 683111028Sjeff 684111028Sjeff mtx_lock_spin(&sched_lock); 685111028Sjeff if (newkg->kg_numupcalls >= ncpus) { 686111595Sdavidxu mtx_unlock_spin(&sched_lock); 687111028Sjeff upcall_free(newku); 688111028Sjeff return (EPROCLIM); 689111028Sjeff } 690111028Sjeff upcall_link(newku, newkg); 691112397Sdavidxu if (mbx.km_quantum) 692112397Sdavidxu newkg->kg_upquantum = max(1, mbx.km_quantum/tick); 693111028Sjeff 694111028Sjeff /* 695111028Sjeff * Each upcall structure has an owner thread, find which 696111028Sjeff * one owns it. 697111028Sjeff */ 698111028Sjeff if (uap->newgroup) { 699111028Sjeff /* 700111028Sjeff * Because new ksegrp hasn't thread, 701111028Sjeff * create an initial upcall thread to own it. 702111028Sjeff */ 703111028Sjeff thread_schedule_upcall(td, newku); 704105854Sjulian } else { 705105854Sjulian /* 706111028Sjeff * If current thread hasn't an upcall structure, 707111028Sjeff * just assign the upcall to it. 708105854Sjulian */ 709111028Sjeff if (td->td_upcall == NULL) { 710111028Sjeff newku->ku_owner = td; 711111028Sjeff td->td_upcall = newku; 712111028Sjeff } else { 713111028Sjeff /* 714111028Sjeff * Create a new upcall thread to own it. 715111028Sjeff */ 716111028Sjeff thread_schedule_upcall(td, newku); 717111028Sjeff } 718105854Sjulian } 719111028Sjeff mtx_unlock_spin(&sched_lock); 720105854Sjulian return (0); 721105854Sjulian} 722105854Sjulian 723105854Sjulian/* 724103410Smini * Fill a ucontext_t with a thread's context information. 725103410Smini * 726103410Smini * This is an analogue to getcontext(3). 727103410Smini */ 728103410Sminivoid 729103410Sminithread_getcontext(struct thread *td, ucontext_t *uc) 730103410Smini{ 731103410Smini 732103464Speter/* 733103464Speter * XXX this is declared in a MD include file, i386/include/ucontext.h but 734103464Speter * is used in MI code. 735103464Speter */ 736103463Speter#ifdef __i386__ 737103410Smini get_mcontext(td, &uc->uc_mcontext); 738103463Speter#endif 739113626Sjhb PROC_LOCK(td->td_proc); 740112888Sjeff uc->uc_sigmask = td->td_sigmask; 741113626Sjhb PROC_UNLOCK(td->td_proc); 742103410Smini} 743103410Smini 744103410Smini/* 745103410Smini * Set a thread's context from a ucontext_t. 746103410Smini * 747103410Smini * This is an analogue to setcontext(3). 748103410Smini */ 749103410Sminiint 750103410Sminithread_setcontext(struct thread *td, ucontext_t *uc) 751103410Smini{ 752103410Smini int ret; 753103410Smini 754103464Speter/* 755103464Speter * XXX this is declared in a MD include file, i386/include/ucontext.h but 756103464Speter * is used in MI code. 757103464Speter */ 758103463Speter#ifdef __i386__ 759103410Smini ret = set_mcontext(td, &uc->uc_mcontext); 760103463Speter#else 761103463Speter ret = ENOSYS; 762103463Speter#endif 763103410Smini if (ret == 0) { 764103410Smini SIG_CANTMASK(uc->uc_sigmask); 765103410Smini PROC_LOCK(td->td_proc); 766112888Sjeff td->td_sigmask = uc->uc_sigmask; 767103410Smini PROC_UNLOCK(td->td_proc); 768103410Smini } 769103410Smini return (ret); 770103410Smini} 771103410Smini 772103410Smini/* 77399026Sjulian * Initialize global thread allocation resources. 77499026Sjulian */ 77599026Sjulianvoid 77699026Sjulianthreadinit(void) 77799026Sjulian{ 77899026Sjulian 779104437Speter#ifndef __ia64__ 780107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 78199026Sjulian thread_ctor, thread_dtor, thread_init, thread_fini, 78299026Sjulian UMA_ALIGN_CACHE, 0); 783104437Speter#else 784104437Speter /* 785104437Speter * XXX the ia64 kstack allocator is really lame and is at the mercy 786104437Speter * of contigmallloc(). This hackery is to pre-construct a whole 787104437Speter * pile of thread structures with associated kernel stacks early 788104437Speter * in the system startup while contigmalloc() still works. Once we 789104437Speter * have them, keep them. Sigh. 790104437Speter */ 791107126Sjeff thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 792104437Speter thread_ctor, thread_dtor, thread_init, thread_fini, 793104437Speter UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); 794104437Speter uma_prealloc(thread_zone, 512); /* XXX arbitary */ 795104437Speter#endif 796107126Sjeff ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 797107126Sjeff NULL, NULL, ksegrp_init, NULL, 798103367Sjulian UMA_ALIGN_CACHE, 0); 799107126Sjeff kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 800107126Sjeff NULL, NULL, kse_init, NULL, 801103367Sjulian UMA_ALIGN_CACHE, 0); 802111028Sjeff upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 803111028Sjeff NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 80499026Sjulian} 80599026Sjulian 80699026Sjulian/* 807103002Sjulian * Stash an embarasingly extra thread into the zombie thread queue. 80899026Sjulian */ 80999026Sjulianvoid 81099026Sjulianthread_stash(struct thread *td) 81199026Sjulian{ 812111028Sjeff mtx_lock_spin(&kse_zombie_lock); 81399026Sjulian TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 814111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 81599026Sjulian} 81699026Sjulian 817103410Smini/* 818105854Sjulian * Stash an embarasingly extra kse into the zombie kse queue. 819105854Sjulian */ 820105854Sjulianvoid 821105854Sjuliankse_stash(struct kse *ke) 822105854Sjulian{ 823111028Sjeff mtx_lock_spin(&kse_zombie_lock); 824105854Sjulian TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 825111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 826105854Sjulian} 827105854Sjulian 828105854Sjulian/* 829111028Sjeff * Stash an embarasingly extra upcall into the zombie upcall queue. 830111028Sjeff */ 831111028Sjeff 832111028Sjeffvoid 833111028Sjeffupcall_stash(struct kse_upcall *ku) 834111028Sjeff{ 835111028Sjeff mtx_lock_spin(&kse_zombie_lock); 836111028Sjeff TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 837111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 838111028Sjeff} 839111028Sjeff 840111028Sjeff/* 841105854Sjulian * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 842105854Sjulian */ 843105854Sjulianvoid 844105854Sjulianksegrp_stash(struct ksegrp *kg) 845105854Sjulian{ 846111028Sjeff mtx_lock_spin(&kse_zombie_lock); 847105854Sjulian TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 848111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 849105854Sjulian} 850105854Sjulian 851105854Sjulian/* 852111028Sjeff * Reap zombie kse resource. 85399026Sjulian */ 85499026Sjulianvoid 85599026Sjulianthread_reap(void) 85699026Sjulian{ 857105854Sjulian struct thread *td_first, *td_next; 858105854Sjulian struct kse *ke_first, *ke_next; 859105854Sjulian struct ksegrp *kg_first, * kg_next; 860111028Sjeff struct kse_upcall *ku_first, *ku_next; 86199026Sjulian 86299026Sjulian /* 863111028Sjeff * Don't even bother to lock if none at this instant, 864111028Sjeff * we really don't care about the next instant.. 86599026Sjulian */ 866105854Sjulian if ((!TAILQ_EMPTY(&zombie_threads)) 867105854Sjulian || (!TAILQ_EMPTY(&zombie_kses)) 868111028Sjeff || (!TAILQ_EMPTY(&zombie_ksegrps)) 869111028Sjeff || (!TAILQ_EMPTY(&zombie_upcalls))) { 870111028Sjeff mtx_lock_spin(&kse_zombie_lock); 871105854Sjulian td_first = TAILQ_FIRST(&zombie_threads); 872105854Sjulian ke_first = TAILQ_FIRST(&zombie_kses); 873105854Sjulian kg_first = TAILQ_FIRST(&zombie_ksegrps); 874111028Sjeff ku_first = TAILQ_FIRST(&zombie_upcalls); 875105854Sjulian if (td_first) 876105854Sjulian TAILQ_INIT(&zombie_threads); 877105854Sjulian if (ke_first) 878105854Sjulian TAILQ_INIT(&zombie_kses); 879105854Sjulian if (kg_first) 880105854Sjulian TAILQ_INIT(&zombie_ksegrps); 881111028Sjeff if (ku_first) 882111028Sjeff TAILQ_INIT(&zombie_upcalls); 883111028Sjeff mtx_unlock_spin(&kse_zombie_lock); 884105854Sjulian while (td_first) { 885105854Sjulian td_next = TAILQ_NEXT(td_first, td_runq); 886111028Sjeff if (td_first->td_ucred) 887111028Sjeff crfree(td_first->td_ucred); 888105854Sjulian thread_free(td_first); 889105854Sjulian td_first = td_next; 89099026Sjulian } 891105854Sjulian while (ke_first) { 892105854Sjulian ke_next = TAILQ_NEXT(ke_first, ke_procq); 893105854Sjulian kse_free(ke_first); 894105854Sjulian ke_first = ke_next; 895105854Sjulian } 896105854Sjulian while (kg_first) { 897105854Sjulian kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 898105854Sjulian ksegrp_free(kg_first); 899105854Sjulian kg_first = kg_next; 900105854Sjulian } 901111028Sjeff while (ku_first) { 902111028Sjeff ku_next = TAILQ_NEXT(ku_first, ku_link); 903111028Sjeff upcall_free(ku_first); 904111028Sjeff ku_first = ku_next; 905111028Sjeff } 90699026Sjulian } 90799026Sjulian} 90899026Sjulian 90999026Sjulian/* 910103367Sjulian * Allocate a ksegrp. 911103367Sjulian */ 912103367Sjulianstruct ksegrp * 913103367Sjulianksegrp_alloc(void) 914103367Sjulian{ 915111119Simp return (uma_zalloc(ksegrp_zone, M_WAITOK)); 916103367Sjulian} 917103367Sjulian 918103367Sjulian/* 919103367Sjulian * Allocate a kse. 920103367Sjulian */ 921103367Sjulianstruct kse * 922103367Sjuliankse_alloc(void) 923103367Sjulian{ 924111119Simp return (uma_zalloc(kse_zone, M_WAITOK)); 925103367Sjulian} 926103367Sjulian 927103367Sjulian/* 92899026Sjulian * Allocate a thread. 92999026Sjulian */ 93099026Sjulianstruct thread * 93199026Sjulianthread_alloc(void) 93299026Sjulian{ 93399026Sjulian thread_reap(); /* check if any zombies to get */ 934111119Simp return (uma_zalloc(thread_zone, M_WAITOK)); 93599026Sjulian} 93699026Sjulian 93799026Sjulian/* 938103367Sjulian * Deallocate a ksegrp. 939103367Sjulian */ 940103367Sjulianvoid 941103367Sjulianksegrp_free(struct ksegrp *td) 942103367Sjulian{ 943103367Sjulian uma_zfree(ksegrp_zone, td); 944103367Sjulian} 945103367Sjulian 946103367Sjulian/* 947103367Sjulian * Deallocate a kse. 948103367Sjulian */ 949103367Sjulianvoid 950103367Sjuliankse_free(struct kse *td) 951103367Sjulian{ 952103367Sjulian uma_zfree(kse_zone, td); 953103367Sjulian} 954103367Sjulian 955103367Sjulian/* 95699026Sjulian * Deallocate a thread. 95799026Sjulian */ 95899026Sjulianvoid 95999026Sjulianthread_free(struct thread *td) 96099026Sjulian{ 961107719Sjulian 962107719Sjulian cpu_thread_clean(td); 96399026Sjulian uma_zfree(thread_zone, td); 96499026Sjulian} 96599026Sjulian 96699026Sjulian/* 96799026Sjulian * Store the thread context in the UTS's mailbox. 968104031Sjulian * then add the mailbox at the head of a list we are building in user space. 969104031Sjulian * The list is anchored in the ksegrp structure. 97099026Sjulian */ 97199026Sjulianint 97299026Sjulianthread_export_context(struct thread *td) 97399026Sjulian{ 974104503Sjmallett struct proc *p; 975104031Sjulian struct ksegrp *kg; 976104031Sjulian uintptr_t mbx; 977104031Sjulian void *addr; 978111028Sjeff int error,temp; 979103410Smini ucontext_t uc; 98099026Sjulian 981104503Sjmallett p = td->td_proc; 982104503Sjmallett kg = td->td_ksegrp; 983104503Sjmallett 984104031Sjulian /* Export the user/machine context. */ 985111028Sjeff addr = (void *)(&td->td_mailbox->tm_context); 986104031Sjulian error = copyin(addr, &uc, sizeof(ucontext_t)); 987108338Sjulian if (error) 988108338Sjulian goto bad; 989104031Sjulian 990108338Sjulian thread_getcontext(td, &uc); 991108338Sjulian error = copyout(&uc, addr, sizeof(ucontext_t)); 992108338Sjulian if (error) 993108338Sjulian goto bad; 994108338Sjulian 995111028Sjeff /* Exports clock ticks in kernel mode */ 996111028Sjeff addr = (caddr_t)(&td->td_mailbox->tm_sticks); 997111028Sjeff temp = fuword(addr) + td->td_usticks; 998111028Sjeff if (suword(addr, temp)) 999111028Sjeff goto bad; 1000111028Sjeff 1001111028Sjeff /* Get address in latest mbox of list pointer */ 1002104031Sjulian addr = (void *)(&td->td_mailbox->tm_next); 1003104031Sjulian /* 1004104031Sjulian * Put the saved address of the previous first 1005104031Sjulian * entry into this one 1006104031Sjulian */ 1007104031Sjulian for (;;) { 1008104031Sjulian mbx = (uintptr_t)kg->kg_completed; 1009104031Sjulian if (suword(addr, mbx)) { 1010108338Sjulian error = EFAULT; 1011107034Sdavidxu goto bad; 1012104031Sjulian } 1013104126Sjulian PROC_LOCK(p); 1014104031Sjulian if (mbx == (uintptr_t)kg->kg_completed) { 1015104031Sjulian kg->kg_completed = td->td_mailbox; 1016111028Sjeff /* 1017111028Sjeff * The thread context may be taken away by 1018111028Sjeff * other upcall threads when we unlock 1019111028Sjeff * process lock. it's no longer valid to 1020111028Sjeff * use it again in any other places. 1021111028Sjeff */ 1022111028Sjeff td->td_mailbox = NULL; 1023104126Sjulian PROC_UNLOCK(p); 1024104031Sjulian break; 1025104031Sjulian } 1026104126Sjulian PROC_UNLOCK(p); 1027104031Sjulian } 1028111028Sjeff td->td_usticks = 0; 1029104031Sjulian return (0); 1030107034Sdavidxu 1031107034Sdavidxubad: 1032107034Sdavidxu PROC_LOCK(p); 1033107034Sdavidxu psignal(p, SIGSEGV); 1034107034Sdavidxu PROC_UNLOCK(p); 1035111028Sjeff /* The mailbox is bad, don't use it */ 1036111028Sjeff td->td_mailbox = NULL; 1037111028Sjeff td->td_usticks = 0; 1038108338Sjulian return (error); 1039104031Sjulian} 104099026Sjulian 1041104031Sjulian/* 1042104031Sjulian * Take the list of completed mailboxes for this KSEGRP and put them on this 1043111028Sjeff * upcall's mailbox as it's the next one going up. 1044104031Sjulian */ 1045104031Sjulianstatic int 1046111028Sjeffthread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1047104031Sjulian{ 1048104126Sjulian struct proc *p = kg->kg_proc; 1049104031Sjulian void *addr; 1050104031Sjulian uintptr_t mbx; 1051104031Sjulian 1052111028Sjeff addr = (void *)(&ku->ku_mailbox->km_completed); 1053104031Sjulian for (;;) { 1054104031Sjulian mbx = (uintptr_t)kg->kg_completed; 1055104031Sjulian if (suword(addr, mbx)) { 1056104126Sjulian PROC_LOCK(p); 1057104126Sjulian psignal(p, SIGSEGV); 1058104126Sjulian PROC_UNLOCK(p); 1059104031Sjulian return (EFAULT); 1060104031Sjulian } 1061104126Sjulian PROC_LOCK(p); 1062104031Sjulian if (mbx == (uintptr_t)kg->kg_completed) { 1063104031Sjulian kg->kg_completed = NULL; 1064104126Sjulian PROC_UNLOCK(p); 1065104031Sjulian break; 1066104031Sjulian } 1067104126Sjulian PROC_UNLOCK(p); 106899026Sjulian } 1069104031Sjulian return (0); 107099026Sjulian} 107199026Sjulian 107299026Sjulian/* 1073107034Sdavidxu * This function should be called at statclock interrupt time 1074107034Sdavidxu */ 1075107034Sdavidxuint 1076111028Sjeffthread_statclock(int user) 1077107034Sdavidxu{ 1078107034Sdavidxu struct thread *td = curthread; 1079107034Sdavidxu 1080111028Sjeff if (td->td_ksegrp->kg_numupcalls == 0) 1081111028Sjeff return (-1); 1082107034Sdavidxu if (user) { 1083107034Sdavidxu /* Current always do via ast() */ 1084111976Sdavidxu mtx_lock_spin(&sched_lock); 1085111032Sjulian td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1086111976Sdavidxu mtx_unlock_spin(&sched_lock); 1087111028Sjeff td->td_uuticks++; 1088107034Sdavidxu } else { 1089107034Sdavidxu if (td->td_mailbox != NULL) 1090111028Sjeff td->td_usticks++; 1091111028Sjeff else { 1092111028Sjeff /* XXXKSE 1093111028Sjeff * We will call thread_user_enter() for every 1094111028Sjeff * kernel entry in future, so if the thread mailbox 1095111028Sjeff * is NULL, it must be a UTS kernel, don't account 1096111028Sjeff * clock ticks for it. 1097111028Sjeff */ 1098111028Sjeff } 1099107034Sdavidxu } 1100111028Sjeff return (0); 1101107034Sdavidxu} 1102107034Sdavidxu 1103111028Sjeff/* 1104111515Sdavidxu * Export state clock ticks for userland 1105111028Sjeff */ 1106107034Sdavidxustatic int 1107111515Sdavidxuthread_update_usr_ticks(struct thread *td, int user) 1108107034Sdavidxu{ 1109107034Sdavidxu struct proc *p = td->td_proc; 1110107034Sdavidxu struct kse_thr_mailbox *tmbx; 1111111028Sjeff struct kse_upcall *ku; 1112112397Sdavidxu struct ksegrp *kg; 1113107034Sdavidxu caddr_t addr; 1114111028Sjeff uint uticks; 1115107034Sdavidxu 1116111028Sjeff if ((ku = td->td_upcall) == NULL) 1117111028Sjeff return (-1); 1118111028Sjeff 1119111028Sjeff tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1120107034Sdavidxu if ((tmbx == NULL) || (tmbx == (void *)-1)) 1121111028Sjeff return (-1); 1122111515Sdavidxu if (user) { 1123111515Sdavidxu uticks = td->td_uuticks; 1124111515Sdavidxu td->td_uuticks = 0; 1125111515Sdavidxu addr = (caddr_t)&tmbx->tm_uticks; 1126111515Sdavidxu } else { 1127111515Sdavidxu uticks = td->td_usticks; 1128111515Sdavidxu td->td_usticks = 0; 1129111515Sdavidxu addr = (caddr_t)&tmbx->tm_sticks; 1130111515Sdavidxu } 1131107034Sdavidxu if (uticks) { 1132111515Sdavidxu if (suword(addr, uticks+fuword(addr))) { 1133111028Sjeff PROC_LOCK(p); 1134111028Sjeff psignal(p, SIGSEGV); 1135111028Sjeff PROC_UNLOCK(p); 1136111028Sjeff return (-2); 1137111028Sjeff } 1138107034Sdavidxu } 1139112397Sdavidxu kg = td->td_ksegrp; 1140112397Sdavidxu if (kg->kg_upquantum && ticks >= kg->kg_nextupcall) { 1141112397Sdavidxu mtx_lock_spin(&sched_lock); 1142112397Sdavidxu td->td_upcall->ku_flags |= KUF_DOUPCALL; 1143112397Sdavidxu mtx_unlock_spin(&sched_lock); 1144112397Sdavidxu } 1145111028Sjeff return (0); 1146111028Sjeff} 1147111028Sjeff 1148111028Sjeff/* 114999026Sjulian * Discard the current thread and exit from its context. 115099026Sjulian * 115199026Sjulian * Because we can't free a thread while we're operating under its context, 1152107719Sjulian * push the current thread into our CPU's deadthread holder. This means 1153107719Sjulian * we needn't worry about someone else grabbing our context before we 1154107719Sjulian * do a cpu_throw(). 115599026Sjulian */ 115699026Sjulianvoid 115799026Sjulianthread_exit(void) 115899026Sjulian{ 115999026Sjulian struct thread *td; 116099026Sjulian struct kse *ke; 116199026Sjulian struct proc *p; 116299026Sjulian struct ksegrp *kg; 116399026Sjulian 116499026Sjulian td = curthread; 116599026Sjulian kg = td->td_ksegrp; 116699026Sjulian p = td->td_proc; 116799026Sjulian ke = td->td_kse; 116899026Sjulian 116999026Sjulian mtx_assert(&sched_lock, MA_OWNED); 1170102581Sjulian KASSERT(p != NULL, ("thread exiting without a process")); 1171102581Sjulian KASSERT(ke != NULL, ("thread exiting without a kse")); 1172102581Sjulian KASSERT(kg != NULL, ("thread exiting without a kse group")); 117399026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 117499026Sjulian CTR1(KTR_PROC, "thread_exit: thread %p", td); 117599026Sjulian KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 117699026Sjulian 1177104695Sjulian if (td->td_standin != NULL) { 1178104695Sjulian thread_stash(td->td_standin); 1179104695Sjulian td->td_standin = NULL; 1180104695Sjulian } 1181104695Sjulian 118299026Sjulian cpu_thread_exit(td); /* XXXSMP */ 118399026Sjulian 1184102581Sjulian /* 1185103002Sjulian * The last thread is left attached to the process 1186103002Sjulian * So that the whole bundle gets recycled. Skip 1187103002Sjulian * all this stuff. 1188102581Sjulian */ 1189103002Sjulian if (p->p_numthreads > 1) { 1190113641Sjulian thread_unlink(td); 1191111115Sdavidxu if (p->p_maxthrwaits) 1192111115Sdavidxu wakeup(&p->p_numthreads); 1193103002Sjulian /* 1194103002Sjulian * The test below is NOT true if we are the 1195103002Sjulian * sole exiting thread. P_STOPPED_SNGL is unset 1196103002Sjulian * in exit1() after it is the only survivor. 1197103002Sjulian */ 1198103002Sjulian if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1199103002Sjulian if (p->p_numthreads == p->p_suspcount) { 1200103216Sjulian thread_unsuspend_one(p->p_singlethread); 1201103002Sjulian } 120299026Sjulian } 1203104695Sjulian 1204111028Sjeff /* 1205111028Sjeff * Because each upcall structure has an owner thread, 1206111028Sjeff * owner thread exits only when process is in exiting 1207111028Sjeff * state, so upcall to userland is no longer needed, 1208111028Sjeff * deleting upcall structure is safe here. 1209111028Sjeff * So when all threads in a group is exited, all upcalls 1210111028Sjeff * in the group should be automatically freed. 1211111028Sjeff */ 1212111028Sjeff if (td->td_upcall) 1213111028Sjeff upcall_remove(td); 1214111028Sjeff 1215104695Sjulian ke->ke_state = KES_UNQUEUED; 1216111028Sjeff ke->ke_thread = NULL; 1217104695Sjulian /* 1218108338Sjulian * Decide what to do with the KSE attached to this thread. 1219104695Sjulian */ 1220111028Sjeff if (ke->ke_flags & KEF_EXIT) 1221105854Sjulian kse_unlink(ke); 1222111028Sjeff else 1223105854Sjulian kse_reassign(ke); 1224105854Sjulian PROC_UNLOCK(p); 1225111028Sjeff td->td_kse = NULL; 1226105854Sjulian td->td_state = TDS_INACTIVE; 1227113244Sdavidxu#if 0 1228105854Sjulian td->td_proc = NULL; 1229113244Sdavidxu#endif 1230105854Sjulian td->td_ksegrp = NULL; 1231105854Sjulian td->td_last_kse = NULL; 1232107719Sjulian PCPU_SET(deadthread, td); 1233103002Sjulian } else { 1234103002Sjulian PROC_UNLOCK(p); 123599026Sjulian } 1236112888Sjeff /* XXX Shouldn't cpu_throw() here. */ 1237112993Speter mtx_assert(&sched_lock, MA_OWNED); 1238112993Speter#if defined(__i386__) || defined(__sparc64__) 1239112993Speter cpu_throw(td, choosethread()); 1240112993Speter#else 124199026Sjulian cpu_throw(); 1242112993Speter#endif 1243112993Speter panic("I'm a teapot!"); 124499026Sjulian /* NOTREACHED */ 124599026Sjulian} 124699026Sjulian 1247107719Sjulian/* 1248107719Sjulian * Do any thread specific cleanups that may be needed in wait() 1249107719Sjulian * called with Giant held, proc and schedlock not held. 1250107719Sjulian */ 1251107719Sjulianvoid 1252107719Sjulianthread_wait(struct proc *p) 1253107719Sjulian{ 1254107719Sjulian struct thread *td; 1255107719Sjulian 1256107719Sjulian KASSERT((p->p_numthreads == 1), ("Muliple threads in wait1()")); 1257107719Sjulian KASSERT((p->p_numksegrps == 1), ("Muliple ksegrps in wait1()")); 1258107719Sjulian FOREACH_THREAD_IN_PROC(p, td) { 1259107719Sjulian if (td->td_standin != NULL) { 1260107719Sjulian thread_free(td->td_standin); 1261107719Sjulian td->td_standin = NULL; 1262107719Sjulian } 1263107719Sjulian cpu_thread_clean(td); 1264107719Sjulian } 1265107719Sjulian thread_reap(); /* check for zombie threads etc. */ 1266107719Sjulian} 1267107719Sjulian 126899026Sjulian/* 126999026Sjulian * Link a thread to a process. 1270103002Sjulian * set up anything that needs to be initialized for it to 1271103002Sjulian * be used by the process. 127299026Sjulian * 127399026Sjulian * Note that we do not link to the proc's ucred here. 127499026Sjulian * The thread is linked as if running but no KSE assigned. 127599026Sjulian */ 127699026Sjulianvoid 127799026Sjulianthread_link(struct thread *td, struct ksegrp *kg) 127899026Sjulian{ 127999026Sjulian struct proc *p; 128099026Sjulian 128199026Sjulian p = kg->kg_proc; 1282111028Sjeff td->td_state = TDS_INACTIVE; 1283111028Sjeff td->td_proc = p; 1284111028Sjeff td->td_ksegrp = kg; 1285111028Sjeff td->td_last_kse = NULL; 1286111028Sjeff td->td_flags = 0; 1287111028Sjeff td->td_kse = NULL; 128899026Sjulian 1289103002Sjulian LIST_INIT(&td->td_contested); 1290103002Sjulian callout_init(&td->td_slpcallout, 1); 129199026Sjulian TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 129299026Sjulian TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 129399026Sjulian p->p_numthreads++; 129499026Sjulian kg->kg_numthreads++; 129599026Sjulian} 129699026Sjulian 1297113641Sjulianvoid 1298113641Sjulianthread_unlink(struct thread *td) 1299113641Sjulian{ 1300113641Sjulian struct proc *p = td->td_proc; 1301113641Sjulian struct ksegrp *kg = td->td_ksegrp; 1302113641Sjulian 1303113641Sjulian TAILQ_REMOVE(&p->p_threads, td, td_plist); 1304113641Sjulian p->p_numthreads--; 1305113641Sjulian TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1306113641Sjulian kg->kg_numthreads--; 1307113641Sjulian /* could clear a few other things here */ 1308113641Sjulian} 1309113641Sjulian 1310111028Sjeff/* 1311111028Sjeff * Purge a ksegrp resource. When a ksegrp is preparing to 1312111028Sjeff * exit, it calls this function. 1313111028Sjeff */ 1314105854Sjulianvoid 1315111028Sjeffkse_purge_group(struct thread *td) 1316111028Sjeff{ 1317111028Sjeff struct ksegrp *kg; 1318111028Sjeff struct kse *ke; 1319111028Sjeff 1320111028Sjeff kg = td->td_ksegrp; 1321111028Sjeff KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1322111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1323111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 1324111028Sjeff ("%s: wrong idle KSE state", __func__)); 1325111028Sjeff kse_unlink(ke); 1326111028Sjeff } 1327111028Sjeff KASSERT((kg->kg_kses == 1), 1328111028Sjeff ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1329111028Sjeff KASSERT((kg->kg_numupcalls == 0), 1330111028Sjeff ("%s: ksegrp still has %d upcall datas", 1331111028Sjeff __func__, kg->kg_numupcalls)); 1332111028Sjeff} 1333111028Sjeff 1334111028Sjeff/* 1335111028Sjeff * Purge a process's KSE resource. When a process is preparing to 1336111028Sjeff * exit, it calls kse_purge to release any extra KSE resources in 1337111028Sjeff * the process. 1338111028Sjeff */ 1339111028Sjeffvoid 1340105854Sjuliankse_purge(struct proc *p, struct thread *td) 1341105854Sjulian{ 1342105854Sjulian struct ksegrp *kg; 1343111028Sjeff struct kse *ke; 1344105854Sjulian 1345105854Sjulian KASSERT(p->p_numthreads == 1, ("bad thread number")); 1346105854Sjulian mtx_lock_spin(&sched_lock); 1347105854Sjulian while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1348105854Sjulian TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1349105854Sjulian p->p_numksegrps--; 1350111028Sjeff /* 1351111028Sjeff * There is no ownership for KSE, after all threads 1352111028Sjeff * in the group exited, it is possible that some KSEs 1353111028Sjeff * were left in idle queue, gc them now. 1354111028Sjeff */ 1355111028Sjeff while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1356111028Sjeff KASSERT(ke->ke_state == KES_IDLE, 1357111028Sjeff ("%s: wrong idle KSE state", __func__)); 1358111028Sjeff TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1359111028Sjeff kg->kg_idle_kses--; 1360111028Sjeff TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1361111028Sjeff kg->kg_kses--; 1362111028Sjeff kse_stash(ke); 1363111028Sjeff } 1364105854Sjulian KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1365111028Sjeff ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1366111028Sjeff ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1367111028Sjeff KASSERT((kg->kg_numupcalls == 0), 1368111028Sjeff ("%s: ksegrp still has %d upcall datas", 1369111028Sjeff __func__, kg->kg_numupcalls)); 1370111028Sjeff 1371111028Sjeff if (kg != td->td_ksegrp) 1372105854Sjulian ksegrp_stash(kg); 1373105854Sjulian } 1374105854Sjulian TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1375105854Sjulian p->p_numksegrps++; 1376105854Sjulian mtx_unlock_spin(&sched_lock); 1377105854Sjulian} 1378105854Sjulian 1379111028Sjeff/* 1380111028Sjeff * This function is intended to be used to initialize a spare thread 1381111028Sjeff * for upcall. Initialize thread's large data area outside sched_lock 1382111028Sjeff * for thread_schedule_upcall(). 1383111028Sjeff */ 1384111028Sjeffvoid 1385111028Sjeffthread_alloc_spare(struct thread *td, struct thread *spare) 1386111028Sjeff{ 1387111028Sjeff if (td->td_standin) 1388111028Sjeff return; 1389111028Sjeff if (spare == NULL) 1390111028Sjeff spare = thread_alloc(); 1391111028Sjeff td->td_standin = spare; 1392111028Sjeff bzero(&spare->td_startzero, 1393111028Sjeff (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1394111028Sjeff spare->td_proc = td->td_proc; 1395111028Sjeff spare->td_ucred = crhold(td->td_ucred); 1396111028Sjeff} 1397105854Sjulian 139899026Sjulian/* 1399103410Smini * Create a thread and schedule it for upcall on the KSE given. 1400108338Sjulian * Use our thread's standin so that we don't have to allocate one. 140199026Sjulian */ 140299026Sjulianstruct thread * 1403111028Sjeffthread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 140499026Sjulian{ 140599026Sjulian struct thread *td2; 140699026Sjulian 140799026Sjulian mtx_assert(&sched_lock, MA_OWNED); 1408104695Sjulian 1409104695Sjulian /* 1410111028Sjeff * Schedule an upcall thread on specified kse_upcall, 1411111028Sjeff * the kse_upcall must be free. 1412111028Sjeff * td must have a spare thread. 1413104695Sjulian */ 1414111028Sjeff KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1415104695Sjulian if ((td2 = td->td_standin) != NULL) { 1416104695Sjulian td->td_standin = NULL; 141799026Sjulian } else { 1418111028Sjeff panic("no reserve thread when scheduling an upcall"); 1419106182Sdavidxu return (NULL); 142099026Sjulian } 142199026Sjulian CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1422104695Sjulian td2, td->td_proc->p_pid, td->td_proc->p_comm); 1423103002Sjulian bcopy(&td->td_startcopy, &td2->td_startcopy, 1424103002Sjulian (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1425111028Sjeff thread_link(td2, ku->ku_ksegrp); 1426113244Sdavidxu /* inherit blocked thread's context */ 1427113244Sdavidxu bcopy(td->td_frame, td2->td_frame, sizeof(struct trapframe)); 1428113244Sdavidxu cpu_set_upcall(td2, td->td_pcb); 1429111028Sjeff /* Let the new thread become owner of the upcall */ 1430111028Sjeff ku->ku_owner = td2; 1431111028Sjeff td2->td_upcall = ku; 1432111028Sjeff td2->td_flags = TDF_UPCALLING; 1433112888Sjeff#if 0 /* XXX This shouldn't be necessary */ 1434111041Sdavidxu if (td->td_proc->p_sflag & PS_NEEDSIGCHK) 1435111041Sdavidxu td2->td_flags |= TDF_ASTPENDING; 1436112888Sjeff#endif 1437111028Sjeff td2->td_kse = NULL; 1438111028Sjeff td2->td_state = TDS_CAN_RUN; 1439104695Sjulian td2->td_inhibitors = 0; 1440111028Sjeff setrunqueue(td2); 1441104695Sjulian return (td2); /* bogus.. should be a void function */ 144299026Sjulian} 144399026Sjulian 1444111033Sjeffvoid 1445111033Sjeffthread_signal_add(struct thread *td, int sig) 1446103410Smini{ 1447111033Sjeff struct kse_upcall *ku; 1448111033Sjeff struct proc *p; 1449103410Smini sigset_t ss; 1450103410Smini int error; 1451103410Smini 1452111033Sjeff PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); 1453111033Sjeff td = curthread; 1454111033Sjeff ku = td->td_upcall; 1455111033Sjeff p = td->td_proc; 1456111033Sjeff 1457103410Smini PROC_UNLOCK(p); 1458111033Sjeff error = copyin(&ku->ku_mailbox->km_sigscaught, &ss, sizeof(sigset_t)); 1459103410Smini if (error) 1460111033Sjeff goto error; 1461111033Sjeff 1462103410Smini SIGADDSET(ss, sig); 1463111033Sjeff 1464111033Sjeff error = copyout(&ss, &ku->ku_mailbox->km_sigscaught, sizeof(sigset_t)); 1465111033Sjeff if (error) 1466111033Sjeff goto error; 1467111033Sjeff 1468103410Smini PROC_LOCK(p); 1469111033Sjeff return; 1470111033Sjefferror: 1471111033Sjeff PROC_LOCK(p); 1472111033Sjeff sigexit(td, SIGILL); 1473111033Sjeff} 1474111033Sjeff 1475111033Sjeff 1476111033Sjeff/* 1477111033Sjeff * Schedule an upcall to notify a KSE process recieved signals. 1478111033Sjeff * 1479111033Sjeff */ 1480111033Sjeffvoid 1481111033Sjeffthread_signal_upcall(struct thread *td) 1482111033Sjeff{ 1483103410Smini mtx_lock_spin(&sched_lock); 1484111033Sjeff td->td_flags |= TDF_UPCALLING; 1485103410Smini mtx_unlock_spin(&sched_lock); 1486111033Sjeff 1487111033Sjeff return; 1488103410Smini} 1489103410Smini 1490112397Sdavidxuvoid 1491112397Sdavidxuthread_switchout(struct thread *td) 1492112397Sdavidxu{ 1493112397Sdavidxu struct kse_upcall *ku; 1494112397Sdavidxu 1495112397Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 1496112397Sdavidxu 1497112397Sdavidxu /* 1498112397Sdavidxu * If the outgoing thread is in threaded group and has never 1499112397Sdavidxu * scheduled an upcall, decide whether this is a short 1500112397Sdavidxu * or long term event and thus whether or not to schedule 1501112397Sdavidxu * an upcall. 1502112397Sdavidxu * If it is a short term event, just suspend it in 1503112397Sdavidxu * a way that takes its KSE with it. 1504112397Sdavidxu * Select the events for which we want to schedule upcalls. 1505112397Sdavidxu * For now it's just sleep. 1506112397Sdavidxu * XXXKSE eventually almost any inhibition could do. 1507112397Sdavidxu */ 1508112397Sdavidxu if (TD_CAN_UNBIND(td) && (td->td_standin) && TD_ON_SLEEPQ(td)) { 1509112397Sdavidxu /* 1510112397Sdavidxu * Release ownership of upcall, and schedule an upcall 1511112397Sdavidxu * thread, this new upcall thread becomes the owner of 1512112397Sdavidxu * the upcall structure. 1513112397Sdavidxu */ 1514112397Sdavidxu ku = td->td_upcall; 1515112397Sdavidxu ku->ku_owner = NULL; 1516112397Sdavidxu td->td_upcall = NULL; 1517112397Sdavidxu td->td_flags &= ~TDF_CAN_UNBIND; 1518112397Sdavidxu thread_schedule_upcall(td, ku); 1519112397Sdavidxu } 1520112397Sdavidxu} 1521112397Sdavidxu 1522103410Smini/* 1523111028Sjeff * Setup done on the thread when it enters the kernel. 1524105900Sjulian * XXXKSE Presently only for syscalls but eventually all kernel entries. 1525105900Sjulian */ 1526105900Sjulianvoid 1527105900Sjulianthread_user_enter(struct proc *p, struct thread *td) 1528105900Sjulian{ 1529111028Sjeff struct ksegrp *kg; 1530111028Sjeff struct kse_upcall *ku; 1531113793Sdavidxu struct kse_thr_mailbox *tmbx; 1532105900Sjulian 1533111028Sjeff kg = td->td_ksegrp; 1534113793Sdavidxu 1535105900Sjulian /* 1536105900Sjulian * First check that we shouldn't just abort. 1537105900Sjulian * But check if we are the single thread first! 1538105900Sjulian */ 1539113686Sjhb PROC_LOCK(p); 1540111028Sjeff if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1541105900Sjulian mtx_lock_spin(&sched_lock); 1542112071Sdavidxu thread_stopped(p); 1543105900Sjulian thread_exit(); 1544105900Sjulian /* NOTREACHED */ 1545105900Sjulian } 1546113686Sjhb PROC_UNLOCK(p); 1547105900Sjulian 1548105900Sjulian /* 1549105900Sjulian * If we are doing a syscall in a KSE environment, 1550105900Sjulian * note where our mailbox is. There is always the 1551108338Sjulian * possibility that we could do this lazily (in kse_reassign()), 1552105900Sjulian * but for now do it every time. 1553105900Sjulian */ 1554111028Sjeff kg = td->td_ksegrp; 1555111028Sjeff if (kg->kg_numupcalls) { 1556111028Sjeff ku = td->td_upcall; 1557111028Sjeff KASSERT(ku, ("%s: no upcall owned", __func__)); 1558111028Sjeff KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1559113793Sdavidxu KASSERT(!TD_CAN_UNBIND(td), ("%s: can unbind", __func__)); 1560113793Sdavidxu ku->ku_mflags = fuword((void *)&ku->ku_mailbox->km_flags); 1561113793Sdavidxu tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1562113793Sdavidxu if ((tmbx == NULL) || (tmbx == (void *)-1)) { 1563111028Sjeff td->td_mailbox = NULL; 1564105900Sjulian } else { 1565113793Sdavidxu td->td_mailbox = tmbx; 1566111115Sdavidxu if (td->td_standin == NULL) 1567111115Sdavidxu thread_alloc_spare(td, NULL); 1568111115Sdavidxu mtx_lock_spin(&sched_lock); 1569113793Sdavidxu if (ku->ku_mflags & KMF_NOUPCALL) 1570113793Sdavidxu td->td_flags &= ~TDF_CAN_UNBIND; 1571113793Sdavidxu else 1572113793Sdavidxu td->td_flags |= TDF_CAN_UNBIND; 1573111115Sdavidxu mtx_unlock_spin(&sched_lock); 1574105900Sjulian } 1575105900Sjulian } 1576105900Sjulian} 1577105900Sjulian 1578105900Sjulian/* 1579103410Smini * The extra work we go through if we are a threaded process when we 1580103410Smini * return to userland. 1581103410Smini * 158299026Sjulian * If we are a KSE process and returning to user mode, check for 158399026Sjulian * extra work to do before we return (e.g. for more syscalls 158499026Sjulian * to complete first). If we were in a critical section, we should 158599026Sjulian * just return to let it finish. Same if we were in the UTS (in 1586103410Smini * which case the mailbox's context's busy indicator will be set). 1587103410Smini * The only traps we suport will have set the mailbox. 1588103410Smini * We will clear it here. 158999026Sjulian */ 159099026Sjulianint 1591103838Sjulianthread_userret(struct thread *td, struct trapframe *frame) 159299026Sjulian{ 1593113793Sdavidxu int error = 0, upcalls, uts_crit; 1594111028Sjeff struct kse_upcall *ku; 1595111115Sdavidxu struct ksegrp *kg, *kg2; 1596104695Sjulian struct proc *p; 1597107060Sdavidxu struct timespec ts; 159899026Sjulian 1599111028Sjeff p = td->td_proc; 1600110190Sjulian kg = td->td_ksegrp; 1601104695Sjulian 1602111028Sjeff /* Nothing to do with non-threaded group/process */ 1603111028Sjeff if (td->td_ksegrp->kg_numupcalls == 0) 1604111028Sjeff return (0); 1605108338Sjulian 1606103410Smini /* 1607111028Sjeff * Stat clock interrupt hit in userland, it 1608111028Sjeff * is returning from interrupt, charge thread's 1609111028Sjeff * userland time for UTS. 1610103410Smini */ 1611111028Sjeff if (td->td_flags & TDF_USTATCLOCK) { 1612111515Sdavidxu thread_update_usr_ticks(td, 1); 1613111028Sjeff mtx_lock_spin(&sched_lock); 1614111028Sjeff td->td_flags &= ~TDF_USTATCLOCK; 1615111028Sjeff mtx_unlock_spin(&sched_lock); 1616111515Sdavidxu if (kg->kg_completed || 1617111515Sdavidxu (td->td_upcall->ku_flags & KUF_DOUPCALL)) 1618111515Sdavidxu thread_user_enter(p, td); 1619111028Sjeff } 1620108338Sjulian 1621113793Sdavidxu uts_crit = (td->td_mailbox == NULL); 1622113793Sdavidxu ku = td->td_upcall; 1623111028Sjeff /* 1624111028Sjeff * Optimisation: 1625111028Sjeff * This thread has not started any upcall. 1626111028Sjeff * If there is no work to report other than ourself, 1627111028Sjeff * then it can return direct to userland. 1628111028Sjeff */ 1629108338Sjulian if (TD_CAN_UNBIND(td)) { 1630111028Sjeff mtx_lock_spin(&sched_lock); 1631111028Sjeff td->td_flags &= ~TDF_CAN_UNBIND; 1632112888Sjeff if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && 1633112077Sdavidxu (kg->kg_completed == NULL) && 1634112397Sdavidxu (ku->ku_flags & KUF_DOUPCALL) == 0 && 1635113708Sdavidxu (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { 1636112888Sjeff mtx_unlock_spin(&sched_lock); 1637111515Sdavidxu thread_update_usr_ticks(td, 0); 1638112222Sdavidxu nanotime(&ts); 1639112397Sdavidxu error = copyout(&ts, 1640112222Sdavidxu (caddr_t)&ku->ku_mailbox->km_timeofday, 1641112222Sdavidxu sizeof(ts)); 1642112077Sdavidxu td->td_mailbox = 0; 1643113793Sdavidxu ku->ku_mflags = 0; 1644112222Sdavidxu if (error) 1645112222Sdavidxu goto out; 1646112077Sdavidxu return (0); 1647108338Sjulian } 1648112888Sjeff mtx_unlock_spin(&sched_lock); 1649104695Sjulian error = thread_export_context(td); 1650104695Sjulian if (error) { 1651104695Sjulian /* 1652111028Sjeff * Failing to do the KSE operation just defaults 1653104695Sjulian * back to synchonous operation, so just return from 1654108338Sjulian * the syscall. 1655104695Sjulian */ 1656113793Sdavidxu goto out; 1657104695Sjulian } 1658104695Sjulian /* 1659111028Sjeff * There is something to report, and we own an upcall 1660111028Sjeff * strucuture, we can go to userland. 1661111028Sjeff * Turn ourself into an upcall thread. 1662104695Sjulian */ 1663111028Sjeff mtx_lock_spin(&sched_lock); 1664104695Sjulian td->td_flags |= TDF_UPCALLING; 1665108338Sjulian mtx_unlock_spin(&sched_lock); 1666113793Sdavidxu } else if (td->td_mailbox && (ku == NULL)) { 1667108338Sjulian error = thread_export_context(td); 1668112071Sdavidxu /* possibly upcall with error? */ 1669112071Sdavidxu PROC_LOCK(p); 1670112071Sdavidxu /* 1671112071Sdavidxu * There are upcall threads waiting for 1672112071Sdavidxu * work to do, wake one of them up. 1673112071Sdavidxu * XXXKSE Maybe wake all of them up. 1674112071Sdavidxu */ 1675112071Sdavidxu if (!error && kg->kg_upsleeps) 1676112071Sdavidxu wakeup_one(&kg->kg_completed); 1677112071Sdavidxu mtx_lock_spin(&sched_lock); 1678112071Sdavidxu thread_stopped(p); 1679108338Sjulian thread_exit(); 1680111028Sjeff /* NOTREACHED */ 1681104695Sjulian } 1682104695Sjulian 1683111154Sdavidxu KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1684111154Sdavidxu 1685111154Sdavidxu if (p->p_numthreads > max_threads_per_proc) { 1686111154Sdavidxu max_threads_hits++; 1687111154Sdavidxu PROC_LOCK(p); 1688111154Sdavidxu while (p->p_numthreads > max_threads_per_proc) { 1689111154Sdavidxu if (P_SHOULDSTOP(p)) 1690111154Sdavidxu break; 1691111154Sdavidxu upcalls = 0; 1692111154Sdavidxu mtx_lock_spin(&sched_lock); 1693111154Sdavidxu FOREACH_KSEGRP_IN_PROC(p, kg2) { 1694111154Sdavidxu if (kg2->kg_numupcalls == 0) 1695111154Sdavidxu upcalls++; 1696111154Sdavidxu else 1697111154Sdavidxu upcalls += kg2->kg_numupcalls; 1698111154Sdavidxu } 1699111154Sdavidxu mtx_unlock_spin(&sched_lock); 1700111154Sdavidxu if (upcalls >= max_threads_per_proc) 1701111154Sdavidxu break; 1702111154Sdavidxu p->p_maxthrwaits++; 1703111154Sdavidxu msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1704111154Sdavidxu "maxthreads", NULL); 1705111154Sdavidxu p->p_maxthrwaits--; 1706111154Sdavidxu } 1707111154Sdavidxu PROC_UNLOCK(p); 1708111154Sdavidxu } 1709111154Sdavidxu 1710108338Sjulian if (td->td_flags & TDF_UPCALLING) { 1711113793Sdavidxu uts_crit = 0; 1712112397Sdavidxu kg->kg_nextupcall = ticks+kg->kg_upquantum; 1713108338Sjulian /* 1714108338Sjulian * There is no more work to do and we are going to ride 1715111028Sjeff * this thread up to userland as an upcall. 1716108338Sjulian * Do the last parts of the setup needed for the upcall. 1717108338Sjulian */ 1718108338Sjulian CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1719108338Sjulian td, td->td_proc->p_pid, td->td_proc->p_comm); 1720104695Sjulian 1721111028Sjeff mtx_lock_spin(&sched_lock); 1722111028Sjeff td->td_flags &= ~TDF_UPCALLING; 1723111028Sjeff if (ku->ku_flags & KUF_DOUPCALL) 1724111028Sjeff ku->ku_flags &= ~KUF_DOUPCALL; 1725111028Sjeff mtx_unlock_spin(&sched_lock); 1726111028Sjeff 1727111028Sjeff /* 1728113793Sdavidxu * Set user context to the UTS 1729113793Sdavidxu */ 1730113793Sdavidxu if (!(ku->ku_mflags & KMF_NOUPCALL)) { 1731113793Sdavidxu cpu_set_upcall_kse(td, ku); 1732113793Sdavidxu error = suword(&ku->ku_mailbox->km_curthread, 0); 1733113793Sdavidxu if (error) 1734113793Sdavidxu goto out; 1735113793Sdavidxu } 1736113793Sdavidxu 1737113793Sdavidxu /* 1738108338Sjulian * Unhook the list of completed threads. 1739108338Sjulian * anything that completes after this gets to 1740108338Sjulian * come in next time. 1741108338Sjulian * Put the list of completed thread mailboxes on 1742108338Sjulian * this KSE's mailbox. 1743108338Sjulian */ 1744113793Sdavidxu if (!(ku->ku_mflags & KMF_NOCOMPLETED) && 1745113793Sdavidxu (error = thread_link_mboxes(kg, ku)) != 0) 1746111115Sdavidxu goto out; 1747113793Sdavidxu } 1748113793Sdavidxu if (!uts_crit) { 1749107060Sdavidxu nanotime(&ts); 1750113793Sdavidxu error = copyout(&ts, &ku->ku_mailbox->km_timeofday, sizeof(ts)); 1751111115Sdavidxu } 1752111115Sdavidxu 1753111115Sdavidxuout: 1754111115Sdavidxu if (error) { 1755111115Sdavidxu /* 1756111129Sdavidxu * Things are going to be so screwed we should just kill 1757111129Sdavidxu * the process. 1758111115Sdavidxu * how do we do that? 1759111115Sdavidxu */ 1760111115Sdavidxu PROC_LOCK(td->td_proc); 1761111115Sdavidxu psignal(td->td_proc, SIGSEGV); 1762111115Sdavidxu PROC_UNLOCK(td->td_proc); 1763111115Sdavidxu } else { 1764111115Sdavidxu /* 1765111115Sdavidxu * Optimisation: 1766111115Sdavidxu * Ensure that we have a spare thread available, 1767111115Sdavidxu * for when we re-enter the kernel. 1768111115Sdavidxu */ 1769111115Sdavidxu if (td->td_standin == NULL) 1770111115Sdavidxu thread_alloc_spare(td, NULL); 1771111115Sdavidxu } 1772111115Sdavidxu 1773113793Sdavidxu ku->ku_mflags = 0; 1774111028Sjeff /* 1775111028Sjeff * Clear thread mailbox first, then clear system tick count. 1776111028Sjeff * The order is important because thread_statclock() use 1777111028Sjeff * mailbox pointer to see if it is an userland thread or 1778111028Sjeff * an UTS kernel thread. 1779111028Sjeff */ 1780108338Sjulian td->td_mailbox = NULL; 1781111028Sjeff td->td_usticks = 0; 1782104695Sjulian return (error); /* go sync */ 178399026Sjulian} 178499026Sjulian 178599026Sjulian/* 178699026Sjulian * Enforce single-threading. 178799026Sjulian * 178899026Sjulian * Returns 1 if the caller must abort (another thread is waiting to 178999026Sjulian * exit the process or similar). Process is locked! 179099026Sjulian * Returns 0 when you are successfully the only thread running. 179199026Sjulian * A process has successfully single threaded in the suspend mode when 179299026Sjulian * There are no threads in user mode. Threads in the kernel must be 179399026Sjulian * allowed to continue until they get to the user boundary. They may even 179499026Sjulian * copy out their return values and data before suspending. They may however be 179599026Sjulian * accellerated in reaching the user boundary as we will wake up 179699026Sjulian * any sleeping threads that are interruptable. (PCATCH). 179799026Sjulian */ 179899026Sjulianint 179999026Sjulianthread_single(int force_exit) 180099026Sjulian{ 180199026Sjulian struct thread *td; 180299026Sjulian struct thread *td2; 180399026Sjulian struct proc *p; 180499026Sjulian 180599026Sjulian td = curthread; 180699026Sjulian p = td->td_proc; 1807107719Sjulian mtx_assert(&Giant, MA_OWNED); 180899026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 180999026Sjulian KASSERT((td != NULL), ("curthread is NULL")); 181099026Sjulian 1811112910Sjeff if ((p->p_flag & P_THREADED) == 0 && p->p_numthreads == 1) 181299026Sjulian return (0); 181399026Sjulian 1814100648Sjulian /* Is someone already single threading? */ 1815100648Sjulian if (p->p_singlethread) 181699026Sjulian return (1); 181799026Sjulian 1818108338Sjulian if (force_exit == SINGLE_EXIT) { 181999026Sjulian p->p_flag |= P_SINGLE_EXIT; 1820108338Sjulian } else 182199026Sjulian p->p_flag &= ~P_SINGLE_EXIT; 1822102950Sdavidxu p->p_flag |= P_STOPPED_SINGLE; 182399026Sjulian p->p_singlethread = td; 1824105911Sjulian /* XXXKSE Which lock protects the below values? */ 182599026Sjulian while ((p->p_numthreads - p->p_suspcount) != 1) { 1826103216Sjulian mtx_lock_spin(&sched_lock); 182799026Sjulian FOREACH_THREAD_IN_PROC(p, td2) { 182899026Sjulian if (td2 == td) 182999026Sjulian continue; 1830113705Sdavidxu td2->td_flags |= TDF_ASTPENDING; 1831103216Sjulian if (TD_IS_INHIBITED(td2)) { 1832105911Sjulian if (force_exit == SINGLE_EXIT) { 1833105911Sjulian if (TD_IS_SUSPENDED(td2)) { 1834103216Sjulian thread_unsuspend_one(td2); 1835105911Sjulian } 1836105911Sjulian if (TD_ON_SLEEPQ(td2) && 1837105911Sjulian (td2->td_flags & TDF_SINTR)) { 1838105911Sjulian if (td2->td_flags & TDF_CVWAITQ) 1839105911Sjulian cv_abort(td2); 1840105911Sjulian else 1841105911Sjulian abortsleep(td2); 1842105911Sjulian } 1843105911Sjulian } else { 1844105911Sjulian if (TD_IS_SUSPENDED(td2)) 1845105874Sdavidxu continue; 1846111028Sjeff /* 1847111028Sjeff * maybe other inhibitted states too? 1848111028Sjeff * XXXKSE Is it totally safe to 1849111028Sjeff * suspend a non-interruptable thread? 1850111028Sjeff */ 1851108338Sjulian if (td2->td_inhibitors & 1852111028Sjeff (TDI_SLEEPING | TDI_SWAPPED)) 1853105911Sjulian thread_suspend_one(td2); 185499026Sjulian } 185599026Sjulian } 185699026Sjulian } 1857105911Sjulian /* 1858105911Sjulian * Maybe we suspended some threads.. was it enough? 1859105911Sjulian */ 1860105911Sjulian if ((p->p_numthreads - p->p_suspcount) == 1) { 1861105911Sjulian mtx_unlock_spin(&sched_lock); 1862105911Sjulian break; 1863105911Sjulian } 1864105911Sjulian 186599026Sjulian /* 186699026Sjulian * Wake us up when everyone else has suspended. 1867100648Sjulian * In the mean time we suspend as well. 186899026Sjulian */ 1869103216Sjulian thread_suspend_one(td); 1870113795Sdavidxu DROP_GIANT(); 187199026Sjulian PROC_UNLOCK(p); 1872107719Sjulian p->p_stats->p_ru.ru_nvcsw++; 187399026Sjulian mi_switch(); 187499026Sjulian mtx_unlock_spin(&sched_lock); 1875113795Sdavidxu PICKUP_GIANT(); 187699026Sjulian PROC_LOCK(p); 187799026Sjulian } 1878111028Sjeff if (force_exit == SINGLE_EXIT) { 1879111028Sjeff if (td->td_upcall) { 1880111028Sjeff mtx_lock_spin(&sched_lock); 1881111028Sjeff upcall_remove(td); 1882111028Sjeff mtx_unlock_spin(&sched_lock); 1883111028Sjeff } 1884105854Sjulian kse_purge(p, td); 1885111028Sjeff } 188699026Sjulian return (0); 188799026Sjulian} 188899026Sjulian 188999026Sjulian/* 189099026Sjulian * Called in from locations that can safely check to see 189199026Sjulian * whether we have to suspend or at least throttle for a 189299026Sjulian * single-thread event (e.g. fork). 189399026Sjulian * 189499026Sjulian * Such locations include userret(). 189599026Sjulian * If the "return_instead" argument is non zero, the thread must be able to 189699026Sjulian * accept 0 (caller may continue), or 1 (caller must abort) as a result. 189799026Sjulian * 189899026Sjulian * The 'return_instead' argument tells the function if it may do a 189999026Sjulian * thread_exit() or suspend, or whether the caller must abort and back 190099026Sjulian * out instead. 190199026Sjulian * 190299026Sjulian * If the thread that set the single_threading request has set the 190399026Sjulian * P_SINGLE_EXIT bit in the process flags then this call will never return 190499026Sjulian * if 'return_instead' is false, but will exit. 190599026Sjulian * 190699026Sjulian * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 190799026Sjulian *---------------+--------------------+--------------------- 190899026Sjulian * 0 | returns 0 | returns 0 or 1 190999026Sjulian * | when ST ends | immediatly 191099026Sjulian *---------------+--------------------+--------------------- 191199026Sjulian * 1 | thread exits | returns 1 191299026Sjulian * | | immediatly 191399026Sjulian * 0 = thread_exit() or suspension ok, 191499026Sjulian * other = return error instead of stopping the thread. 191599026Sjulian * 191699026Sjulian * While a full suspension is under effect, even a single threading 191799026Sjulian * thread would be suspended if it made this call (but it shouldn't). 191899026Sjulian * This call should only be made from places where 191999026Sjulian * thread_exit() would be safe as that may be the outcome unless 192099026Sjulian * return_instead is set. 192199026Sjulian */ 192299026Sjulianint 192399026Sjulianthread_suspend_check(int return_instead) 192499026Sjulian{ 1925104502Sjmallett struct thread *td; 1926104502Sjmallett struct proc *p; 1927105854Sjulian struct ksegrp *kg; 192899026Sjulian 192999026Sjulian td = curthread; 193099026Sjulian p = td->td_proc; 1931105854Sjulian kg = td->td_ksegrp; 193299026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 193399026Sjulian while (P_SHOULDSTOP(p)) { 1934102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 193599026Sjulian KASSERT(p->p_singlethread != NULL, 193699026Sjulian ("singlethread not set")); 193799026Sjulian /* 1938100648Sjulian * The only suspension in action is a 1939100648Sjulian * single-threading. Single threader need not stop. 1940100646Sjulian * XXX Should be safe to access unlocked 1941100646Sjulian * as it can only be set to be true by us. 194299026Sjulian */ 1943100648Sjulian if (p->p_singlethread == td) 194499026Sjulian return (0); /* Exempt from stopping. */ 194599026Sjulian } 1946100648Sjulian if (return_instead) 194799026Sjulian return (1); 194899026Sjulian 1949112071Sdavidxu mtx_lock_spin(&sched_lock); 1950112071Sdavidxu thread_stopped(p); 195199026Sjulian /* 195299026Sjulian * If the process is waiting for us to exit, 195399026Sjulian * this thread should just suicide. 1954102950Sdavidxu * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 195599026Sjulian */ 195699026Sjulian if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 195799026Sjulian while (mtx_owned(&Giant)) 195899026Sjulian mtx_unlock(&Giant); 1959112910Sjeff if (p->p_flag & P_THREADED) 1960112910Sjeff thread_exit(); 1961112910Sjeff else 1962112910Sjeff thr_exit1(); 196399026Sjulian } 196499026Sjulian 1965112910Sjeff mtx_assert(&Giant, MA_NOTOWNED); 196699026Sjulian /* 196799026Sjulian * When a thread suspends, it just 196899026Sjulian * moves to the processes's suspend queue 196999026Sjulian * and stays there. 197099026Sjulian */ 1971103216Sjulian thread_suspend_one(td); 197299026Sjulian PROC_UNLOCK(p); 1973102950Sdavidxu if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1974100632Sjulian if (p->p_numthreads == p->p_suspcount) { 1975103216Sjulian thread_unsuspend_one(p->p_singlethread); 1976100632Sjulian } 1977100632Sjulian } 1978100594Sjulian p->p_stats->p_ru.ru_nivcsw++; 197999026Sjulian mi_switch(); 198099026Sjulian mtx_unlock_spin(&sched_lock); 198199026Sjulian PROC_LOCK(p); 198299026Sjulian } 198399026Sjulian return (0); 198499026Sjulian} 198599026Sjulian 1986102898Sdavidxuvoid 1987102898Sdavidxuthread_suspend_one(struct thread *td) 1988102898Sdavidxu{ 1989102898Sdavidxu struct proc *p = td->td_proc; 1990102898Sdavidxu 1991102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 1992112071Sdavidxu KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 1993102898Sdavidxu p->p_suspcount++; 1994103216Sjulian TD_SET_SUSPENDED(td); 1995102898Sdavidxu TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 1996103216Sjulian /* 1997103216Sjulian * Hack: If we are suspending but are on the sleep queue 1998103216Sjulian * then we are in msleep or the cv equivalent. We 1999103216Sjulian * want to look like we have two Inhibitors. 2000105911Sjulian * May already be set.. doesn't matter. 2001103216Sjulian */ 2002103216Sjulian if (TD_ON_SLEEPQ(td)) 2003103216Sjulian TD_SET_SLEEPING(td); 2004102898Sdavidxu} 2005102898Sdavidxu 2006102898Sdavidxuvoid 2007102898Sdavidxuthread_unsuspend_one(struct thread *td) 2008102898Sdavidxu{ 2009102898Sdavidxu struct proc *p = td->td_proc; 2010102898Sdavidxu 2011102898Sdavidxu mtx_assert(&sched_lock, MA_OWNED); 2012102898Sdavidxu TAILQ_REMOVE(&p->p_suspended, td, td_runq); 2013103216Sjulian TD_CLR_SUSPENDED(td); 2014102898Sdavidxu p->p_suspcount--; 2015103216Sjulian setrunnable(td); 2016102898Sdavidxu} 2017102898Sdavidxu 201899026Sjulian/* 201999026Sjulian * Allow all threads blocked by single threading to continue running. 202099026Sjulian */ 202199026Sjulianvoid 202299026Sjulianthread_unsuspend(struct proc *p) 202399026Sjulian{ 202499026Sjulian struct thread *td; 202599026Sjulian 2026100646Sjulian mtx_assert(&sched_lock, MA_OWNED); 202799026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 202899026Sjulian if (!P_SHOULDSTOP(p)) { 202999026Sjulian while (( td = TAILQ_FIRST(&p->p_suspended))) { 2030102898Sdavidxu thread_unsuspend_one(td); 203199026Sjulian } 2032102950Sdavidxu } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 203399026Sjulian (p->p_numthreads == p->p_suspcount)) { 203499026Sjulian /* 203599026Sjulian * Stopping everything also did the job for the single 203699026Sjulian * threading request. Now we've downgraded to single-threaded, 203799026Sjulian * let it continue. 203899026Sjulian */ 2039102898Sdavidxu thread_unsuspend_one(p->p_singlethread); 204099026Sjulian } 204199026Sjulian} 204299026Sjulian 204399026Sjulianvoid 204499026Sjulianthread_single_end(void) 204599026Sjulian{ 204699026Sjulian struct thread *td; 204799026Sjulian struct proc *p; 204899026Sjulian 204999026Sjulian td = curthread; 205099026Sjulian p = td->td_proc; 205199026Sjulian PROC_LOCK_ASSERT(p, MA_OWNED); 2052102950Sdavidxu p->p_flag &= ~P_STOPPED_SINGLE; 205399026Sjulian p->p_singlethread = NULL; 2054102292Sjulian /* 2055102292Sjulian * If there are other threads they mey now run, 2056102292Sjulian * unless of course there is a blanket 'stop order' 2057102292Sjulian * on the process. The single threader must be allowed 2058102292Sjulian * to continue however as this is a bad place to stop. 2059102292Sjulian */ 2060102292Sjulian if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2061102292Sjulian mtx_lock_spin(&sched_lock); 2062102292Sjulian while (( td = TAILQ_FIRST(&p->p_suspended))) { 2063103216Sjulian thread_unsuspend_one(td); 2064102292Sjulian } 2065102292Sjulian mtx_unlock_spin(&sched_lock); 2066102292Sjulian } 206799026Sjulian} 206899026Sjulian 2069102292Sjulian 2070