kern_thread.c revision 106182
1/* 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 * 28 * $FreeBSD: head/sys/kern/kern_thread.c 106182 2002-10-30 03:01:28Z davidxu $ 29 */ 30 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/kernel.h> 34#include <sys/lock.h> 35#include <sys/malloc.h> 36#include <sys/mutex.h> 37#include <sys/proc.h> 38#include <sys/sysctl.h> 39#include <sys/sysproto.h> 40#include <sys/filedesc.h> 41#include <sys/tty.h> 42#include <sys/signalvar.h> 43#include <sys/sx.h> 44#include <sys/user.h> 45#include <sys/jail.h> 46#include <sys/kse.h> 47#include <sys/ktr.h> 48#include <sys/ucontext.h> 49 50#include <vm/vm.h> 51#include <vm/vm_object.h> 52#include <vm/pmap.h> 53#include <vm/uma.h> 54#include <vm/vm_map.h> 55 56#include <machine/frame.h> 57 58/* 59 * KSEGRP related storage. 60 */ 61static uma_zone_t ksegrp_zone; 62static uma_zone_t kse_zone; 63static uma_zone_t thread_zone; 64 65/* DEBUG ONLY */ 66SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 67static int oiks_debug = 1; /* 0 disable, 1 printf, 2 enter debugger */ 68SYSCTL_INT(_kern_threads, OID_AUTO, oiks, CTLFLAG_RW, 69 &oiks_debug, 0, "OIKS thread debug"); 70 71static int max_threads_per_proc = 10; 72SYSCTL_INT(_kern_threads, OID_AUTO, max_per_proc, CTLFLAG_RW, 73 &max_threads_per_proc, 0, "Limit on threads per proc"); 74 75#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 76 77struct threadqueue zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 78TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 79TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 80struct mtx zombie_thread_lock; 81MTX_SYSINIT(zombie_thread_lock, &zombie_thread_lock, 82 "zombie_thread_lock", MTX_SPIN); 83 84 85 86void kse_purge(struct proc *p, struct thread *td); 87/* 88 * Pepare a thread for use. 89 */ 90static void 91thread_ctor(void *mem, int size, void *arg) 92{ 93 struct thread *td; 94 95 KASSERT((size == sizeof(struct thread)), 96 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 97 98 td = (struct thread *)mem; 99 td->td_state = TDS_INACTIVE; 100 td->td_flags |= TDF_UNBOUND; 101} 102 103/* 104 * Reclaim a thread after use. 105 */ 106static void 107thread_dtor(void *mem, int size, void *arg) 108{ 109 struct thread *td; 110 111 KASSERT((size == sizeof(struct thread)), 112 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 113 114 td = (struct thread *)mem; 115 116#ifdef INVARIANTS 117 /* Verify that this thread is in a safe state to free. */ 118 switch (td->td_state) { 119 case TDS_INHIBITED: 120 case TDS_RUNNING: 121 case TDS_CAN_RUN: 122 case TDS_RUNQ: 123 /* 124 * We must never unlink a thread that is in one of 125 * these states, because it is currently active. 126 */ 127 panic("bad state for thread unlinking"); 128 /* NOTREACHED */ 129 case TDS_INACTIVE: 130 break; 131 default: 132 panic("bad thread state"); 133 /* NOTREACHED */ 134 } 135#endif 136} 137 138/* 139 * Initialize type-stable parts of a thread (when newly created). 140 */ 141static void 142thread_init(void *mem, int size) 143{ 144 struct thread *td; 145 146 KASSERT((size == sizeof(struct thread)), 147 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 148 149 td = (struct thread *)mem; 150 mtx_lock(&Giant); 151 pmap_new_thread(td, 0); 152 mtx_unlock(&Giant); 153 cpu_thread_setup(td); 154} 155 156/* 157 * Tear down type-stable parts of a thread (just before being discarded). 158 */ 159static void 160thread_fini(void *mem, int size) 161{ 162 struct thread *td; 163 164 KASSERT((size == sizeof(struct thread)), 165 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 166 167 td = (struct thread *)mem; 168 pmap_dispose_thread(td); 169} 170 171/* 172 * KSE is linked onto the idle queue. 173 */ 174void 175kse_link(struct kse *ke, struct ksegrp *kg) 176{ 177 struct proc *p = kg->kg_proc; 178 179 TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 180 kg->kg_kses++; 181 ke->ke_state = KES_UNQUEUED; 182 ke->ke_proc = p; 183 ke->ke_ksegrp = kg; 184 ke->ke_thread = NULL; 185 ke->ke_oncpu = NOCPU; 186} 187 188void 189kse_unlink(struct kse *ke) 190{ 191 struct ksegrp *kg; 192 193 mtx_assert(&sched_lock, MA_OWNED); 194 kg = ke->ke_ksegrp; 195 if (ke->ke_state == KES_IDLE) { 196 kg->kg_idle_kses--; 197 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 198 } 199 200 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 201 if (--kg->kg_kses == 0) { 202 ksegrp_unlink(kg); 203 } 204 /* 205 * Aggregate stats from the KSE 206 */ 207 kse_stash(ke); 208} 209 210void 211ksegrp_link(struct ksegrp *kg, struct proc *p) 212{ 213 214 TAILQ_INIT(&kg->kg_threads); 215 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 216 TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 217 TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 218 TAILQ_INIT(&kg->kg_iq); /* idle kses in ksegrp */ 219 TAILQ_INIT(&kg->kg_lq); /* loan kses in ksegrp */ 220 kg->kg_proc = p; 221/* the following counters are in the -zero- section and may not need clearing */ 222 kg->kg_numthreads = 0; 223 kg->kg_runnable = 0; 224 kg->kg_kses = 0; 225 kg->kg_idle_kses = 0; 226 kg->kg_loan_kses = 0; 227 kg->kg_runq_kses = 0; /* XXXKSE change name */ 228/* link it in now that it's consistent */ 229 p->p_numksegrps++; 230 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 231} 232 233void 234ksegrp_unlink(struct ksegrp *kg) 235{ 236 struct proc *p; 237 238 mtx_assert(&sched_lock, MA_OWNED); 239 p = kg->kg_proc; 240 KASSERT(((kg->kg_numthreads == 0) && (kg->kg_kses == 0)), 241 ("kseg_unlink: residual threads or KSEs")); 242 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 243 p->p_numksegrps--; 244 /* 245 * Aggregate stats from the KSE 246 */ 247 ksegrp_stash(kg); 248} 249 250/* 251 * for a newly created process, 252 * link up a the structure and its initial threads etc. 253 */ 254void 255proc_linkup(struct proc *p, struct ksegrp *kg, 256 struct kse *ke, struct thread *td) 257{ 258 259 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 260 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 261 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 262 p->p_numksegrps = 0; 263 p->p_numthreads = 0; 264 265 ksegrp_link(kg, p); 266 kse_link(ke, kg); 267 thread_link(td, kg); 268} 269 270int 271kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 272{ 273 struct proc *p; 274 struct thread *td2; 275 276 p = td->td_proc; 277 mtx_lock_spin(&sched_lock); 278 FOREACH_THREAD_IN_PROC(p, td2) { 279 if (td2->td_mailbox == uap->tmbx) { 280 td2->td_flags |= TDF_INTERRUPT; 281 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { 282 if (td2->td_flags & TDF_CVWAITQ) 283 cv_abort(td2); 284 else 285 abortsleep(td2); 286 } 287 mtx_unlock_spin(&sched_lock); 288 td->td_retval[0] = 0; 289 td->td_retval[1] = 0; 290 return (0); 291 } 292 } 293 mtx_unlock_spin(&sched_lock); 294 return (ESRCH); 295} 296 297int 298kse_exit(struct thread *td, struct kse_exit_args *uap) 299{ 300 struct proc *p; 301 struct ksegrp *kg; 302 303 p = td->td_proc; 304 /* KSE-enabled processes only, please. */ 305 if (!(p->p_flag & P_KSES)) 306 return (EINVAL); 307 /* must be a bound thread */ 308 if (td->td_flags & TDF_UNBOUND) 309 return (EINVAL); 310 kg = td->td_ksegrp; 311 /* serialize killing kse */ 312 PROC_LOCK(p); 313 mtx_lock_spin(&sched_lock); 314 if ((kg->kg_kses == 1) && (kg->kg_numthreads > 1)) { 315 mtx_unlock_spin(&sched_lock); 316 PROC_UNLOCK(p); 317 return (EDEADLK); 318 } 319 if ((p->p_numthreads == 1) && (p->p_numksegrps == 1)) { 320 p->p_flag &= ~P_KSES; 321 mtx_unlock_spin(&sched_lock); 322 PROC_UNLOCK(p); 323 } else { 324 while (mtx_owned(&Giant)) 325 mtx_unlock(&Giant); 326 td->td_kse->ke_flags |= KEF_EXIT; 327 thread_exit(); 328 /* NOTREACHED */ 329 } 330 return (0); 331} 332 333int 334kse_release(struct thread *td, struct kse_release_args *uap) 335{ 336 struct proc *p; 337 338 p = td->td_proc; 339 /* KSE-enabled processes only, please. */ 340 if (p->p_flag & P_KSES) { 341 PROC_LOCK(p); 342 mtx_lock_spin(&sched_lock); 343 thread_exit(); 344 /* NOTREACHED */ 345 } 346 return (EINVAL); 347} 348 349/* struct kse_wakeup_args { 350 struct kse_mailbox *mbx; 351}; */ 352int 353kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 354{ 355 struct proc *p; 356 struct kse *ke, *ke2; 357 struct ksegrp *kg; 358 359 p = td->td_proc; 360 /* KSE-enabled processes only, please. */ 361 if (!(p->p_flag & P_KSES)) 362 return EINVAL; 363 if (td->td_standin == NULL) 364 td->td_standin = thread_alloc(); 365 ke = NULL; 366 mtx_lock_spin(&sched_lock); 367 if (uap->mbx) { 368 FOREACH_KSEGRP_IN_PROC(p, kg) { 369 FOREACH_KSE_IN_GROUP(kg, ke2) { 370 if (ke2->ke_mailbox != uap->mbx) 371 continue; 372 if (ke2->ke_state == KES_IDLE) { 373 ke = ke2; 374 goto found; 375 } else { 376 mtx_unlock_spin(&sched_lock); 377 td->td_retval[0] = 0; 378 td->td_retval[1] = 0; 379 return (0); 380 } 381 } 382 } 383 } else { 384 kg = td->td_ksegrp; 385 ke = TAILQ_FIRST(&kg->kg_iq); 386 } 387 if (ke == NULL) { 388 mtx_unlock_spin(&sched_lock); 389 return (ESRCH); 390 } 391found: 392 thread_schedule_upcall(td, ke); 393 mtx_unlock_spin(&sched_lock); 394 td->td_retval[0] = 0; 395 td->td_retval[1] = 0; 396 return (0); 397} 398 399/* 400 * No new KSEG: first call: use current KSE, don't schedule an upcall 401 * All other situations, do allocate a new KSE and schedule an upcall on it. 402 */ 403/* struct kse_create_args { 404 struct kse_mailbox *mbx; 405 int newgroup; 406}; */ 407int 408kse_create(struct thread *td, struct kse_create_args *uap) 409{ 410 struct kse *newke; 411 struct kse *ke; 412 struct ksegrp *newkg; 413 struct ksegrp *kg; 414 struct proc *p; 415 struct kse_mailbox mbx; 416 int err; 417 418 p = td->td_proc; 419 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 420 return (err); 421 422 p->p_flag |= P_KSES; /* easier to just set it than to test and set */ 423 kg = td->td_ksegrp; 424 if (uap->newgroup) { 425 /* 426 * If we want a new KSEGRP it doesn't matter whether 427 * we have already fired up KSE mode before or not. 428 * We put the process in KSE mode and create a new KSEGRP 429 * and KSE. If our KSE has not got a mailbox yet then 430 * that doesn't matter, just leave it that way. It will 431 * ensure that this thread stay BOUND. It's possible 432 * that the call came form a threaded library and the main 433 * program knows nothing of threads. 434 */ 435 newkg = ksegrp_alloc(); 436 bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 437 kg_startzero, kg_endzero)); 438 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 439 RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 440 newke = kse_alloc(); 441 } else { 442 /* 443 * Otherwise, if we have already set this KSE 444 * to have a mailbox, we want to make another KSE here, 445 * but only if there are not already the limit, which 446 * is 1 per CPU max. 447 * 448 * If the current KSE doesn't have a mailbox we just use it 449 * and give it one. 450 * 451 * Because we don't like to access 452 * the KSE outside of schedlock if we are UNBOUND, 453 * (because it can change if we are preempted by an interrupt) 454 * we can deduce it as having a mailbox if we are UNBOUND, 455 * and only need to actually look at it if we are BOUND, 456 * which is safe. 457 */ 458 if ((td->td_flags & TDF_UNBOUND) || td->td_kse->ke_mailbox) { 459#if 0 /* while debugging */ 460#ifdef SMP 461 if (kg->kg_kses > mp_ncpus) 462#endif 463 return (EPROCLIM); 464#endif 465 newke = kse_alloc(); 466 } else { 467 newke = NULL; 468 } 469 newkg = NULL; 470 } 471 if (newke) { 472 bzero(&newke->ke_startzero, RANGEOF(struct kse, 473 ke_startzero, ke_endzero)); 474#if 0 475 bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 476 RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 477#endif 478 /* For the first call this may not have been set */ 479 if (td->td_standin == NULL) { 480 td->td_standin = thread_alloc(); 481 } 482 mtx_lock_spin(&sched_lock); 483 if (newkg) 484 ksegrp_link(newkg, p); 485 else 486 newkg = kg; 487 kse_link(newke, newkg); 488 if (p->p_sflag & PS_NEEDSIGCHK) 489 newke->ke_flags |= KEF_ASTPENDING; 490 newke->ke_mailbox = uap->mbx; 491 newke->ke_upcall = mbx.km_func; 492 bcopy(&mbx.km_stack, &newke->ke_stack, sizeof(stack_t)); 493 thread_schedule_upcall(td, newke); 494 mtx_unlock_spin(&sched_lock); 495 } else { 496 /* 497 * If we didn't allocate a new KSE then the we are using 498 * the exisiting (BOUND) kse. 499 */ 500 ke = td->td_kse; 501 ke->ke_mailbox = uap->mbx; 502 ke->ke_upcall = mbx.km_func; 503 bcopy(&mbx.km_stack, &ke->ke_stack, sizeof(stack_t)); 504 } 505 /* 506 * Fill out the KSE-mode specific fields of the new kse. 507 */ 508 509 td->td_retval[0] = 0; 510 td->td_retval[1] = 0; 511 return (0); 512} 513 514/* 515 * Fill a ucontext_t with a thread's context information. 516 * 517 * This is an analogue to getcontext(3). 518 */ 519void 520thread_getcontext(struct thread *td, ucontext_t *uc) 521{ 522 523/* 524 * XXX this is declared in a MD include file, i386/include/ucontext.h but 525 * is used in MI code. 526 */ 527#ifdef __i386__ 528 get_mcontext(td, &uc->uc_mcontext); 529#endif 530 uc->uc_sigmask = td->td_proc->p_sigmask; 531} 532 533/* 534 * Set a thread's context from a ucontext_t. 535 * 536 * This is an analogue to setcontext(3). 537 */ 538int 539thread_setcontext(struct thread *td, ucontext_t *uc) 540{ 541 int ret; 542 543/* 544 * XXX this is declared in a MD include file, i386/include/ucontext.h but 545 * is used in MI code. 546 */ 547#ifdef __i386__ 548 ret = set_mcontext(td, &uc->uc_mcontext); 549#else 550 ret = ENOSYS; 551#endif 552 if (ret == 0) { 553 SIG_CANTMASK(uc->uc_sigmask); 554 PROC_LOCK(td->td_proc); 555 td->td_proc->p_sigmask = uc->uc_sigmask; 556 PROC_UNLOCK(td->td_proc); 557 } 558 return (ret); 559} 560 561/* 562 * Initialize global thread allocation resources. 563 */ 564void 565threadinit(void) 566{ 567 568#ifndef __ia64__ 569 thread_zone = uma_zcreate("THREAD", sizeof (struct thread), 570 thread_ctor, thread_dtor, thread_init, thread_fini, 571 UMA_ALIGN_CACHE, 0); 572#else 573 /* 574 * XXX the ia64 kstack allocator is really lame and is at the mercy 575 * of contigmallloc(). This hackery is to pre-construct a whole 576 * pile of thread structures with associated kernel stacks early 577 * in the system startup while contigmalloc() still works. Once we 578 * have them, keep them. Sigh. 579 */ 580 thread_zone = uma_zcreate("THREAD", sizeof (struct thread), 581 thread_ctor, thread_dtor, thread_init, thread_fini, 582 UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); 583 uma_prealloc(thread_zone, 512); /* XXX arbitary */ 584#endif 585 ksegrp_zone = uma_zcreate("KSEGRP", sizeof (struct ksegrp), 586 NULL, NULL, NULL, NULL, 587 UMA_ALIGN_CACHE, 0); 588 kse_zone = uma_zcreate("KSE", sizeof (struct kse), 589 NULL, NULL, NULL, NULL, 590 UMA_ALIGN_CACHE, 0); 591} 592 593/* 594 * Stash an embarasingly extra thread into the zombie thread queue. 595 */ 596void 597thread_stash(struct thread *td) 598{ 599 mtx_lock_spin(&zombie_thread_lock); 600 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 601 mtx_unlock_spin(&zombie_thread_lock); 602} 603 604/* 605 * Stash an embarasingly extra kse into the zombie kse queue. 606 */ 607void 608kse_stash(struct kse *ke) 609{ 610 mtx_lock_spin(&zombie_thread_lock); 611 TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 612 mtx_unlock_spin(&zombie_thread_lock); 613} 614 615/* 616 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 617 */ 618void 619ksegrp_stash(struct ksegrp *kg) 620{ 621 mtx_lock_spin(&zombie_thread_lock); 622 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 623 mtx_unlock_spin(&zombie_thread_lock); 624} 625 626/* 627 * Reap zombie threads. 628 */ 629void 630thread_reap(void) 631{ 632 struct thread *td_first, *td_next; 633 struct kse *ke_first, *ke_next; 634 struct ksegrp *kg_first, * kg_next; 635 636 /* 637 * don't even bother to lock if none at this instant 638 * We really don't care about the next instant.. 639 */ 640 if ((!TAILQ_EMPTY(&zombie_threads)) 641 || (!TAILQ_EMPTY(&zombie_kses)) 642 || (!TAILQ_EMPTY(&zombie_ksegrps))) { 643 mtx_lock_spin(&zombie_thread_lock); 644 td_first = TAILQ_FIRST(&zombie_threads); 645 ke_first = TAILQ_FIRST(&zombie_kses); 646 kg_first = TAILQ_FIRST(&zombie_ksegrps); 647 if (td_first) 648 TAILQ_INIT(&zombie_threads); 649 if (ke_first) 650 TAILQ_INIT(&zombie_kses); 651 if (kg_first) 652 TAILQ_INIT(&zombie_ksegrps); 653 mtx_unlock_spin(&zombie_thread_lock); 654 while (td_first) { 655 td_next = TAILQ_NEXT(td_first, td_runq); 656 thread_free(td_first); 657 td_first = td_next; 658 } 659 while (ke_first) { 660 ke_next = TAILQ_NEXT(ke_first, ke_procq); 661 kse_free(ke_first); 662 ke_first = ke_next; 663 } 664 while (kg_first) { 665 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 666 ksegrp_free(kg_first); 667 kg_first = kg_next; 668 } 669 } 670} 671 672/* 673 * Allocate a ksegrp. 674 */ 675struct ksegrp * 676ksegrp_alloc(void) 677{ 678 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 679} 680 681/* 682 * Allocate a kse. 683 */ 684struct kse * 685kse_alloc(void) 686{ 687 return (uma_zalloc(kse_zone, M_WAITOK)); 688} 689 690/* 691 * Allocate a thread. 692 */ 693struct thread * 694thread_alloc(void) 695{ 696 thread_reap(); /* check if any zombies to get */ 697 return (uma_zalloc(thread_zone, M_WAITOK)); 698} 699 700/* 701 * Deallocate a ksegrp. 702 */ 703void 704ksegrp_free(struct ksegrp *td) 705{ 706 uma_zfree(ksegrp_zone, td); 707} 708 709/* 710 * Deallocate a kse. 711 */ 712void 713kse_free(struct kse *td) 714{ 715 uma_zfree(kse_zone, td); 716} 717 718/* 719 * Deallocate a thread. 720 */ 721void 722thread_free(struct thread *td) 723{ 724 uma_zfree(thread_zone, td); 725} 726 727/* 728 * Store the thread context in the UTS's mailbox. 729 * then add the mailbox at the head of a list we are building in user space. 730 * The list is anchored in the ksegrp structure. 731 */ 732int 733thread_export_context(struct thread *td) 734{ 735 struct proc *p; 736 struct ksegrp *kg; 737 uintptr_t mbx; 738 void *addr; 739 int error; 740 ucontext_t uc; 741 742 p = td->td_proc; 743 kg = td->td_ksegrp; 744 745 /* Export the user/machine context. */ 746#if 0 747 addr = (caddr_t)td->td_mailbox + 748 offsetof(struct kse_thr_mailbox, tm_context); 749#else /* if user pointer arithmetic is valid in the kernel */ 750 addr = (void *)(&td->td_mailbox->tm_context); 751#endif 752 error = copyin(addr, &uc, sizeof(ucontext_t)); 753 if (error == 0) { 754 thread_getcontext(td, &uc); 755 error = copyout(&uc, addr, sizeof(ucontext_t)); 756 757 } 758 if (error) { 759 PROC_LOCK(p); 760 psignal(p, SIGSEGV); 761 PROC_UNLOCK(p); 762 return (error); 763 } 764 /* get address in latest mbox of list pointer */ 765#if 0 766 addr = (caddr_t)td->td_mailbox 767 + offsetof(struct kse_thr_mailbox , tm_next); 768#else /* if user pointer arithmetic is valid in the kernel */ 769 addr = (void *)(&td->td_mailbox->tm_next); 770#endif 771 /* 772 * Put the saved address of the previous first 773 * entry into this one 774 */ 775 for (;;) { 776 mbx = (uintptr_t)kg->kg_completed; 777 if (suword(addr, mbx)) { 778 PROC_LOCK(p); 779 psignal(p, SIGSEGV); 780 PROC_UNLOCK(p); 781 return (EFAULT); 782 } 783 PROC_LOCK(p); 784 if (mbx == (uintptr_t)kg->kg_completed) { 785 kg->kg_completed = td->td_mailbox; 786 PROC_UNLOCK(p); 787 break; 788 } 789 PROC_UNLOCK(p); 790 } 791 return (0); 792} 793 794/* 795 * Take the list of completed mailboxes for this KSEGRP and put them on this 796 * KSE's mailbox as it's the next one going up. 797 */ 798static int 799thread_link_mboxes(struct ksegrp *kg, struct kse *ke) 800{ 801 struct proc *p = kg->kg_proc; 802 void *addr; 803 uintptr_t mbx; 804 805#if 0 806 addr = (caddr_t)ke->ke_mailbox 807 + offsetof(struct kse_mailbox, km_completed); 808#else /* if user pointer arithmetic is valid in the kernel */ 809 addr = (void *)(&ke->ke_mailbox->km_completed); 810#endif 811 for (;;) { 812 mbx = (uintptr_t)kg->kg_completed; 813 if (suword(addr, mbx)) { 814 PROC_LOCK(p); 815 psignal(p, SIGSEGV); 816 PROC_UNLOCK(p); 817 return (EFAULT); 818 } 819 /* XXXKSE could use atomic CMPXCH here */ 820 PROC_LOCK(p); 821 if (mbx == (uintptr_t)kg->kg_completed) { 822 kg->kg_completed = NULL; 823 PROC_UNLOCK(p); 824 break; 825 } 826 PROC_UNLOCK(p); 827 } 828 return (0); 829} 830 831/* 832 * Discard the current thread and exit from its context. 833 * 834 * Because we can't free a thread while we're operating under its context, 835 * push the current thread into our KSE's ke_tdspare slot, freeing the 836 * thread that might be there currently. Because we know that only this 837 * processor will run our KSE, we needn't worry about someone else grabbing 838 * our context before we do a cpu_throw. 839 */ 840void 841thread_exit(void) 842{ 843 struct thread *td; 844 struct kse *ke; 845 struct proc *p; 846 struct ksegrp *kg; 847 848 td = curthread; 849 kg = td->td_ksegrp; 850 p = td->td_proc; 851 ke = td->td_kse; 852 853 mtx_assert(&sched_lock, MA_OWNED); 854 KASSERT(p != NULL, ("thread exiting without a process")); 855 KASSERT(ke != NULL, ("thread exiting without a kse")); 856 KASSERT(kg != NULL, ("thread exiting without a kse group")); 857 PROC_LOCK_ASSERT(p, MA_OWNED); 858 CTR1(KTR_PROC, "thread_exit: thread %p", td); 859 KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 860 861 if (ke->ke_tdspare != NULL) { 862 thread_stash(ke->ke_tdspare); 863 ke->ke_tdspare = NULL; 864 } 865 if (td->td_standin != NULL) { 866 thread_stash(td->td_standin); 867 td->td_standin = NULL; 868 } 869 870 cpu_thread_exit(td); /* XXXSMP */ 871 872 /* 873 * The last thread is left attached to the process 874 * So that the whole bundle gets recycled. Skip 875 * all this stuff. 876 */ 877 if (p->p_numthreads > 1) { 878 /* 879 * Unlink this thread from its proc and the kseg. 880 * In keeping with the other structs we probably should 881 * have a thread_unlink() that does some of this but it 882 * would only be called from here (I think) so it would 883 * be a waste. (might be useful for proc_fini() as well.) 884 */ 885 TAILQ_REMOVE(&p->p_threads, td, td_plist); 886 p->p_numthreads--; 887 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 888 kg->kg_numthreads--; 889 /* 890 * The test below is NOT true if we are the 891 * sole exiting thread. P_STOPPED_SNGL is unset 892 * in exit1() after it is the only survivor. 893 */ 894 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 895 if (p->p_numthreads == p->p_suspcount) { 896 thread_unsuspend_one(p->p_singlethread); 897 } 898 } 899 900 /* Reassign this thread's KSE. */ 901 ke->ke_thread = NULL; 902 td->td_kse = NULL; 903 ke->ke_state = KES_UNQUEUED; 904 KASSERT((ke->ke_bound != td), 905 ("thread_exit: entered with ke_bound set")); 906 907 /* 908 * The reason for all this hoopla is 909 * an attempt to stop our thread stack from being freed 910 * until AFTER we have stopped running on it. 911 * Since we are under schedlock, almost any method where 912 * it is eventually freed by someone else is probably ok. 913 * (Especially if they do it under schedlock). We could 914 * almost free it here if we could be certain that 915 * the uma code wouldn't pull it apart immediatly, 916 * but unfortunatly we can not guarantee that. 917 * 918 * For threads that are exiting and NOT killing their 919 * KSEs we can just stash it in the KSE, however 920 * in the case where the KSE is also being deallocated, 921 * we need to store it somewhere else. It turns out that 922 * we will never free the last KSE, so there is always one 923 * other KSE available. We might as well just choose one 924 * and stash it there. Being under schedlock should make that 925 * safe. 926 * 927 * In borrower threads, we can stash it in the lender 928 * Where it won't be needed until this thread is long gone. 929 * Borrower threads can't kill their KSE anyhow, so even 930 * the KSE would be a safe place for them. It is not 931 * necessary to have a KSE (or KSEGRP) at all beyond this 932 * point, while we are under the protection of schedlock. 933 * 934 * Either give the KSE to another thread to use (or make 935 * it idle), or free it entirely, possibly along with its 936 * ksegrp if it's the last one. 937 */ 938 if (ke->ke_flags & KEF_EXIT) { 939 kse_unlink(ke); 940 /* 941 * Designate another KSE to hold our thread. 942 * Safe as long as we abide by whatever lock 943 * we control it with.. The other KSE will not 944 * be able to run it until we release the schelock, 945 * but we need to be careful about it deciding to 946 * write to the stack before then. Luckily 947 * I believe that while another thread's 948 * standin thread can be used in this way, the 949 * spare thread for the KSE cannot be used without 950 * holding schedlock at least once. 951 */ 952 ke = FIRST_KSE_IN_PROC(p); 953 } else { 954 kse_reassign(ke); 955 } 956 if (ke->ke_bound) { 957 /* 958 * WE are a borrower.. 959 * stash our thread with the owner. 960 */ 961 if (ke->ke_bound->td_standin) { 962 thread_stash(ke->ke_bound->td_standin); 963 } 964 ke->ke_bound->td_standin = td; 965 } else { 966 if (ke->ke_tdspare != NULL) { 967 thread_stash(ke->ke_tdspare); 968 ke->ke_tdspare = NULL; 969 } 970 ke->ke_tdspare = td; 971 } 972 PROC_UNLOCK(p); 973 td->td_state = TDS_INACTIVE; 974 td->td_proc = NULL; 975 td->td_ksegrp = NULL; 976 td->td_last_kse = NULL; 977 } else { 978 PROC_UNLOCK(p); 979 } 980 981 cpu_throw(); 982 /* NOTREACHED */ 983} 984 985/* 986 * Link a thread to a process. 987 * set up anything that needs to be initialized for it to 988 * be used by the process. 989 * 990 * Note that we do not link to the proc's ucred here. 991 * The thread is linked as if running but no KSE assigned. 992 */ 993void 994thread_link(struct thread *td, struct ksegrp *kg) 995{ 996 struct proc *p; 997 998 p = kg->kg_proc; 999 td->td_state = TDS_INACTIVE; 1000 td->td_proc = p; 1001 td->td_ksegrp = kg; 1002 td->td_last_kse = NULL; 1003 1004 LIST_INIT(&td->td_contested); 1005 callout_init(&td->td_slpcallout, 1); 1006 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 1007 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 1008 p->p_numthreads++; 1009 kg->kg_numthreads++; 1010 if (oiks_debug && p->p_numthreads > max_threads_per_proc) { 1011 printf("OIKS %d\n", p->p_numthreads); 1012 if (oiks_debug > 1) 1013 Debugger("OIKS"); 1014 } 1015 td->td_kse = NULL; 1016} 1017 1018void 1019kse_purge(struct proc *p, struct thread *td) 1020{ 1021 struct kse *ke; 1022 struct ksegrp *kg; 1023 1024 KASSERT(p->p_numthreads == 1, ("bad thread number")); 1025 mtx_lock_spin(&sched_lock); 1026 while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1027 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1028 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1029 kg->kg_idle_kses--; 1030 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1031 kg->kg_kses--; 1032 if (ke->ke_tdspare) 1033 thread_stash(ke->ke_tdspare); 1034 kse_stash(ke); 1035 } 1036 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1037 p->p_numksegrps--; 1038 KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1039 ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1040 ("wrong kg_kses")); 1041 if (kg != td->td_ksegrp) { 1042 ksegrp_stash(kg); 1043 } 1044 } 1045 TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1046 p->p_numksegrps++; 1047 mtx_unlock_spin(&sched_lock); 1048} 1049 1050 1051/* 1052 * Create a thread and schedule it for upcall on the KSE given. 1053 */ 1054struct thread * 1055thread_schedule_upcall(struct thread *td, struct kse *ke) 1056{ 1057 struct thread *td2; 1058 struct ksegrp *kg; 1059 int newkse; 1060 1061 mtx_assert(&sched_lock, MA_OWNED); 1062 newkse = (ke != td->td_kse); 1063 1064 /* 1065 * If the kse is already owned by another thread then we can't 1066 * schedule an upcall because the other thread must be BOUND 1067 * which means it is not in a position to take an upcall. 1068 * We must be borrowing the KSE to allow us to complete some in-kernel 1069 * work. When we complete, the Bound thread will have teh chance to 1070 * complete. This thread will sleep as planned. Hopefully there will 1071 * eventually be un unbound thread that can be converted to an 1072 * upcall to report the completion of this thread. 1073 */ 1074 if (ke->ke_bound && ((ke->ke_bound->td_flags & TDF_UNBOUND) == 0)) { 1075 return (NULL); 1076 } 1077 KASSERT((ke->ke_bound == NULL), ("kse already bound")); 1078 1079 if (ke->ke_state == KES_IDLE) { 1080 kg = ke->ke_ksegrp; 1081 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1082 kg->kg_idle_kses--; 1083 ke->ke_state = KES_UNQUEUED; 1084 } 1085 if ((td2 = td->td_standin) != NULL) { 1086 td->td_standin = NULL; 1087 } else { 1088 if (newkse) 1089 panic("no reserve thread when called with a new kse"); 1090 /* 1091 * If called from (e.g.) sleep and we do not have 1092 * a reserve thread, then we've used it, so do not 1093 * create an upcall. 1094 */ 1095 return (NULL); 1096 } 1097 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1098 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1099 bzero(&td2->td_startzero, 1100 (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1101 bcopy(&td->td_startcopy, &td2->td_startcopy, 1102 (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1103 thread_link(td2, ke->ke_ksegrp); 1104 cpu_set_upcall(td2, td->td_pcb); 1105 1106 /* 1107 * XXXKSE do we really need this? (default values for the 1108 * frame). 1109 */ 1110 bcopy(td->td_frame, td2->td_frame, sizeof(struct trapframe)); 1111 1112 /* 1113 * Bind the new thread to the KSE, 1114 * and if it's our KSE, lend it back to ourself 1115 * so we can continue running. 1116 */ 1117 td2->td_ucred = crhold(td->td_ucred); 1118 td2->td_flags = TDF_UPCALLING; /* note: BOUND */ 1119 td2->td_kse = ke; 1120 td2->td_state = TDS_CAN_RUN; 1121 td2->td_inhibitors = 0; 1122 /* 1123 * If called from msleep(), we are working on the current 1124 * KSE so fake that we borrowed it. If called from 1125 * kse_create(), don't, as we have a new kse too. 1126 */ 1127 if (!newkse) { 1128 /* 1129 * This thread will be scheduled when the current thread 1130 * blocks, exits or tries to enter userspace, (which ever 1131 * happens first). When that happens the KSe will "revert" 1132 * to this thread in a BOUND manner. Since we are called 1133 * from msleep() this is going to be "very soon" in nearly 1134 * all cases. 1135 */ 1136 ke->ke_bound = td2; 1137 TD_SET_LOAN(td2); 1138 } else { 1139 ke->ke_bound = NULL; 1140 ke->ke_thread = td2; 1141 ke->ke_state = KES_THREAD; 1142 setrunqueue(td2); 1143 } 1144 return (td2); /* bogus.. should be a void function */ 1145} 1146 1147/* 1148 * Schedule an upcall to notify a KSE process recieved signals. 1149 * 1150 * XXX - Modifying a sigset_t like this is totally bogus. 1151 */ 1152struct thread * 1153signal_upcall(struct proc *p, int sig) 1154{ 1155 struct thread *td, *td2; 1156 struct kse *ke; 1157 sigset_t ss; 1158 int error; 1159 1160 PROC_LOCK_ASSERT(p, MA_OWNED); 1161return (NULL); 1162 1163 td = FIRST_THREAD_IN_PROC(p); 1164 ke = td->td_kse; 1165 PROC_UNLOCK(p); 1166 error = copyin(&ke->ke_mailbox->km_sigscaught, &ss, sizeof(sigset_t)); 1167 PROC_LOCK(p); 1168 if (error) 1169 return (NULL); 1170 SIGADDSET(ss, sig); 1171 PROC_UNLOCK(p); 1172 error = copyout(&ss, &ke->ke_mailbox->km_sigscaught, sizeof(sigset_t)); 1173 PROC_LOCK(p); 1174 if (error) 1175 return (NULL); 1176 if (td->td_standin == NULL) 1177 td->td_standin = thread_alloc(); 1178 mtx_lock_spin(&sched_lock); 1179 td2 = thread_schedule_upcall(td, ke); /* Bogus JRE */ 1180 mtx_unlock_spin(&sched_lock); 1181 return (td2); 1182} 1183 1184/* 1185 * setup done on the thread when it enters the kernel. 1186 * XXXKSE Presently only for syscalls but eventually all kernel entries. 1187 */ 1188void 1189thread_user_enter(struct proc *p, struct thread *td) 1190{ 1191 struct kse *ke; 1192 1193 /* 1194 * First check that we shouldn't just abort. 1195 * But check if we are the single thread first! 1196 * XXX p_singlethread not locked, but should be safe. 1197 */ 1198 if ((p->p_flag & P_WEXIT) && (p->p_singlethread != td)) { 1199 PROC_LOCK(p); 1200 mtx_lock_spin(&sched_lock); 1201 thread_exit(); 1202 /* NOTREACHED */ 1203 } 1204 1205 /* 1206 * If we are doing a syscall in a KSE environment, 1207 * note where our mailbox is. There is always the 1208 * possibility that we could do this lazily (in sleep()), 1209 * but for now do it every time. 1210 */ 1211 ke = td->td_kse; 1212 if (ke->ke_mailbox != NULL) { 1213#if 0 1214 td->td_mailbox = (void *)fuword((caddr_t)ke->ke_mailbox 1215 + offsetof(struct kse_mailbox, km_curthread)); 1216#else /* if user pointer arithmetic is ok in the kernel */ 1217 td->td_mailbox = 1218 (void *)fuword( (void *)&ke->ke_mailbox->km_curthread); 1219#endif 1220 if ((td->td_mailbox == NULL) || 1221 (td->td_mailbox == (void *)-1)) { 1222 td->td_mailbox = NULL; /* single thread it.. */ 1223 td->td_flags &= ~TDF_UNBOUND; 1224 } else { 1225 if (td->td_standin == NULL) 1226 td->td_standin = thread_alloc(); 1227 td->td_flags |= TDF_UNBOUND; 1228 } 1229 } 1230} 1231 1232/* 1233 * The extra work we go through if we are a threaded process when we 1234 * return to userland. 1235 * 1236 * If we are a KSE process and returning to user mode, check for 1237 * extra work to do before we return (e.g. for more syscalls 1238 * to complete first). If we were in a critical section, we should 1239 * just return to let it finish. Same if we were in the UTS (in 1240 * which case the mailbox's context's busy indicator will be set). 1241 * The only traps we suport will have set the mailbox. 1242 * We will clear it here. 1243 */ 1244int 1245thread_userret(struct thread *td, struct trapframe *frame) 1246{ 1247 int error; 1248 int unbound; 1249 struct kse *ke; 1250 struct ksegrp *kg; 1251 struct thread *td2; 1252 struct proc *p; 1253 1254 error = 0; 1255 1256 unbound = td->td_flags & TDF_UNBOUND; 1257 1258 kg = td->td_ksegrp; 1259 p = td->td_proc; 1260 1261 /* 1262 * Originally bound threads never upcall but they may 1263 * loan out their KSE at this point. 1264 * Upcalls imply bound.. They also may want to do some Philantropy. 1265 * Unbound threads on the other hand either yield to other work 1266 * or transform into an upcall. 1267 * (having saved their context to user space in both cases) 1268 */ 1269 if (unbound) { 1270 /* 1271 * We are an unbound thread, looking to return to 1272 * user space. 1273 * THere are several possibilities: 1274 * 1) we are using a borrowed KSE. save state and exit. 1275 * kse_reassign() will recycle the kse as needed, 1276 * 2) we are not.. save state, and then convert ourself 1277 * to be an upcall, bound to the KSE. 1278 * if there are others that need the kse, 1279 * give them a chance by doing an mi_switch(). 1280 * Because we are bound, control will eventually return 1281 * to us here. 1282 * *** 1283 * Save the thread's context, and link it 1284 * into the KSEGRP's list of completed threads. 1285 */ 1286 error = thread_export_context(td); 1287 td->td_mailbox = NULL; 1288 if (error) { 1289 /* 1290 * If we are not running on a borrowed KSE, then 1291 * failing to do the KSE operation just defaults 1292 * back to synchonous operation, so just return from 1293 * the syscall. If it IS borrowed, there is nothing 1294 * we can do. We just lose that context. We 1295 * probably should note this somewhere and send 1296 * the process a signal. 1297 */ 1298 PROC_LOCK(td->td_proc); 1299 psignal(td->td_proc, SIGSEGV); 1300 mtx_lock_spin(&sched_lock); 1301 if (td->td_kse->ke_bound == NULL) { 1302 td->td_flags &= ~TDF_UNBOUND; 1303 PROC_UNLOCK(td->td_proc); 1304 mtx_unlock_spin(&sched_lock); 1305 return (error); /* go sync */ 1306 } 1307 thread_exit(); 1308 } 1309 1310 /* 1311 * if the KSE is owned and we are borrowing it, 1312 * don't make an upcall, just exit so that the owner 1313 * can get its KSE if it wants it. 1314 * Our context is already safely stored for later 1315 * use by the UTS. 1316 */ 1317 PROC_LOCK(p); 1318 mtx_lock_spin(&sched_lock); 1319 if (td->td_kse->ke_bound) { 1320 thread_exit(); 1321 } 1322 PROC_UNLOCK(p); 1323 1324 /* 1325 * Turn ourself into a bound upcall. 1326 * We will rely on kse_reassign() 1327 * to make us run at a later time. 1328 * We should look just like a sheduled upcall 1329 * from msleep() or cv_wait(). 1330 */ 1331 td->td_flags &= ~TDF_UNBOUND; 1332 td->td_flags |= TDF_UPCALLING; 1333 /* Only get here if we have become an upcall */ 1334 1335 } else { 1336 mtx_lock_spin(&sched_lock); 1337 } 1338 /* 1339 * We ARE going back to userland with this KSE. 1340 * Check for threads that need to borrow it. 1341 * Optimisation: don't call mi_switch if no-one wants the KSE. 1342 * Any other thread that comes ready after this missed the boat. 1343 */ 1344 ke = td->td_kse; 1345 if ((td2 = kg->kg_last_assigned)) 1346 td2 = TAILQ_NEXT(td2, td_runq); 1347 else 1348 td2 = TAILQ_FIRST(&kg->kg_runq); 1349 if (td2) { 1350 /* 1351 * force a switch to more urgent 'in kernel' 1352 * work. Control will return to this thread 1353 * when there is no more work to do. 1354 * kse_reassign() will do tha for us. 1355 */ 1356 TD_SET_LOAN(td); 1357 ke->ke_bound = td; 1358 ke->ke_thread = NULL; 1359 mi_switch(); /* kse_reassign() will (re)find td2 */ 1360 } 1361 mtx_unlock_spin(&sched_lock); 1362 1363 /* 1364 * Optimisation: 1365 * Ensure that we have a spare thread available, 1366 * for when we re-enter the kernel. 1367 */ 1368 if (td->td_standin == NULL) { 1369 if (ke->ke_tdspare) { 1370 td->td_standin = ke->ke_tdspare; 1371 ke->ke_tdspare = NULL; 1372 } else { 1373 td->td_standin = thread_alloc(); 1374 } 1375 } 1376 1377 /* 1378 * To get here, we know there is no other need for our 1379 * KSE so we can proceed. If not upcalling, go back to 1380 * userspace. If we are, get the upcall set up. 1381 */ 1382 if ((td->td_flags & TDF_UPCALLING) == 0) 1383 return (0); 1384 1385 /* 1386 * We must be an upcall to get this far. 1387 * There is no more work to do and we are going to ride 1388 * this thead/KSE up to userland as an upcall. 1389 * Do the last parts of the setup needed for the upcall. 1390 */ 1391 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1392 td, td->td_proc->p_pid, td->td_proc->p_comm); 1393 1394 /* 1395 * Set user context to the UTS. 1396 */ 1397 cpu_set_upcall_kse(td, ke); 1398 1399 /* 1400 * Put any completed mailboxes on this KSE's list. 1401 */ 1402 error = thread_link_mboxes(kg, ke); 1403 if (error) 1404 goto bad; 1405 1406 /* 1407 * Set state and mailbox. 1408 * From now on we are just a bound outgoing process. 1409 * **Problem** userret is often called several times. 1410 * it would be nice if this all happenned only on the first time 1411 * through. (the scan for extra work etc.) 1412 */ 1413 mtx_lock_spin(&sched_lock); 1414 td->td_flags &= ~TDF_UPCALLING; 1415 mtx_unlock_spin(&sched_lock); 1416#if 0 1417 error = suword((caddr_t)ke->ke_mailbox + 1418 offsetof(struct kse_mailbox, km_curthread), 0); 1419#else /* if user pointer arithmetic is ok in the kernel */ 1420 error = suword((caddr_t)&ke->ke_mailbox->km_curthread, 0); 1421#endif 1422 if (!error) 1423 return (0); 1424 1425bad: 1426 /* 1427 * Things are going to be so screwed we should just kill the process. 1428 * how do we do that? 1429 */ 1430 PROC_LOCK(td->td_proc); 1431 psignal(td->td_proc, SIGSEGV); 1432 PROC_UNLOCK(td->td_proc); 1433 return (error); /* go sync */ 1434} 1435 1436/* 1437 * Enforce single-threading. 1438 * 1439 * Returns 1 if the caller must abort (another thread is waiting to 1440 * exit the process or similar). Process is locked! 1441 * Returns 0 when you are successfully the only thread running. 1442 * A process has successfully single threaded in the suspend mode when 1443 * There are no threads in user mode. Threads in the kernel must be 1444 * allowed to continue until they get to the user boundary. They may even 1445 * copy out their return values and data before suspending. They may however be 1446 * accellerated in reaching the user boundary as we will wake up 1447 * any sleeping threads that are interruptable. (PCATCH). 1448 */ 1449int 1450thread_single(int force_exit) 1451{ 1452 struct thread *td; 1453 struct thread *td2; 1454 struct proc *p; 1455 1456 td = curthread; 1457 p = td->td_proc; 1458 PROC_LOCK_ASSERT(p, MA_OWNED); 1459 KASSERT((td != NULL), ("curthread is NULL")); 1460 1461 if ((p->p_flag & P_KSES) == 0) 1462 return (0); 1463 1464 /* Is someone already single threading? */ 1465 if (p->p_singlethread) 1466 return (1); 1467 1468 if (force_exit == SINGLE_EXIT) 1469 p->p_flag |= P_SINGLE_EXIT; 1470 else 1471 p->p_flag &= ~P_SINGLE_EXIT; 1472 p->p_flag |= P_STOPPED_SINGLE; 1473 p->p_singlethread = td; 1474 /* XXXKSE Which lock protects the below values? */ 1475 while ((p->p_numthreads - p->p_suspcount) != 1) { 1476 mtx_lock_spin(&sched_lock); 1477 FOREACH_THREAD_IN_PROC(p, td2) { 1478 if (td2 == td) 1479 continue; 1480 if (TD_IS_INHIBITED(td2)) { 1481 if (force_exit == SINGLE_EXIT) { 1482 if (TD_IS_SUSPENDED(td2)) { 1483 thread_unsuspend_one(td2); 1484 } 1485 if (TD_ON_SLEEPQ(td2) && 1486 (td2->td_flags & TDF_SINTR)) { 1487 if (td2->td_flags & TDF_CVWAITQ) 1488 cv_abort(td2); 1489 else 1490 abortsleep(td2); 1491 } 1492 } else { 1493 if (TD_IS_SUSPENDED(td2)) 1494 continue; 1495 /* maybe other inhibitted states too? */ 1496 if (TD_IS_SLEEPING(td2)) 1497 thread_suspend_one(td2); 1498 } 1499 } 1500 } 1501 /* 1502 * Maybe we suspended some threads.. was it enough? 1503 */ 1504 if ((p->p_numthreads - p->p_suspcount) == 1) { 1505 mtx_unlock_spin(&sched_lock); 1506 break; 1507 } 1508 1509 /* 1510 * Wake us up when everyone else has suspended. 1511 * In the mean time we suspend as well. 1512 */ 1513 thread_suspend_one(td); 1514 mtx_unlock(&Giant); 1515 PROC_UNLOCK(p); 1516 mi_switch(); 1517 mtx_unlock_spin(&sched_lock); 1518 mtx_lock(&Giant); 1519 PROC_LOCK(p); 1520 } 1521 if (force_exit == SINGLE_EXIT) 1522 kse_purge(p, td); 1523 return (0); 1524} 1525 1526/* 1527 * Called in from locations that can safely check to see 1528 * whether we have to suspend or at least throttle for a 1529 * single-thread event (e.g. fork). 1530 * 1531 * Such locations include userret(). 1532 * If the "return_instead" argument is non zero, the thread must be able to 1533 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 1534 * 1535 * The 'return_instead' argument tells the function if it may do a 1536 * thread_exit() or suspend, or whether the caller must abort and back 1537 * out instead. 1538 * 1539 * If the thread that set the single_threading request has set the 1540 * P_SINGLE_EXIT bit in the process flags then this call will never return 1541 * if 'return_instead' is false, but will exit. 1542 * 1543 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 1544 *---------------+--------------------+--------------------- 1545 * 0 | returns 0 | returns 0 or 1 1546 * | when ST ends | immediatly 1547 *---------------+--------------------+--------------------- 1548 * 1 | thread exits | returns 1 1549 * | | immediatly 1550 * 0 = thread_exit() or suspension ok, 1551 * other = return error instead of stopping the thread. 1552 * 1553 * While a full suspension is under effect, even a single threading 1554 * thread would be suspended if it made this call (but it shouldn't). 1555 * This call should only be made from places where 1556 * thread_exit() would be safe as that may be the outcome unless 1557 * return_instead is set. 1558 */ 1559int 1560thread_suspend_check(int return_instead) 1561{ 1562 struct thread *td; 1563 struct proc *p; 1564 struct kse *ke; 1565 struct ksegrp *kg; 1566 1567 td = curthread; 1568 p = td->td_proc; 1569 kg = td->td_ksegrp; 1570 PROC_LOCK_ASSERT(p, MA_OWNED); 1571 while (P_SHOULDSTOP(p)) { 1572 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1573 KASSERT(p->p_singlethread != NULL, 1574 ("singlethread not set")); 1575 /* 1576 * The only suspension in action is a 1577 * single-threading. Single threader need not stop. 1578 * XXX Should be safe to access unlocked 1579 * as it can only be set to be true by us. 1580 */ 1581 if (p->p_singlethread == td) 1582 return (0); /* Exempt from stopping. */ 1583 } 1584 if (return_instead) 1585 return (1); 1586 1587 /* 1588 * If the process is waiting for us to exit, 1589 * this thread should just suicide. 1590 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 1591 */ 1592 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1593 mtx_lock_spin(&sched_lock); 1594 while (mtx_owned(&Giant)) 1595 mtx_unlock(&Giant); 1596 /* 1597 * free extra kses and ksegrps, we needn't worry 1598 * about if current thread is in same ksegrp as 1599 * p_singlethread and last kse in the group 1600 * could be killed, this is protected by kg_numthreads, 1601 * in this case, we deduce that kg_numthreads must > 1. 1602 */ 1603 ke = td->td_kse; 1604 if (ke->ke_bound == NULL && 1605 ((kg->kg_kses != 1) || (kg->kg_numthreads == 1))) 1606 ke->ke_flags |= KEF_EXIT; 1607 thread_exit(); 1608 } 1609 1610 /* 1611 * When a thread suspends, it just 1612 * moves to the processes's suspend queue 1613 * and stays there. 1614 * 1615 * XXXKSE if TDF_BOUND is true 1616 * it will not release it's KSE which might 1617 * lead to deadlock if there are not enough KSEs 1618 * to complete all waiting threads. 1619 * Maybe be able to 'lend' it out again. 1620 * (lent kse's can not go back to userland?) 1621 * and can only be lent in STOPPED state. 1622 */ 1623 mtx_lock_spin(&sched_lock); 1624 if ((p->p_flag & P_STOPPED_SIG) && 1625 (p->p_suspcount+1 == p->p_numthreads)) { 1626 mtx_unlock_spin(&sched_lock); 1627 PROC_LOCK(p->p_pptr); 1628 if ((p->p_pptr->p_procsig->ps_flag & 1629 PS_NOCLDSTOP) == 0) { 1630 psignal(p->p_pptr, SIGCHLD); 1631 } 1632 PROC_UNLOCK(p->p_pptr); 1633 mtx_lock_spin(&sched_lock); 1634 } 1635 mtx_assert(&Giant, MA_NOTOWNED); 1636 thread_suspend_one(td); 1637 PROC_UNLOCK(p); 1638 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1639 if (p->p_numthreads == p->p_suspcount) { 1640 thread_unsuspend_one(p->p_singlethread); 1641 } 1642 } 1643 p->p_stats->p_ru.ru_nivcsw++; 1644 mi_switch(); 1645 mtx_unlock_spin(&sched_lock); 1646 PROC_LOCK(p); 1647 } 1648 return (0); 1649} 1650 1651void 1652thread_suspend_one(struct thread *td) 1653{ 1654 struct proc *p = td->td_proc; 1655 1656 mtx_assert(&sched_lock, MA_OWNED); 1657 p->p_suspcount++; 1658 TD_SET_SUSPENDED(td); 1659 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 1660 /* 1661 * Hack: If we are suspending but are on the sleep queue 1662 * then we are in msleep or the cv equivalent. We 1663 * want to look like we have two Inhibitors. 1664 * May already be set.. doesn't matter. 1665 */ 1666 if (TD_ON_SLEEPQ(td)) 1667 TD_SET_SLEEPING(td); 1668} 1669 1670void 1671thread_unsuspend_one(struct thread *td) 1672{ 1673 struct proc *p = td->td_proc; 1674 1675 mtx_assert(&sched_lock, MA_OWNED); 1676 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 1677 TD_CLR_SUSPENDED(td); 1678 p->p_suspcount--; 1679 setrunnable(td); 1680} 1681 1682/* 1683 * Allow all threads blocked by single threading to continue running. 1684 */ 1685void 1686thread_unsuspend(struct proc *p) 1687{ 1688 struct thread *td; 1689 1690 mtx_assert(&sched_lock, MA_OWNED); 1691 PROC_LOCK_ASSERT(p, MA_OWNED); 1692 if (!P_SHOULDSTOP(p)) { 1693 while (( td = TAILQ_FIRST(&p->p_suspended))) { 1694 thread_unsuspend_one(td); 1695 } 1696 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 1697 (p->p_numthreads == p->p_suspcount)) { 1698 /* 1699 * Stopping everything also did the job for the single 1700 * threading request. Now we've downgraded to single-threaded, 1701 * let it continue. 1702 */ 1703 thread_unsuspend_one(p->p_singlethread); 1704 } 1705} 1706 1707void 1708thread_single_end(void) 1709{ 1710 struct thread *td; 1711 struct proc *p; 1712 1713 td = curthread; 1714 p = td->td_proc; 1715 PROC_LOCK_ASSERT(p, MA_OWNED); 1716 p->p_flag &= ~P_STOPPED_SINGLE; 1717 p->p_singlethread = NULL; 1718 /* 1719 * If there are other threads they mey now run, 1720 * unless of course there is a blanket 'stop order' 1721 * on the process. The single threader must be allowed 1722 * to continue however as this is a bad place to stop. 1723 */ 1724 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 1725 mtx_lock_spin(&sched_lock); 1726 while (( td = TAILQ_FIRST(&p->p_suspended))) { 1727 thread_unsuspend_one(td); 1728 } 1729 mtx_unlock_spin(&sched_lock); 1730 } 1731} 1732 1733 1734