kern_switch.c revision 139804
14Srgrimes/*- 2549Srgrimes * Copyright (c) 2001 Jake Burkholder <jake@FreeBSD.org> 34Srgrimes * All rights reserved. 44Srgrimes * 54Srgrimes * Redistribution and use in source and binary forms, with or without 64Srgrimes * modification, are permitted provided that the following conditions 74Srgrimes * are met: 84Srgrimes * 1. Redistributions of source code must retain the above copyright 94Srgrimes * notice, this list of conditions and the following disclaimer. 104Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 114Srgrimes * notice, this list of conditions and the following disclaimer in the 124Srgrimes * documentation and/or other materials provided with the distribution. 134Srgrimes * 144Srgrimes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 154Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 164Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 174Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 184Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 194Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 204Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 214Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 224Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 234Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 244Srgrimes * SUCH DAMAGE. 254Srgrimes */ 264Srgrimes 274Srgrimes/*** 284SrgrimesHere is the logic.. 294Srgrimes 304SrgrimesIf there are N processors, then there are at most N KSEs (kernel 314Srgrimesschedulable entities) working to process threads that belong to a 324SrgrimesKSEGROUP (kg). If there are X of these KSEs actually running at the 334Srgrimesmoment in question, then there are at most M (N-X) of these KSEs on 344Srgrimesthe run queue, as running KSEs are not on the queue. 354Srgrimes 364SrgrimesRunnable threads are queued off the KSEGROUP in priority order. 37549SrgrimesIf there are M or more threads runnable, the top M threads 3812186Sphk(by priority) are 'preassigned' to the M KSEs not running. The KSEs take 394Srgrimestheir priority from those threads and are put on the run queue. 404Srgrimes 41556SrgrimesThe last thread that had a priority high enough to have a KSE associated 42556Srgrimeswith it, AND IS ON THE RUN QUEUE is pointed to by 434Srgrimeskg->kg_last_assigned. If no threads queued off the KSEGROUP have KSEs 441549Srgrimesassigned as all the available KSEs are activly running, or because there 451549Srgrimesare no threads queued, that pointer is NULL. 4611390Sbde 471549SrgrimesWhen a KSE is removed from the run queue to become runnable, we know 481549Srgrimesit was associated with the highest priority thread in the queue (at the head 491549Srgrimesof the queue). If it is also the last assigned we know M was 1 and must 501549Srgrimesnow be 0. Since the thread is no longer queued that pointer must be 511549Srgrimesremoved from it. Since we know there were no more KSEs available, 521549Srgrimes(M was 1 and is now 0) and since we are not FREEING our KSE 531549Srgrimesbut using it, we know there are STILL no more KSEs available, we can prove 541549Srgrimesthat the next thread in the ksegrp list will not have a KSE to assign to 551549Srgrimesit, so we can show that the pointer must be made 'invalid' (NULL). 561549Srgrimes 571549SrgrimesThe pointer exists so that when a new thread is made runnable, it can 587090Sbdehave its priority compared with the last assigned thread to see if 591549Srgrimesit should 'steal' its KSE or not.. i.e. is it 'earlier' 601549Srgrimeson the list than that thread or later.. If it's earlier, then the KSE is 612254Ssosremoved from the last assigned (which is now not assigned a KSE) 621549Srgrimesand reassigned to the new thread, which is placed earlier in the list. 631549SrgrimesThe pointer is then backed up to the previous thread (which may or may not 647103Sdgbe the new thread). 654Srgrimes 66556SrgrimesWhen a thread sleeps or is removed, the KSE becomes available and if there 672056Swollmanare queued threads that are not assigned KSEs, the highest priority one of 68556Srgrimesthem is assigned the KSE, which is then placed back on the run queue at 69556Srgrimesthe approipriate place, and the kg->kg_last_assigned pointer is adjusted down 70990Sdgto point to it. 712056Swollman 72990SdgThe following diagram shows 2 KSEs and 3 threads from a single process. 73990Sdg 74990Sdg RUNQ: --->KSE---KSE--... (KSEs queued at priorities from threads) 752056Swollman \ \____ 76990Sdg \ \ 77990Sdg KSEGROUP---thread--thread--thread (queued in priority order) 782056Swollman \ / 792056Swollman \_______________/ 802056Swollman (last_assigned) 819507Sdg 824SrgrimesThe result of this scheme is that the M available KSEs are always 832056Swollmanqueued at the priorities they have inherrited from the M highest priority 842056Swollmanthreads for that KSEGROUP. If this situation changes, the KSEs are 85556Srgrimesreassigned to keep this true. 867090Sbde***/ 877090Sbde 882772Swollman#include <sys/cdefs.h> 892772Swollman__FBSDID("$FreeBSD: head/sys/kern/kern_switch.c 139804 2005-01-06 23:35:40Z imp $"); 907090Sbde 917090Sbde#include "opt_sched.h" 927090Sbde 934Srgrimes#ifndef KERN_SWITCH_INCLUDE 947090Sbde#include <sys/param.h> 957090Sbde#include <sys/systm.h> 967090Sbde#include <sys/kdb.h> 977090Sbde#include <sys/kernel.h> 987090Sbde#include <sys/ktr.h> 997090Sbde#include <sys/lock.h> 1007090Sbde#include <sys/mutex.h> 1017090Sbde#include <sys/proc.h> 1027090Sbde#include <sys/queue.h> 1037090Sbde#include <sys/sched.h> 1047090Sbde#else /* KERN_SWITCH_INCLUDE */ 1057090Sbde#if defined(SMP) && (defined(__i386__) || defined(__amd64__)) 1067090Sbde#include <sys/smp.h> 1077090Sbde#endif 1087090Sbde#include <machine/critical.h> 1097090Sbde#if defined(SMP) && defined(SCHED_4BSD) 1107090Sbde#include <sys/sysctl.h> 1117090Sbde#endif 1124193Sbde 1134193Sbde#ifdef FULL_PREEMPTION 1142056Swollman#ifndef PREEMPTION 1154193Sbde#error "The FULL_PREEMPTION option requires the PREEMPTION option" 1162056Swollman#endif 1172056Swollman#endif 1184193Sbde 1192056SwollmanCTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS); 1202056Swollman 1212056Swollman#define td_kse td_sched 1223703Swollman 1234819Sphk/************************************************************************ 1247090Sbde * Functions that manipulate runnability from a thread perspective. * 125556Srgrimes ************************************************************************/ 1262056Swollman/* 1274193Sbde * Select the KSE that will be run next. From that find the thread, and 1282056Swollman * remove it from the KSEGRP's run queue. If there is thread clustering, 12911875Smarkm * this will be what does it. 1304Srgrimes */ 13111390Sbdestruct thread * 13211390Sbdechoosethread(void) 13311390Sbde{ 13411390Sbde struct kse *ke; 13511390Sbde struct thread *td; 13611390Sbde struct ksegrp *kg; 13711390Sbde 13810358Sjulian#if defined(SMP) && (defined(__i386__) || defined(__amd64__)) 13910653Sdg if (smp_active == 0 && PCPU_GET(cpuid) != 0) { 14010358Sjulian /* Shutting down, run idlethread on AP's */ 14110358Sjulian td = PCPU_GET(idlethread); 142798Swollman ke = td->td_kse; 1434Srgrimes CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td); 1441999Swollman ke->ke_flags |= KEF_DIDRUN; 1457780Swollman TD_SET_RUNNING(td); 1461127Sdg return (td); 1477780Swollman } 1487780Swollman#endif 1497780Swollman 1507780Swollmanretry: 1517780Swollman ke = sched_choose(); 1527780Swollman if (ke) { 1537780Swollman td = ke->ke_thread; 1547780Swollman KASSERT((td->td_kse == ke), ("kse/thread mismatch")); 1557780Swollman kg = ke->ke_ksegrp; 1567780Swollman if (td->td_proc->p_flag & P_HADTHREADS) { 1577780Swollman if (kg->kg_last_assigned == td) { 158757Sdg kg->kg_last_assigned = TAILQ_PREV(td, 159757Sdg threadqueue, td_runq); 160757Sdg } 161757Sdg TAILQ_REMOVE(&kg->kg_runq, td, td_runq); 1622426Sdg } 1632426Sdg CTR2(KTR_RUNQ, "choosethread: td=%p pri=%d", 1642426Sdg td, td->td_priority); 1651298Sdg } else { 1661298Sdg /* Simulate runq_choose() having returned the idle thread */ 1671298Sdg td = PCPU_GET(idlethread); 1681298Sdg ke = td->td_kse; 1691298Sdg CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td); 1702426Sdg } 1712426Sdg ke->ke_flags |= KEF_DIDRUN; 1722426Sdg 1731549Srgrimes /* 174556Srgrimes * If we are in panic, only allow system threads, 175556Srgrimes * plus the one we are running in, to be run. 1762818Sache */ 1774Srgrimes if (panicstr && ((td->td_proc->p_flag & P_SYSTEM) == 0 && 1784Srgrimes (td->td_flags & TDF_INPANIC) == 0)) { 1794Srgrimes /* note that it is no longer on the run queue */ 1804819Sphk TD_SET_CAN_RUN(td); 1814Srgrimes goto retry; 1824Srgrimes } 1834Srgrimes 1844Srgrimes TD_SET_RUNNING(td); 1859744Sdg return (td); 186974Sdg} 1879578Sdg 1889578Sdg/* 1899578Sdg * Given a surplus system slot, try assign a new runnable thread to it. 190556Srgrimes * Called from: 191556Srgrimes * sched_thread_exit() (local) 192556Srgrimes * sched_switch() (local) 1938426Swollman * sched_thread_exit() (local) 1948426Swollman * remrunqueue() (local) (not at the moment) 1951549Srgrimes */ 1961549Srgrimesstatic void 1971549Srgrimesslot_fill(struct ksegrp *kg) 1988426Swollman{ 199556Srgrimes struct thread *td; 2001549Srgrimes 2011549Srgrimes mtx_assert(&sched_lock, MA_OWNED); 20210358Sjulian while (kg->kg_avail_opennings > 0) { 20311390Sbde /* 20411390Sbde * Find the first unassigned thread 2054Srgrimes */ 2064Srgrimes if ((td = kg->kg_last_assigned) != NULL) 2074Srgrimes td = TAILQ_NEXT(td, td_runq); 2083489Sphk else 209798Swollman td = TAILQ_FIRST(&kg->kg_runq); 2109578Sdg 2113502Sphk /* 2124Srgrimes * If we found one, send it to the system scheduler. 2134819Sphk */ 2144819Sphk if (td) { 2154819Sphk kg->kg_last_assigned = td; 2164Srgrimes sched_add(td, SRQ_YIELDING); 2174Srgrimes CTR2(KTR_RUNQ, "slot_fill: td%p -> kg%p", td, kg); 2184Srgrimes } else { 2194Srgrimes /* no threads to use up the slots. quit now */ 2201055Sdg break; 2214Srgrimes } 222879Swollman } 223879Swollman} 2244Srgrimes 2254Srgrimes#ifdef SCHED_4BSD 2264Srgrimes/* 2274Srgrimes * Remove a thread from its KSEGRP's run queue. 2284Srgrimes * This in turn may remove it from a KSE if it was already assigned 2294Srgrimes * to one, possibly causing a new thread to be assigned to the KSE 230556Srgrimes * and the KSE getting a new priority. 2312014Swollman */ 232556Srgrimesstatic void 23310616Sdgremrunqueue(struct thread *td) 2349578Sdg{ 2359578Sdg struct thread *td2, *td3; 2369578Sdg struct ksegrp *kg; 2379578Sdg struct kse *ke; 2389578Sdg 2394Srgrimes mtx_assert(&sched_lock, MA_OWNED); 2409578Sdg KASSERT((TD_ON_RUNQ(td)), ("remrunqueue: Bad state on run queue")); 2419578Sdg kg = td->td_ksegrp; 2429578Sdg ke = td->td_kse; 2439578Sdg CTR1(KTR_RUNQ, "remrunqueue: td%p", td); 2449578Sdg TD_SET_CAN_RUN(td); 2459578Sdg /* 2469578Sdg * If it is not a threaded process, take the shortcut. 2479578Sdg */ 2489578Sdg if ((td->td_proc->p_flag & P_HADTHREADS) == 0) { 2499578Sdg /* remve from sys run queue and free up a slot */ 2509578Sdg sched_rem(td); 2519578Sdg ke->ke_state = KES_THREAD; 2529578Sdg return; 2539578Sdg } 2549578Sdg td3 = TAILQ_PREV(td, threadqueue, td_runq); 2554Srgrimes TAILQ_REMOVE(&kg->kg_runq, td, td_runq); 2562772Swollman if (ke->ke_state == KES_ONRUNQ) { 2572772Swollman /* 2588426Swollman * This thread has been assigned to the system run queue. 2598426Swollman * We need to dissociate it and try assign the 2608426Swollman * KSE to the next available thread. Then, we should 2615413Sse * see if we need to move the KSE in the run queues. 2625413Sse */ 2635413Sse sched_rem(td); 2648426Swollman ke->ke_state = KES_THREAD; 2652772Swollman td2 = kg->kg_last_assigned; 2662772Swollman KASSERT((td2 != NULL), ("last assigned has wrong value")); 2674Srgrimes if (td2 == td) 2684Srgrimes kg->kg_last_assigned = td3; 2694Srgrimes /* slot_fill(kg); */ /* will replace it with another */ 2704Srgrimes } 2714Srgrimes} 2724Srgrimes#endif 2734Srgrimes 2744Srgrimes/* 2754Srgrimes * Change the priority of a thread that is on the run queue. 2764Srgrimes */ 2774Srgrimesvoid 2784Srgrimesadjustrunqueue( struct thread *td, int newpri) 2794Srgrimes{ 2804Srgrimes struct ksegrp *kg; 2814Srgrimes struct kse *ke; 2824Srgrimes 2834Srgrimes mtx_assert(&sched_lock, MA_OWNED); 2844Srgrimes KASSERT((TD_ON_RUNQ(td)), ("adjustrunqueue: Bad state on run queue")); 2854Srgrimes 2864Srgrimes ke = td->td_kse; 2874Srgrimes CTR1(KTR_RUNQ, "adjustrunqueue: td%p", td); 2884Srgrimes /* 2894Srgrimes * If it is not a threaded process, take the shortcut. 2904Srgrimes */ 2914Srgrimes if ((td->td_proc->p_flag & P_HADTHREADS) == 0) { 2924Srgrimes /* We only care about the kse in the run queue. */ 293990Sdg td->td_priority = newpri; 294990Sdg if (ke->ke_rqindex != (newpri / RQ_PPQ)) { 295990Sdg sched_rem(td); 296990Sdg sched_add(td, SRQ_BORING); 297990Sdg } 298990Sdg return; 299990Sdg } 300990Sdg 301990Sdg /* It is a threaded process */ 302990Sdg kg = td->td_ksegrp; 303990Sdg if (ke->ke_state == KES_ONRUNQ) { 304990Sdg if (kg->kg_last_assigned == td) { 3054Srgrimes kg->kg_last_assigned = 3065837Sdg TAILQ_PREV(td, threadqueue, td_runq); 3076327Sdg } 3085837Sdg sched_rem(td); 3096949Sdg } 3105837Sdg TAILQ_REMOVE(&kg->kg_runq, td, td_runq); 3115837Sdg TD_SET_CAN_RUN(td); 3125455Sdg td->td_priority = newpri; 3134Srgrimes setrunqueue(td, SRQ_BORING); 3144Srgrimes} 3154Srgrimes 3162422Sdg/* 3174Srgrimes * This function is called when a thread is about to be put on a 3181298Sdg * ksegrp run queue because it has been made runnable or its 3191298Sdg * priority has been adjusted and the ksegrp does not have a 3201298Sdg * free kse slot. It determines if a thread from the same ksegrp 3215455Sdg * should be preempted. If so, it tries to switch threads 3225455Sdg * if the thread is on the same cpu or notifies another cpu that 3235455Sdg * it should switch threads. 3245455Sdg */ 3251298Sdg 3261298Sdgstatic void 3271298Sdgmaybe_preempt_in_ksegrp(struct thread *td) 3281298Sdg#if !defined(SMP) 3291298Sdg{ 3301298Sdg struct thread *running_thread; 3314Srgrimes 3324Srgrimes#ifndef FULL_PREEMPTION 3334Srgrimes int pri; 3344Srgrimes pri = td->td_priority; 3354Srgrimes if (!(pri >= PRI_MIN_ITHD && pri <= PRI_MAX_ITHD)) 3364Srgrimes return; 3374Srgrimes#endif 3384Srgrimes mtx_assert(&sched_lock, MA_OWNED); 3394Srgrimes running_thread = curthread; 3401321Sdg 3414Srgrimes if (running_thread->td_ksegrp != td->td_ksegrp) 3424Srgrimes return; 3434Srgrimes 3444Srgrimes if (td->td_priority > running_thread->td_priority) 3454Srgrimes return; 346556Srgrimes#ifdef PREEMPTION 3472426Sdg if (running_thread->td_critnest > 1) 3481549Srgrimes running_thread->td_pflags |= TDP_OWEPREEMPT; 3491887Sdg else 3501887Sdg mi_switch(SW_INVOL, NULL); 3512426Sdg 3522426Sdg#else 3532426Sdg running_thread->td_flags |= TDF_NEEDRESCHED; 3542426Sdg#endif 3552426Sdg return; 3561887Sdg} 3571887Sdg 3581887Sdg#else /* SMP */ 3591887Sdg{ 3605455Sdg struct thread *running_thread; 3615455Sdg int worst_pri; 3625455Sdg struct ksegrp *kg; 3635455Sdg cpumask_t cpumask,dontuse; 3641549Srgrimes struct pcpu *pc; 3654Srgrimes struct pcpu *best_pcpu; 3664Srgrimes struct thread *cputhread; 3674Srgrimes 3684Srgrimes#ifndef FULL_PREEMPTION 3698748Sdg int pri; 3704Srgrimes pri = td->td_priority; 3718748Sdg if (!(pri >= PRI_MIN_ITHD && pri <= PRI_MAX_ITHD)) 3721549Srgrimes return; 3738748Sdg#endif 3744Srgrimes 3754Srgrimes mtx_assert(&sched_lock, MA_OWNED); 3764Srgrimes 3774Srgrimes running_thread = curthread; 3784Srgrimes 3794Srgrimes#if !defined(KSEG_PEEMPT_BEST_CPU) 3804475Sbde if (running_thread->td_ksegrp != td->td_ksegrp) { 38110666Sbde#endif 3823907Sjkh kg = td->td_ksegrp; 38310666Sbde 38410666Sbde /* if someone is ahead of this thread, wait our turn */ 3854Srgrimes if (td != TAILQ_FIRST(&kg->kg_runq)) 3862422Sdg return; 3874Srgrimes 3881298Sdg worst_pri = td->td_priority; 3891298Sdg best_pcpu = NULL; 3901298Sdg dontuse = stopped_cpus | idle_cpus_mask; 3911298Sdg 39210782Sdg /* 39310782Sdg * Find a cpu with the worst priority that runs at thread from 39410782Sdg * the same ksegrp - if multiple exist give first the last run 39510782Sdg * cpu and then the current cpu priority 39610782Sdg */ 39710782Sdg 39810782Sdg SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 39910782Sdg cpumask = pc->pc_cpumask; 40010782Sdg cputhread = pc->pc_curthread; 40110616Sdg 40210616Sdg if ((cpumask & dontuse) || 4031298Sdg cputhread->td_ksegrp != kg) 4041298Sdg continue; 4054Srgrimes 4064Srgrimes if (cputhread->td_priority > worst_pri) { 4074Srgrimes worst_pri = cputhread->td_priority; 4081887Sdg best_pcpu = pc; 4094Srgrimes continue; 4104Srgrimes } 41111390Sbde 4124Srgrimes if (cputhread->td_priority == worst_pri && 4134819Sphk best_pcpu != NULL && 4147908Sphk (td->td_lastcpu == pc->pc_cpuid || 41511390Sbde (PCPU_GET(cpumask) == cpumask && 41611390Sbde td->td_lastcpu != best_pcpu->pc_cpuid))) 41711390Sbde best_pcpu = pc; 41811390Sbde } 41911390Sbde 42011390Sbde /* Check if we need to preempt someone */ 42111390Sbde if (best_pcpu == NULL) 42211390Sbde return; 42311390Sbde 42411390Sbde if (PCPU_GET(cpuid) != best_pcpu->pc_cpuid) { 42511390Sbde best_pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED; 42611390Sbde ipi_selected(best_pcpu->pc_cpumask, IPI_AST); 4277908Sphk return; 42811390Sbde } 42911390Sbde#if !defined(KSEG_PEEMPT_BEST_CPU) 43011390Sbde } 43111390Sbde#endif 43211390Sbde 43311390Sbde if (td->td_priority > running_thread->td_priority) 43411390Sbde return; 43511390Sbde#ifdef PREEMPTION 43611390Sbde if (running_thread->td_critnest > 1) 43711390Sbde running_thread->td_pflags |= TDP_OWEPREEMPT; 4387908Sphk else 4395908Sbde mi_switch(SW_INVOL, NULL); 4404819Sphk 4414Srgrimes#else 4424Srgrimes running_thread->td_flags |= TDF_NEEDRESCHED; 44311963Speter#endif 44411963Speter return; 44511963Speter} 44611963Speter#endif /* !SMP */ 44711963Speter 44811963Speter 44911963Speterint limitcount; 45011963Spetervoid 45111963Spetersetrunqueue(struct thread *td, int flags) 45211963Speter{ 45311963Speter struct ksegrp *kg; 45411963Speter struct thread *td2; 45511963Speter struct thread *tda; 45611963Speter 4578426Swollman CTR3(KTR_RUNQ, "setrunqueue: td:%p kg:%p pid:%d", 45811963Speter td, td->td_ksegrp, td->td_proc->p_pid); 45911963Speter CTR5(KTR_SCHED, "setrunqueue: %p(%s) prio %d by %p(%s)", 4608426Swollman td, td->td_proc->p_comm, td->td_priority, curthread, 4618426Swollman curthread->td_proc->p_comm); 4628426Swollman mtx_assert(&sched_lock, MA_OWNED); 463556Srgrimes KASSERT((td->td_inhibitors == 0), 4648426Swollman ("setrunqueue: trying to run inhibitted thread")); 4658426Swollman KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)), 46611963Speter ("setrunqueue: bad thread state")); 4678426Swollman TD_SET_RUNQ(td); 4688426Swollman kg = td->td_ksegrp; 4698426Swollman if ((td->td_proc->p_flag & P_HADTHREADS) == 0) { 470556Srgrimes /* 471556Srgrimes * Common path optimisation: Only one of everything 472556Srgrimes * and the KSE is always already attached. 473556Srgrimes * Totally ignore the ksegrp run queue. 474556Srgrimes */ 475556Srgrimes if (kg->kg_avail_opennings != 1) { 4761998Swollman if (limitcount < 1) { 4772495Spst limitcount++; 478556Srgrimes printf("pid %d: corrected slot count (%d->1)\n", 479556Srgrimes td->td_proc->p_pid, kg->kg_avail_opennings); 480798Swollman 481988Sdg } 482556Srgrimes kg->kg_avail_opennings = 1; 483556Srgrimes } 4848876Srgrimes sched_add(td, flags); 4851998Swollman return; 486556Srgrimes } 4871998Swollman 488556Srgrimes /* 489556Srgrimes * If the concurrency has reduced, and we would go in the 490556Srgrimes * assigned section, then keep removing entries from the 491556Srgrimes * system run queue, until we are not in that section 4927780Swollman * or there is room for us to be put in that section. 4937814Swpaul * What we MUST avoid is the case where there are threads of less 4947780Swollman * priority than the new one scheduled, but it can not 4957780Swollman * be scheduled itself. That would lead to a non contiguous set 4967780Swollman * of scheduled threads, and everything would break. 4977780Swollman */ 4987780Swollman tda = kg->kg_last_assigned; 4997814Swpaul while ((kg->kg_avail_opennings <= 0) && 5007780Swollman (tda && (tda->td_priority > td->td_priority))) { 5017780Swollman /* 5027780Swollman * None free, but there is one we can commandeer. 5037780Swollman */ 5047780Swollman CTR2(KTR_RUNQ, 5058876Srgrimes "setrunqueue: kg:%p: take slot from td: %p", kg, tda); 5068876Srgrimes sched_rem(tda); 5077780Swollman tda = kg->kg_last_assigned = 5087780Swollman TAILQ_PREV(tda, threadqueue, td_runq); 5097780Swollman } 5107780Swollman 5117780Swollman /* 5127780Swollman * Add the thread to the ksegrp's run queue at 5137780Swollman * the appropriate place. 5147814Swpaul */ 5157780Swollman TAILQ_FOREACH(td2, &kg->kg_runq, td_runq) { 5167780Swollman if (td2->td_priority > td->td_priority) { 5177814Swpaul TAILQ_INSERT_BEFORE(td2, td, td_runq); 5187780Swollman break; 5197780Swollman } 5207780Swollman } 5217780Swollman if (td2 == NULL) { 5227780Swollman /* We ran off the end of the TAILQ or it was empty. */ 5237780Swollman TAILQ_INSERT_TAIL(&kg->kg_runq, td, td_runq); 5247780Swollman } 5257780Swollman 5267780Swollman /* 5277780Swollman * If we have a slot to use, then put the thread on the system 5287780Swollman * run queue and if needed, readjust the last_assigned pointer. 5297780Swollman * it may be that we need to schedule something anyhow 5307780Swollman * even if the availabel slots are -ve so that 5317780Swollman * all the items < last_assigned are scheduled. 5327780Swollman */ 5337780Swollman if (kg->kg_avail_opennings > 0) { 5347780Swollman if (tda == NULL) { 5357780Swollman /* 5367780Swollman * No pre-existing last assigned so whoever is first 5377780Swollman * gets the slot.. (maybe us) 5387780Swollman */ 5397780Swollman td2 = TAILQ_FIRST(&kg->kg_runq); 5407814Swpaul kg->kg_last_assigned = td2; 5418876Srgrimes } else if (tda->td_priority > td->td_priority) { 5427780Swollman td2 = td; 5437780Swollman } else { 5447780Swollman /* 5457780Swollman * We are past last_assigned, so 5467780Swollman * give the next slot to whatever is next, 5477780Swollman * which may or may not be us. 5487780Swollman */ 5497780Swollman td2 = TAILQ_NEXT(tda, td_runq); 5507780Swollman kg->kg_last_assigned = td2; 5517780Swollman } 5527780Swollman sched_add(td2, flags); 5537780Swollman } else { 5547780Swollman CTR3(KTR_RUNQ, "setrunqueue: held: td%p kg%p pid%d", 5557780Swollman td, td->td_ksegrp, td->td_proc->p_pid); 5567780Swollman if ((flags & SRQ_YIELDING) == 0) 5577780Swollman maybe_preempt_in_ksegrp(td); 5587780Swollman } 5597792Swpaul} 5607780Swollman 5617780Swollman/* 5627780Swollman * Kernel thread preemption implementation. Critical sections mark 5637814Swpaul * regions of code in which preemptions are not allowed. 5647780Swollman */ 565556Srgrimesvoid 566556Srgrimescritical_enter(void) 567556Srgrimes{ 568556Srgrimes struct thread *td; 5697814Swpaul 570556Srgrimes td = curthread; 571556Srgrimes if (td->td_critnest == 0) 572556Srgrimes cpu_critical_enter(td); 5737814Swpaul td->td_critnest++; 5747814Swpaul CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, 575556Srgrimes (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest); 576556Srgrimes} 577556Srgrimes 5787814Swpaulvoid 5797814Swpaulcritical_exit(void) 580556Srgrimes{ 5811998Swollman struct thread *td; 582556Srgrimes 5837814Swpaul td = curthread; 584556Srgrimes KASSERT(td->td_critnest != 0, 585556Srgrimes ("critical_exit: td_critnest == 0")); 586556Srgrimes if (td->td_critnest == 1) { 5877780Swollman if (td->td_pflags & TDP_WAKEPROC0) { 5887814Swpaul td->td_pflags &= ~TDP_WAKEPROC0; 5896308Sphk wakeup(&proc0); 5906308Sphk } 5911998Swollman#ifdef PREEMPTION 5923306Sphk mtx_assert(&sched_lock, MA_NOTOWNED); 5937780Swollman if (td->td_pflags & TDP_OWEPREEMPT) { 5947780Swollman mtx_lock_spin(&sched_lock); 5958456Srgrimes mi_switch(SW_INVOL, NULL); 5966308Sphk mtx_unlock_spin(&sched_lock); 5977780Swollman } 5987930Srgrimes#endif 5996308Sphk td->td_critnest = 0; 6006308Sphk cpu_critical_exit(td); 6017994Swpaul } else { 6027994Swpaul td->td_critnest--; 6037994Swpaul } 6047814Swpaul CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, 605556Srgrimes (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest); 606556Srgrimes} 607556Srgrimes 608556Srgrimes/* 609556Srgrimes * This function is called when a thread is about to be put on run queue 610556Srgrimes * because it has been made runnable or its priority has been adjusted. It 611569Srgrimes * determines if the new thread should be immediately preempted to. If so, 612569Srgrimes * it switches to it and eventually returns true. If not, it returns false 613569Srgrimes * so that the caller may place the thread on an appropriate run queue. 614556Srgrimes */ 615556Srgrimesint 616556Srgrimesmaybe_preempt(struct thread *td) 617556Srgrimes{ 618556Srgrimes#ifdef PREEMPTION 619556Srgrimes struct thread *ctd; 620556Srgrimes int cpri, pri; 621556Srgrimes#endif 622556Srgrimes 623556Srgrimes mtx_assert(&sched_lock, MA_OWNED); 624556Srgrimes#ifdef PREEMPTION 625556Srgrimes /* 626556Srgrimes * The new thread should not preempt the current thread if any of the 6277780Swollman * following conditions are true: 628556Srgrimes * 629556Srgrimes * - The current thread has a higher (numerically lower) or 6304Srgrimes * equivalent priority. Note that this prevents curthread from 6314Srgrimes * trying to preempt to itself. 6324Srgrimes * - It is too early in the boot for context switches (cold is set). 6334Srgrimes * - The current thread has an inhibitor set or is in the process of 6344Srgrimes * exiting. In this case, the current thread is about to switch 6354Srgrimes * out anyways, so there's no point in preempting. If we did, 6364Srgrimes * the current thread would not be properly resumed as well, so 6374Srgrimes * just avoid that whole landmine. 6384Srgrimes * - If the new thread's priority is not a realtime priority and 6394Srgrimes * the current thread's priority is not an idle priority and 6404Srgrimes * FULL_PREEMPTION is disabled. 6414Srgrimes * 6424Srgrimes * If all of these conditions are false, but the current thread is in 6434Srgrimes * a nested critical section, then we have to defer the preemption 6444Srgrimes * until we exit the critical section. Otherwise, switch immediately 6454Srgrimes * to the new thread. 6464Srgrimes */ 6474Srgrimes ctd = curthread; 6484Srgrimes KASSERT ((ctd->td_kse != NULL && ctd->td_kse->ke_thread == ctd), 6496846Sdg ("thread has no (or wrong) sched-private part.")); 6501549Srgrimes KASSERT((td->td_inhibitors == 0), 6513306Sphk ("maybe_preempt: trying to run inhibitted thread")); 6524Srgrimes pri = td->td_priority; 6531549Srgrimes cpri = ctd->td_priority; 6541549Srgrimes if (pri >= cpri || cold /* || dumping */ || TD_IS_INHIBITED(ctd) || 6554Srgrimes td->td_kse->ke_state != KES_THREAD) 6564Srgrimes return (0); 6574Srgrimes#ifndef FULL_PREEMPTION 6584Srgrimes if (!(pri >= PRI_MIN_ITHD && pri <= PRI_MAX_ITHD) && 6594Srgrimes !(cpri >= PRI_MIN_IDLE)) 6604Srgrimes return (0); 6614Srgrimes#endif 6621549Srgrimes if (ctd->td_critnest > 1) { 6631549Srgrimes CTR1(KTR_PROC, "maybe_preempt: in critical section %d", 6641549Srgrimes ctd->td_critnest); 6655999Sats ctd->td_pflags |= TDP_OWEPREEMPT; 6661549Srgrimes return (0); 6671549Srgrimes } 6684Srgrimes 669924Sdg /* 670924Sdg * Thread is runnable but not yet put on system run queue. 6714Srgrimes */ 6724Srgrimes MPASS(TD_ON_RUNQ(td)); 6731127Sdg MPASS(td->td_sched->ke_state != KES_ONRUNQ); 6741127Sdg if (td->td_proc->p_flag & P_HADTHREADS) { 6751127Sdg /* 6761127Sdg * If this is a threaded process we actually ARE on the 6771127Sdg * ksegrp run queue so take it off that first. 6781127Sdg * Also undo any damage done to the last_assigned pointer. 6791127Sdg * XXX Fix setrunqueue so this isn't needed 6804Srgrimes */ 6814Srgrimes struct ksegrp *kg; 6824Srgrimes 6834Srgrimes kg = td->td_ksegrp; 6844Srgrimes if (kg->kg_last_assigned == td) 6854Srgrimes kg->kg_last_assigned = 6864Srgrimes TAILQ_PREV(td, threadqueue, td_runq); 6874Srgrimes TAILQ_REMOVE(&kg->kg_runq, td, td_runq); 6884Srgrimes } 6894Srgrimes 6904Srgrimes TD_SET_RUNNING(td); 6914Srgrimes CTR3(KTR_PROC, "preempting to thread %p (pid %d, %s)\n", td, 6924Srgrimes td->td_proc->p_pid, td->td_proc->p_comm); 6938876Srgrimes mi_switch(SW_INVOL|SW_PREEMPT, td); 6944Srgrimes return (1); 6954Srgrimes#else 6962254Ssos return (0); 6972254Ssos#endif 6982254Ssos} 6992254Ssos 7002254Ssos#if 0 7012254Ssos#ifndef PREEMPTION 7026846Sdg/* XXX: There should be a non-static version of this. */ 7036846Sdgstatic void 7046846Sdgprintf_caddr_t(void *data) 7056846Sdg{ 7066846Sdg printf("%s", (char *)data); 7074Srgrimes} 7084Srgrimesstatic char preempt_warning[] = 7096846Sdg "WARNING: Kernel preemption is disabled, expect reduced performance.\n"; 7106846SdgSYSINIT(preempt_warning, SI_SUB_COPYRIGHT, SI_ORDER_ANY, printf_caddr_t, 7116846Sdg preempt_warning) 7126846Sdg#endif 7136846Sdg#endif 7146846Sdg 7156846Sdg/************************************************************************ 7166846Sdg * SYSTEM RUN QUEUE manipulations and tests * 7176846Sdg ************************************************************************/ 7186846Sdg/* 7196846Sdg * Initialize a run structure. 720924Sdg */ 7214Srgrimesvoid 7224Srgrimesrunq_init(struct runq *rq) 7234Srgrimes{ 7246846Sdg int i; 7256846Sdg 7266846Sdg bzero(rq, sizeof *rq); 7276846Sdg for (i = 0; i < RQ_NQS; i++) 7286846Sdg TAILQ_INIT(&rq->rq_queues[i]); 7296846Sdg} 7306846Sdg 7316846Sdg/* 7326846Sdg * Clear the status bit of the queue corresponding to priority level pri, 7336846Sdg * indicating that it is empty. 7346846Sdg */ 7356846Sdgstatic __inline void 7366846Sdgrunq_clrbit(struct runq *rq, int pri) 7376846Sdg{ 7386846Sdg struct rqbits *rqb; 7396846Sdg 7406846Sdg rqb = &rq->rq_status; 7416846Sdg CTR4(KTR_RUNQ, "runq_clrbit: bits=%#x %#x bit=%#x word=%d", 742924Sdg rqb->rqb_bits[RQB_WORD(pri)], 743924Sdg rqb->rqb_bits[RQB_WORD(pri)] & ~RQB_BIT(pri), 7441208Shsu RQB_BIT(pri), RQB_WORD(pri)); 7451066Sdg rqb->rqb_bits[RQB_WORD(pri)] &= ~RQB_BIT(pri); 7461066Sdg} 7471066Sdg 7481066Sdg/* 7494Srgrimes * Find the index of the first non-empty run queue. This is done by 7504Srgrimes * scanning the status bits, a set bit indicates a non-empty queue. 7514Srgrimes */ 7524Srgrimesstatic __inline int 7534Srgrimesrunq_findbit(struct runq *rq) 7544Srgrimes{ 7554Srgrimes struct rqbits *rqb; 7564Srgrimes int pri; 7574Srgrimes int i; 7585675Sbde 7594Srgrimes rqb = &rq->rq_status; 760798Swollman for (i = 0; i < RQB_LEN; i++) 7614Srgrimes if (rqb->rqb_bits[i]) { 7624Srgrimes pri = RQB_FFS(rqb->rqb_bits[i]) + (i << RQB_L2BPW); 76311390Sbde CTR3(KTR_RUNQ, "runq_findbit: bits=%#x i=%d pri=%d", 76411390Sbde rqb->rqb_bits[i], i, pri); 76511390Sbde return (pri); 7664Srgrimes } 7674Srgrimes 7684Srgrimes return (-1); 7694Srgrimes} 7701549Srgrimes 7711208Shsu/* 7724Srgrimes * Set the status bit of the queue corresponding to priority level pri, 7734Srgrimes * indicating that it is non-empty. 774924Sdg */ 7754Srgrimesstatic __inline void 7764Srgrimesrunq_setbit(struct runq *rq, int pri) 7774Srgrimes{ 7784Srgrimes struct rqbits *rqb; 7794Srgrimes 7804Srgrimes rqb = &rq->rq_status; 7814Srgrimes CTR4(KTR_RUNQ, "runq_setbit: bits=%#x %#x bit=%#x word=%d", 7824Srgrimes rqb->rqb_bits[RQB_WORD(pri)], 7834Srgrimes rqb->rqb_bits[RQB_WORD(pri)] | RQB_BIT(pri), 7844Srgrimes RQB_BIT(pri), RQB_WORD(pri)); 7854Srgrimes rqb->rqb_bits[RQB_WORD(pri)] |= RQB_BIT(pri); 7865603Sbde} 7875675Sbde 7885603Sbde/* 7895603Sbde * Add the KSE to the queue specified by its priority, and set the 7901208Shsu * corresponding status bit. 7915675Sbde */ 7925675Sbdevoid 7935675Sbderunq_add(struct runq *rq, struct kse *ke, int flags) 7945675Sbde{ 7955675Sbde struct rqhead *rqh; 7965675Sbde int pri; 7975675Sbde 7985675Sbde pri = ke->ke_thread->td_priority / RQ_PPQ; 7995675Sbde ke->ke_rqindex = pri; 8005675Sbde runq_setbit(rq, pri); 8015675Sbde rqh = &rq->rq_queues[pri]; 8021208Shsu CTR5(KTR_RUNQ, "runq_add: td=%p ke=%p pri=%d %d rqh=%p", 8035603Sbde ke->ke_thread, ke, ke->ke_thread->td_priority, pri, rqh); 8041208Shsu if (flags & SRQ_PREEMPTED) { 8051208Shsu TAILQ_INSERT_HEAD(rqh, ke, ke_procq); 8061208Shsu } else { 8071208Shsu TAILQ_INSERT_TAIL(rqh, ke, ke_procq); 8081208Shsu } 8095603Sbde} 8105603Sbde 8115603Sbde/* 8121208Shsu * Return true if there are runnable processes of any priority on the run 8135603Sbde * queue, false otherwise. Has no side effects, does not modify the run 8145603Sbde * queue structure. 8151208Shsu */ 8165603Sbdeint 8171208Shsurunq_check(struct runq *rq) 8181208Shsu{ 8191208Shsu struct rqbits *rqb; 8201208Shsu int i; 8211208Shsu 8224Srgrimes rqb = &rq->rq_status; 8231051Sdg for (i = 0; i < RQB_LEN; i++) 8241051Sdg if (rqb->rqb_bits[i]) { 8251051Sdg CTR2(KTR_RUNQ, "runq_check: bits=%#x i=%d", 8261051Sdg rqb->rqb_bits[i], i); 8271051Sdg return (1); 8281051Sdg } 8291051Sdg CTR0(KTR_RUNQ, "runq_check: empty"); 8301051Sdg 8311051Sdg return (0); 8321051Sdg} 8331051Sdg 8344Srgrimes#if defined(SMP) && defined(SCHED_4BSD) 8354Srgrimesint runq_fuzz = 1; 8364SrgrimesSYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, ""); 8371208Shsu#endif 8381549Srgrimes 8391549Srgrimes/* 8401549Srgrimes * Find the highest priority process on the run queue. 8411549Srgrimes */ 8424Srgrimesstruct kse * 8434Srgrimesrunq_choose(struct runq *rq) 844924Sdg{ 845924Sdg struct rqhead *rqh; 846924Sdg struct kse *ke; 8471208Shsu int pri; 8484Srgrimes 8494Srgrimes mtx_assert(&sched_lock, MA_OWNED); 8504Srgrimes while ((pri = runq_findbit(rq)) != -1) { 851556Srgrimes rqh = &rq->rq_queues[pri]; 852556Srgrimes#if defined(SMP) && defined(SCHED_4BSD) 853556Srgrimes /* fuzz == 1 is normal.. 0 or less are ignored */ 854556Srgrimes if (runq_fuzz > 1) { 855556Srgrimes /* 856556Srgrimes * In the first couple of entries, check if 857556Srgrimes * there is one for our CPU as a preference. 858556Srgrimes */ 859556Srgrimes int count = runq_fuzz; 8604Srgrimes int cpu = PCPU_GET(cpuid); 8614Srgrimes struct kse *ke2; 8624Srgrimes ke2 = ke = TAILQ_FIRST(rqh); 8632059Sdg 86410126Sdg while (count-- && ke2) { 86510126Sdg if (ke->ke_thread->td_lastcpu == cpu) { 8664Srgrimes ke = ke2; 86710126Sdg break; 8684Srgrimes } 8694Srgrimes ke2 = TAILQ_NEXT(ke2, ke_procq); 8704Srgrimes } 8714Srgrimes } else 8722152Sdg#endif 8739345Sdg ke = TAILQ_FIRST(rqh); 8746439Sdg KASSERT(ke != NULL, ("runq_choose: no proc on busy queue")); 8754Srgrimes CTR3(KTR_RUNQ, 8764Srgrimes "runq_choose: pri=%d kse=%p rqh=%p", pri, ke, rqh); 8774Srgrimes return (ke); 8785455Sdg } 87910126Sdg CTR1(KTR_RUNQ, "runq_choose: idleproc pri=%d", pri); 8804Srgrimes 8815455Sdg return (NULL); 8825455Sdg} 8834Srgrimes 8844Srgrimes/* 885556Srgrimes * Remove the KSE from the queue specified by its priority, and clear the 8864Srgrimes * corresponding status bit if the queue becomes empty. 8874Srgrimes * Caller must set ke->ke_state afterwards. 8882152Sdg */ 8892152Sdgvoid 8902152Sdgrunq_remove(struct runq *rq, struct kse *ke) 8912152Sdg{ 8922152Sdg struct rqhead *rqh; 893556Srgrimes int pri; 89412008Speter 89512008Speter KASSERT(ke->ke_proc->p_sflag & PS_INMEM, 89612008Speter ("runq_remove: process swapped out")); 89712008Speter pri = ke->ke_rqindex; 89812008Speter rqh = &rq->rq_queues[pri]; 89912008Speter CTR5(KTR_RUNQ, "runq_remove: td=%p, ke=%p pri=%d %d rqh=%p", 90012008Speter ke->ke_thread, ke, ke->ke_thread->td_priority, pri, rqh); 90112008Speter KASSERT(ke != NULL, ("runq_remove: no proc on busy queue")); 90212008Speter TAILQ_REMOVE(rqh, ke, ke_procq); 90312008Speter if (TAILQ_EMPTY(rqh)) { 9042152Sdg CTR0(KTR_RUNQ, "runq_remove: empty"); 905556Srgrimes runq_clrbit(rq, pri); 9062152Sdg } 9072152Sdg} 9082152Sdg 9092152Sdg/****** functions that are temporarily here ***********/ 9102152Sdg#include <vm/uma.h> 9112152Sdgextern struct mtx kse_zombie_lock; 9122152Sdg 9137103Sdg/* 9144Srgrimes * Allocate scheduler specific per-process resources. 9154Srgrimes * The thread and ksegrp have already been linked in. 91610126Sdg * In this case just set the default concurrency value. 917556Srgrimes * 918556Srgrimes * Called from: 919556Srgrimes * proc_init() (UMA init method) 920556Srgrimes */ 9214Srgrimesvoid 9224Srgrimessched_newproc(struct proc *p, struct ksegrp *kg, struct thread *td) 92310126Sdg{ 92410126Sdg 92510126Sdg /* This can go in sched_fork */ 92610126Sdg sched_init_concurrency(kg); 92710126Sdg} 928757Sdg 929757Sdg/* 930757Sdg * thread is being either created or recycled. 931757Sdg * Fix up the per-scheduler resources associated with it. 932757Sdg * Called from: 933757Sdg * sched_fork_thread() 93410126Sdg * thread_dtor() (*may go away) 93510126Sdg * thread_init() (*may go away) 9363728Sphk */ 937757Sdgvoid 938757Sdgsched_newthread(struct thread *td) 939757Sdg{ 940757Sdg struct td_sched *ke; 941757Sdg 942757Sdg ke = (struct td_sched *) (td + 1); 943757Sdg bzero(ke, sizeof(*ke)); 944757Sdg td->td_sched = ke; 945757Sdg ke->ke_thread = td; 946757Sdg ke->ke_state = KES_THREAD; 9474Srgrimes} 9484Srgrimes 949757Sdg/* 950757Sdg * Set up an initial concurrency of 1 9511321Sdg * and set the given thread (if given) to be using that 9524Srgrimes * concurrency slot. 9534Srgrimes * May be used "offline"..before the ksegrp is attached to the world 9541247Sdg * and thus wouldn't need schedlock in that case. 9554Srgrimes * Called from: 9564Srgrimes * thr_create() 957879Swollman * proc_init() (UMA) via sched_newproc() 958556Srgrimes */ 9594221Sphkvoid 9604221Sphksched_init_concurrency(struct ksegrp *kg) 9618481Swollman{ 9624Srgrimes 9634Srgrimes CTR1(KTR_RUNQ,"kg %p init slots and concurrency to 1", kg); 9644Srgrimes kg->kg_concurrency = 1; 9654Srgrimes kg->kg_avail_opennings = 1; 9664Srgrimes} 967556Srgrimes 9684Srgrimes/* 9694Srgrimes * Change the concurrency of an existing ksegrp to N 9704Srgrimes * Called from: 9714221Sphk * kse_create() 9724221Sphk * kse_exit() 9734Srgrimes * thread_exit() 9744Srgrimes * thread_single() 9754Srgrimes */ 9764Srgrimesvoid 9771055Sdgsched_set_concurrency(struct ksegrp *kg, int concurrency) 9783306Sphk{ 979556Srgrimes 9804Srgrimes CTR4(KTR_RUNQ,"kg %p set concurrency to %d, slots %d -> %d", 9814Srgrimes kg, 9824Srgrimes concurrency, 983556Srgrimes kg->kg_avail_opennings, 9844Srgrimes kg->kg_avail_opennings + (concurrency - kg->kg_concurrency)); 9854Srgrimes kg->kg_avail_opennings += (concurrency - kg->kg_concurrency); 9864Srgrimes kg->kg_concurrency = concurrency; 987556Srgrimes} 9884Srgrimes 9894Srgrimes/* 9904Srgrimes * Called from thread_exit() for all exiting thread 991556Srgrimes * 9924Srgrimes * Not to be confused with sched_exit_thread() 9934Srgrimes * that is only called from thread_exit() for threads exiting 9944Srgrimes * without the rest of the process exiting because it is also called from 995556Srgrimes * sched_exit() and we wouldn't want to call it twice. 9964Srgrimes * XXX This can probably be fixed. 9974Srgrimes */ 9984Srgrimesvoid 999556Srgrimessched_thread_exit(struct thread *td) 10004Srgrimes{ 10014Srgrimes 10024Srgrimes SLOT_RELEASE(td->td_ksegrp); 1003556Srgrimes slot_fill(td->td_ksegrp); 10044Srgrimes} 10054Srgrimes 10064Srgrimes#endif /* KERN_SWITCH_INCLUDE */ 10074Srgrimes