kern_switch.c revision 172207
1139804Simp/*- 272376Sjake * Copyright (c) 2001 Jake Burkholder <jake@FreeBSD.org> 372376Sjake * All rights reserved. 450027Speter * 550027Speter * Redistribution and use in source and binary forms, with or without 650027Speter * modification, are permitted provided that the following conditions 750027Speter * are met: 850027Speter * 1. Redistributions of source code must retain the above copyright 950027Speter * notice, this list of conditions and the following disclaimer. 1050027Speter * 2. Redistributions in binary form must reproduce the above copyright 1150027Speter * notice, this list of conditions and the following disclaimer in the 1250027Speter * documentation and/or other materials provided with the distribution. 1350027Speter * 1450027Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1550027Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1650027Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1750027Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1850027Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1950027Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2050027Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2150027Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2250027Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2350027Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2450027Speter * SUCH DAMAGE. 2550027Speter */ 2650027Speter 2799072Sjulian 28116182Sobrien#include <sys/cdefs.h> 29116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_switch.c 172207 2007-09-17 05:31:39Z jeff $"); 30116182Sobrien 31134591Sjulian#include "opt_sched.h" 32131481Sjhb 33134791Sjulian#ifndef KERN_SWITCH_INCLUDE 3450027Speter#include <sys/param.h> 3550027Speter#include <sys/systm.h> 36131927Smarcel#include <sys/kdb.h> 3750027Speter#include <sys/kernel.h> 3865557Sjasone#include <sys/ktr.h> 3974914Sjhb#include <sys/lock.h> 4067365Sjhb#include <sys/mutex.h> 4150027Speter#include <sys/proc.h> 4250027Speter#include <sys/queue.h> 43104964Sjeff#include <sys/sched.h> 44134791Sjulian#else /* KERN_SWITCH_INCLUDE */ 45122849Speter#if defined(SMP) && (defined(__i386__) || defined(__amd64__)) 46112993Speter#include <sys/smp.h> 47112993Speter#endif 48134591Sjulian#if defined(SMP) && defined(SCHED_4BSD) 49134591Sjulian#include <sys/sysctl.h> 50134591Sjulian#endif 5150027Speter 52170293Sjeff#include <machine/cpu.h> 53170293Sjeff 54153510Snjl/* Uncomment this to enable logging of critical_enter/exit. */ 55153510Snjl#if 0 56153510Snjl#define KTR_CRITICAL KTR_SCHED 57153510Snjl#else 58153510Snjl#define KTR_CRITICAL 0 59153510Snjl#endif 60153510Snjl 61134649Sscottl#ifdef FULL_PREEMPTION 62134649Sscottl#ifndef PREEMPTION 63134649Sscottl#error "The FULL_PREEMPTION option requires the PREEMPTION option" 64134649Sscottl#endif 65134649Sscottl#endif 66134591Sjulian 6797261SjakeCTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS); 6897261Sjake 69143884Srwatson/* 70143884Srwatson * kern.sched.preemption allows user space to determine if preemption support 71143884Srwatson * is compiled in or not. It is not currently a boot or runtime flag that 72143884Srwatson * can be changed. 73143884Srwatson */ 74143884Srwatson#ifdef PREEMPTION 75143884Srwatsonstatic int kern_sched_preemption = 1; 76143884Srwatson#else 77143884Srwatsonstatic int kern_sched_preemption = 0; 78143884Srwatson#endif 79143884SrwatsonSYSCTL_INT(_kern_sched, OID_AUTO, preemption, CTLFLAG_RD, 80143884Srwatson &kern_sched_preemption, 0, "Kernel preemption enabled"); 81143884Srwatson 82170293Sjeff#ifdef SCHED_STATS 83170293Sjefflong switch_preempt; 84170293Sjefflong switch_owepreempt; 85170293Sjefflong switch_turnstile; 86170293Sjefflong switch_sleepq; 87170293Sjefflong switch_sleepqtimo; 88170293Sjefflong switch_relinquish; 89170293Sjefflong switch_needresched; 90170293Sjeffstatic SYSCTL_NODE(_kern_sched, OID_AUTO, stats, CTLFLAG_RW, 0, "switch stats"); 91170293SjeffSYSCTL_INT(_kern_sched_stats, OID_AUTO, preempt, CTLFLAG_RD, &switch_preempt, 0, ""); 92170293SjeffSYSCTL_INT(_kern_sched_stats, OID_AUTO, owepreempt, CTLFLAG_RD, &switch_owepreempt, 0, ""); 93170293SjeffSYSCTL_INT(_kern_sched_stats, OID_AUTO, turnstile, CTLFLAG_RD, &switch_turnstile, 0, ""); 94170293SjeffSYSCTL_INT(_kern_sched_stats, OID_AUTO, sleepq, CTLFLAG_RD, &switch_sleepq, 0, ""); 95170293SjeffSYSCTL_INT(_kern_sched_stats, OID_AUTO, sleepqtimo, CTLFLAG_RD, &switch_sleepqtimo, 0, ""); 96170293SjeffSYSCTL_INT(_kern_sched_stats, OID_AUTO, relinquish, CTLFLAG_RD, &switch_relinquish, 0, ""); 97170293SjeffSYSCTL_INT(_kern_sched_stats, OID_AUTO, needresched, CTLFLAG_RD, &switch_needresched, 0, ""); 98170293Sjeffstatic int 99170293Sjeffsysctl_stats_reset(SYSCTL_HANDLER_ARGS) 100170293Sjeff{ 101170293Sjeff int error; 102170293Sjeff int val; 103170293Sjeff 104170293Sjeff val = 0; 105170293Sjeff error = sysctl_handle_int(oidp, &val, 0, req); 106170293Sjeff if (error != 0 || req->newptr == NULL) 107170293Sjeff return (error); 108170293Sjeff if (val == 0) 109170293Sjeff return (0); 110170293Sjeff switch_preempt = 0; 111170293Sjeff switch_owepreempt = 0; 112170293Sjeff switch_turnstile = 0; 113170293Sjeff switch_sleepq = 0; 114170293Sjeff switch_sleepqtimo = 0; 115170293Sjeff switch_relinquish = 0; 116170293Sjeff switch_needresched = 0; 117170293Sjeff 118170293Sjeff return (0); 119170293Sjeff} 120170293Sjeff 121170293SjeffSYSCTL_PROC(_kern_sched_stats, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_WR, NULL, 122170293Sjeff 0, sysctl_stats_reset, "I", "Reset scheduler statistics"); 123170293Sjeff#endif 124170293Sjeff 12599072Sjulian/************************************************************************ 12699072Sjulian * Functions that manipulate runnability from a thread perspective. * 12799072Sjulian ************************************************************************/ 12850027Speter/* 129163709Sjb * Select the thread that will be run next. 130163709Sjb */ 13183366Sjulianstruct thread * 13283366Sjulianchoosethread(void) 13350027Speter{ 13499072Sjulian struct thread *td; 13599072Sjulian 136122849Speter#if defined(SMP) && (defined(__i386__) || defined(__amd64__)) 137112993Speter if (smp_active == 0 && PCPU_GET(cpuid) != 0) { 138112993Speter /* Shutting down, run idlethread on AP's */ 139112993Speter td = PCPU_GET(idlethread); 140112993Speter CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td); 141112993Speter TD_SET_RUNNING(td); 142112993Speter return (td); 143112993Speter } 144112993Speter#endif 145112993Speter 146100209Sgallatinretry: 147166188Sjeff td = sched_choose(); 148108338Sjulian 149108338Sjulian /* 150115215Sjulian * If we are in panic, only allow system threads, 151115215Sjulian * plus the one we are running in, to be run. 152108338Sjulian */ 153100209Sgallatin if (panicstr && ((td->td_proc->p_flag & P_SYSTEM) == 0 && 154115215Sjulian (td->td_flags & TDF_INPANIC) == 0)) { 155115215Sjulian /* note that it is no longer on the run queue */ 156115215Sjulian TD_SET_CAN_RUN(td); 157100209Sgallatin goto retry; 158115215Sjulian } 159108338Sjulian 160103216Sjulian TD_SET_RUNNING(td); 16199072Sjulian return (td); 16272376Sjake} 16350027Speter 16499072Sjulian/* 165131481Sjhb * Kernel thread preemption implementation. Critical sections mark 166131481Sjhb * regions of code in which preemptions are not allowed. 167131481Sjhb */ 16888088Sjhbvoid 16988088Sjhbcritical_enter(void) 17088088Sjhb{ 17188088Sjhb struct thread *td; 17288088Sjhb 17388088Sjhb td = curthread; 17488088Sjhb td->td_critnest++; 175153510Snjl CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, 176137364Srwatson (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest); 17788088Sjhb} 17888088Sjhb 17988088Sjhbvoid 18088088Sjhbcritical_exit(void) 18188088Sjhb{ 18288088Sjhb struct thread *td; 18388088Sjhb 18488088Sjhb td = curthread; 185125315Sjeff KASSERT(td->td_critnest != 0, 186125315Sjeff ("critical_exit: td_critnest == 0")); 187146554Sups#ifdef PREEMPTION 18888088Sjhb if (td->td_critnest == 1) { 189144777Sups td->td_critnest = 0; 190144777Sups if (td->td_owepreempt) { 191144777Sups td->td_critnest = 1; 192170293Sjeff thread_lock(td); 193144777Sups td->td_critnest--; 194170293Sjeff SCHED_STAT_INC(switch_owepreempt); 195171712Sjeff mi_switch(SW_INVOL|SW_PREEMPT, NULL); 196170293Sjeff thread_unlock(td); 197131481Sjhb } 198153797Skan } else 199131481Sjhb#endif 20088088Sjhb td->td_critnest--; 201153797Skan 202153510Snjl CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, 203137364Srwatson (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest); 20488088Sjhb} 20588088Sjhb 206131481Sjhb/* 207131481Sjhb * This function is called when a thread is about to be put on run queue 208131481Sjhb * because it has been made runnable or its priority has been adjusted. It 209131481Sjhb * determines if the new thread should be immediately preempted to. If so, 210131481Sjhb * it switches to it and eventually returns true. If not, it returns false 211131481Sjhb * so that the caller may place the thread on an appropriate run queue. 212131481Sjhb */ 213131481Sjhbint 214131481Sjhbmaybe_preempt(struct thread *td) 215131481Sjhb{ 216131508Smarcel#ifdef PREEMPTION 217131481Sjhb struct thread *ctd; 218131481Sjhb int cpri, pri; 219131508Smarcel#endif 22099072Sjulian 221131481Sjhb#ifdef PREEMPTION 222131481Sjhb /* 223131481Sjhb * The new thread should not preempt the current thread if any of the 224131481Sjhb * following conditions are true: 225131481Sjhb * 226143757Srwatson * - The kernel is in the throes of crashing (panicstr). 227132266Sjhb * - The current thread has a higher (numerically lower) or 228132266Sjhb * equivalent priority. Note that this prevents curthread from 229132266Sjhb * trying to preempt to itself. 230131481Sjhb * - It is too early in the boot for context switches (cold is set). 231131481Sjhb * - The current thread has an inhibitor set or is in the process of 232131481Sjhb * exiting. In this case, the current thread is about to switch 233131481Sjhb * out anyways, so there's no point in preempting. If we did, 234131481Sjhb * the current thread would not be properly resumed as well, so 235131481Sjhb * just avoid that whole landmine. 236131481Sjhb * - If the new thread's priority is not a realtime priority and 237131481Sjhb * the current thread's priority is not an idle priority and 238131481Sjhb * FULL_PREEMPTION is disabled. 239131481Sjhb * 240131481Sjhb * If all of these conditions are false, but the current thread is in 241131481Sjhb * a nested critical section, then we have to defer the preemption 242131481Sjhb * until we exit the critical section. Otherwise, switch immediately 243131481Sjhb * to the new thread. 244131481Sjhb */ 245131481Sjhb ctd = curthread; 246170293Sjeff THREAD_LOCK_ASSERT(td, MA_OWNED); 247164936Sjulian KASSERT ((ctd->td_sched != NULL && ctd->td_sched->ts_thread == ctd), 248134837Sjulian ("thread has no (or wrong) sched-private part.")); 249135181Sjulian KASSERT((td->td_inhibitors == 0), 250165693Srwatson ("maybe_preempt: trying to run inhibited thread")); 251131481Sjhb pri = td->td_priority; 252131481Sjhb cpri = ctd->td_priority; 253143757Srwatson if (panicstr != NULL || pri >= cpri || cold /* || dumping */ || 254166188Sjeff TD_IS_INHIBITED(ctd)) 255131481Sjhb return (0); 256131481Sjhb#ifndef FULL_PREEMPTION 257147216Sups if (pri > PRI_MAX_ITHD && cpri < PRI_MIN_IDLE) 258131481Sjhb return (0); 259131481Sjhb#endif 260147190Sups 261131481Sjhb if (ctd->td_critnest > 1) { 262131481Sjhb CTR1(KTR_PROC, "maybe_preempt: in critical section %d", 263131481Sjhb ctd->td_critnest); 264144777Sups ctd->td_owepreempt = 1; 265131481Sjhb return (0); 266131481Sjhb } 267131481Sjhb /* 268136170Sjulian * Thread is runnable but not yet put on system run queue. 269131481Sjhb */ 270171495Sjeff MPASS(ctd->td_lock == td->td_lock); 271131481Sjhb MPASS(TD_ON_RUNQ(td)); 272131481Sjhb TD_SET_RUNNING(td); 273131481Sjhb CTR3(KTR_PROC, "preempting to thread %p (pid %d, %s)\n", td, 274131481Sjhb td->td_proc->p_pid, td->td_proc->p_comm); 275170293Sjeff SCHED_STAT_INC(switch_preempt); 276136170Sjulian mi_switch(SW_INVOL|SW_PREEMPT, td); 277170293Sjeff /* 278170293Sjeff * td's lock pointer may have changed. We have to return with it 279170293Sjeff * locked. 280170293Sjeff */ 281170293Sjeff spinlock_enter(); 282170293Sjeff thread_unlock(ctd); 283170293Sjeff thread_lock(td); 284170293Sjeff spinlock_exit(); 285131481Sjhb return (1); 286131481Sjhb#else 287131481Sjhb return (0); 288131481Sjhb#endif 289131481Sjhb} 290131481Sjhb 291133219Sjhb#if 0 292131481Sjhb#ifndef PREEMPTION 293131481Sjhb/* XXX: There should be a non-static version of this. */ 294131481Sjhbstatic void 295131481Sjhbprintf_caddr_t(void *data) 296131481Sjhb{ 297131481Sjhb printf("%s", (char *)data); 298131481Sjhb} 299131481Sjhbstatic char preempt_warning[] = 300131481Sjhb "WARNING: Kernel preemption is disabled, expect reduced performance.\n"; 301131481SjhbSYSINIT(preempt_warning, SI_SUB_COPYRIGHT, SI_ORDER_ANY, printf_caddr_t, 302131481Sjhb preempt_warning) 303131481Sjhb#endif 304133219Sjhb#endif 305131481Sjhb 30699072Sjulian/************************************************************************ 30799072Sjulian * SYSTEM RUN QUEUE manipulations and tests * 30899072Sjulian ************************************************************************/ 30972376Sjake/* 31099072Sjulian * Initialize a run structure. 31199072Sjulian */ 31299072Sjulianvoid 31399072Sjulianrunq_init(struct runq *rq) 31499072Sjulian{ 31599072Sjulian int i; 31699072Sjulian 31799072Sjulian bzero(rq, sizeof *rq); 31899072Sjulian for (i = 0; i < RQ_NQS; i++) 31999072Sjulian TAILQ_INIT(&rq->rq_queues[i]); 32099072Sjulian} 32199072Sjulian 32299072Sjulian/* 32372376Sjake * Clear the status bit of the queue corresponding to priority level pri, 32472376Sjake * indicating that it is empty. 32572376Sjake */ 32672376Sjakestatic __inline void 32772376Sjakerunq_clrbit(struct runq *rq, int pri) 32872376Sjake{ 32972376Sjake struct rqbits *rqb; 33065557Sjasone 33172376Sjake rqb = &rq->rq_status; 33272376Sjake CTR4(KTR_RUNQ, "runq_clrbit: bits=%#x %#x bit=%#x word=%d", 33372376Sjake rqb->rqb_bits[RQB_WORD(pri)], 33472376Sjake rqb->rqb_bits[RQB_WORD(pri)] & ~RQB_BIT(pri), 33572376Sjake RQB_BIT(pri), RQB_WORD(pri)); 33672376Sjake rqb->rqb_bits[RQB_WORD(pri)] &= ~RQB_BIT(pri); 33750027Speter} 33850027Speter 33950027Speter/* 34072376Sjake * Find the index of the first non-empty run queue. This is done by 34172376Sjake * scanning the status bits, a set bit indicates a non-empty queue. 34250027Speter */ 34372376Sjakestatic __inline int 34472376Sjakerunq_findbit(struct runq *rq) 34572376Sjake{ 34672376Sjake struct rqbits *rqb; 34772376Sjake int pri; 34872376Sjake int i; 34972376Sjake 35072376Sjake rqb = &rq->rq_status; 35172376Sjake for (i = 0; i < RQB_LEN; i++) 35272376Sjake if (rqb->rqb_bits[i]) { 35398469Speter pri = RQB_FFS(rqb->rqb_bits[i]) + (i << RQB_L2BPW); 35472376Sjake CTR3(KTR_RUNQ, "runq_findbit: bits=%#x i=%d pri=%d", 35572376Sjake rqb->rqb_bits[i], i, pri); 35672376Sjake return (pri); 35772376Sjake } 35872376Sjake 35972376Sjake return (-1); 36072376Sjake} 36172376Sjake 362165761Sjeffstatic __inline int 363171900Sjeffrunq_findbit_from(struct runq *rq, u_char pri) 364165761Sjeff{ 365165761Sjeff struct rqbits *rqb; 366171900Sjeff rqb_word_t mask; 367165761Sjeff int i; 368165761Sjeff 369171900Sjeff /* 370171900Sjeff * Set the mask for the first word so we ignore priorities before 'pri'. 371171900Sjeff */ 372171900Sjeff mask = (rqb_word_t)-1 << (pri & (RQB_BPW - 1)); 373165761Sjeff rqb = &rq->rq_status; 374165761Sjeffagain: 375171900Sjeff for (i = RQB_WORD(pri); i < RQB_LEN; mask = -1, i++) { 376171900Sjeff mask = rqb->rqb_bits[i] & mask; 377171900Sjeff if (mask == 0) 378171900Sjeff continue; 379171900Sjeff pri = RQB_FFS(mask) + (i << RQB_L2BPW); 380171900Sjeff CTR3(KTR_RUNQ, "runq_findbit_from: bits=%#x i=%d pri=%d", 381171900Sjeff mask, i, pri); 382171900Sjeff return (pri); 383165761Sjeff } 384171900Sjeff if (pri == 0) 385171900Sjeff return (-1); 386171900Sjeff /* 387171900Sjeff * Wrap back around to the beginning of the list just once so we 388171900Sjeff * scan the whole thing. 389171900Sjeff */ 390171900Sjeff pri = 0; 391171900Sjeff goto again; 392165761Sjeff} 393165761Sjeff 39472376Sjake/* 39572376Sjake * Set the status bit of the queue corresponding to priority level pri, 39672376Sjake * indicating that it is non-empty. 39772376Sjake */ 39872376Sjakestatic __inline void 39972376Sjakerunq_setbit(struct runq *rq, int pri) 40072376Sjake{ 40172376Sjake struct rqbits *rqb; 40272376Sjake 40372376Sjake rqb = &rq->rq_status; 40472376Sjake CTR4(KTR_RUNQ, "runq_setbit: bits=%#x %#x bit=%#x word=%d", 40572376Sjake rqb->rqb_bits[RQB_WORD(pri)], 40672376Sjake rqb->rqb_bits[RQB_WORD(pri)] | RQB_BIT(pri), 40772376Sjake RQB_BIT(pri), RQB_WORD(pri)); 40872376Sjake rqb->rqb_bits[RQB_WORD(pri)] |= RQB_BIT(pri); 40972376Sjake} 41072376Sjake 41172376Sjake/* 412164936Sjulian * Add the thread to the queue specified by its priority, and set the 41372376Sjake * corresponding status bit. 41472376Sjake */ 41550027Spetervoid 416164936Sjulianrunq_add(struct runq *rq, struct td_sched *ts, int flags) 41750027Speter{ 41872376Sjake struct rqhead *rqh; 41972376Sjake int pri; 42050027Speter 421164936Sjulian pri = ts->ts_thread->td_priority / RQ_PPQ; 422164936Sjulian ts->ts_rqindex = pri; 42372376Sjake runq_setbit(rq, pri); 42472376Sjake rqh = &rq->rq_queues[pri]; 425164936Sjulian CTR5(KTR_RUNQ, "runq_add: td=%p ts=%p pri=%d %d rqh=%p", 426164936Sjulian ts->ts_thread, ts, ts->ts_thread->td_priority, pri, rqh); 427136170Sjulian if (flags & SRQ_PREEMPTED) { 428164936Sjulian TAILQ_INSERT_HEAD(rqh, ts, ts_procq); 429136170Sjulian } else { 430164936Sjulian TAILQ_INSERT_TAIL(rqh, ts, ts_procq); 431136170Sjulian } 43250027Speter} 43350027Speter 434165761Sjeffvoid 435166557Sjeffrunq_add_pri(struct runq *rq, struct td_sched *ts, u_char pri, int flags) 436165761Sjeff{ 437165761Sjeff struct rqhead *rqh; 438165761Sjeff 439165761Sjeff KASSERT(pri < RQ_NQS, ("runq_add_pri: %d out of range", pri)); 440165761Sjeff ts->ts_rqindex = pri; 441165761Sjeff runq_setbit(rq, pri); 442165761Sjeff rqh = &rq->rq_queues[pri]; 443165761Sjeff CTR5(KTR_RUNQ, "runq_add_pri: td=%p ke=%p pri=%d idx=%d rqh=%p", 444165761Sjeff ts->ts_thread, ts, ts->ts_thread->td_priority, pri, rqh); 445165761Sjeff if (flags & SRQ_PREEMPTED) { 446165761Sjeff TAILQ_INSERT_HEAD(rqh, ts, ts_procq); 447165761Sjeff } else { 448165761Sjeff TAILQ_INSERT_TAIL(rqh, ts, ts_procq); 449165761Sjeff } 450165761Sjeff} 45150027Speter/* 45272376Sjake * Return true if there are runnable processes of any priority on the run 45372376Sjake * queue, false otherwise. Has no side effects, does not modify the run 45472376Sjake * queue structure. 45550027Speter */ 45672376Sjakeint 45772376Sjakerunq_check(struct runq *rq) 45850027Speter{ 45972376Sjake struct rqbits *rqb; 46072376Sjake int i; 46172376Sjake 46272376Sjake rqb = &rq->rq_status; 46372376Sjake for (i = 0; i < RQB_LEN; i++) 46472376Sjake if (rqb->rqb_bits[i]) { 46572376Sjake CTR2(KTR_RUNQ, "runq_check: bits=%#x i=%d", 46672376Sjake rqb->rqb_bits[i], i); 46772376Sjake return (1); 46872376Sjake } 46972376Sjake CTR0(KTR_RUNQ, "runq_check: empty"); 47072376Sjake 47172376Sjake return (0); 47250027Speter} 47350027Speter 474134591Sjulian#if defined(SMP) && defined(SCHED_4BSD) 475134591Sjulianint runq_fuzz = 1; 476134591SjulianSYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, ""); 477134591Sjulian#endif 478134591Sjulian 47950027Speter/* 480104964Sjeff * Find the highest priority process on the run queue. 48150027Speter */ 482164936Sjulianstruct td_sched * 48372376Sjakerunq_choose(struct runq *rq) 48450027Speter{ 48572376Sjake struct rqhead *rqh; 486164936Sjulian struct td_sched *ts; 48772376Sjake int pri; 48850027Speter 48999072Sjulian while ((pri = runq_findbit(rq)) != -1) { 49072376Sjake rqh = &rq->rq_queues[pri]; 491134591Sjulian#if defined(SMP) && defined(SCHED_4BSD) 492134591Sjulian /* fuzz == 1 is normal.. 0 or less are ignored */ 493134591Sjulian if (runq_fuzz > 1) { 494134591Sjulian /* 495134591Sjulian * In the first couple of entries, check if 496134591Sjulian * there is one for our CPU as a preference. 497134591Sjulian */ 498134591Sjulian int count = runq_fuzz; 499134591Sjulian int cpu = PCPU_GET(cpuid); 500164936Sjulian struct td_sched *ts2; 501164936Sjulian ts2 = ts = TAILQ_FIRST(rqh); 502134591Sjulian 503164936Sjulian while (count-- && ts2) { 504164936Sjulian if (ts->ts_thread->td_lastcpu == cpu) { 505164936Sjulian ts = ts2; 506134591Sjulian break; 507134591Sjulian } 508164936Sjulian ts2 = TAILQ_NEXT(ts2, ts_procq); 509134591Sjulian } 510153797Skan } else 511134591Sjulian#endif 512164936Sjulian ts = TAILQ_FIRST(rqh); 513164936Sjulian KASSERT(ts != NULL, ("runq_choose: no proc on busy queue")); 51499072Sjulian CTR3(KTR_RUNQ, 515164936Sjulian "runq_choose: pri=%d td_sched=%p rqh=%p", pri, ts, rqh); 516164936Sjulian return (ts); 51750027Speter } 51872376Sjake CTR1(KTR_RUNQ, "runq_choose: idleproc pri=%d", pri); 51972376Sjake 52099072Sjulian return (NULL); 52150027Speter} 52272376Sjake 523165761Sjeffstruct td_sched * 524166557Sjeffrunq_choose_from(struct runq *rq, u_char idx) 525165761Sjeff{ 526165761Sjeff struct rqhead *rqh; 527165761Sjeff struct td_sched *ts; 528165761Sjeff int pri; 529165761Sjeff 530165765Sjeff if ((pri = runq_findbit_from(rq, idx)) != -1) { 531165761Sjeff rqh = &rq->rq_queues[pri]; 532165761Sjeff ts = TAILQ_FIRST(rqh); 533165761Sjeff KASSERT(ts != NULL, ("runq_choose: no proc on busy queue")); 534165761Sjeff CTR4(KTR_RUNQ, 535165761Sjeff "runq_choose_from: pri=%d kse=%p idx=%d rqh=%p", 536165761Sjeff pri, ts, ts->ts_rqindex, rqh); 537165761Sjeff return (ts); 538165761Sjeff } 539165761Sjeff CTR1(KTR_RUNQ, "runq_choose_from: idleproc pri=%d", pri); 540165761Sjeff 541165761Sjeff return (NULL); 542165761Sjeff} 54372376Sjake/* 544164936Sjulian * Remove the thread from the queue specified by its priority, and clear the 54572376Sjake * corresponding status bit if the queue becomes empty. 546166188Sjeff * Caller must set state afterwards. 54772376Sjake */ 54872376Sjakevoid 549164936Sjulianrunq_remove(struct runq *rq, struct td_sched *ts) 55072376Sjake{ 551165761Sjeff 552165761Sjeff runq_remove_idx(rq, ts, NULL); 553165761Sjeff} 554165761Sjeff 555165761Sjeffvoid 556166557Sjeffrunq_remove_idx(struct runq *rq, struct td_sched *ts, u_char *idx) 557165761Sjeff{ 55872376Sjake struct rqhead *rqh; 559166557Sjeff u_char pri; 56072376Sjake 561172207Sjeff KASSERT(ts->ts_thread->td_flags & TDF_INMEM, 562172207Sjeff ("runq_remove_idx: thread swapped out")); 563164936Sjulian pri = ts->ts_rqindex; 564170293Sjeff KASSERT(pri < RQ_NQS, ("runq_remove_idx: Invalid index %d\n", pri)); 56572376Sjake rqh = &rq->rq_queues[pri]; 566165761Sjeff CTR5(KTR_RUNQ, "runq_remove_idx: td=%p, ts=%p pri=%d %d rqh=%p", 567164936Sjulian ts->ts_thread, ts, ts->ts_thread->td_priority, pri, rqh); 568170293Sjeff { 569170293Sjeff struct td_sched *nts; 570170293Sjeff 571170293Sjeff TAILQ_FOREACH(nts, rqh, ts_procq) 572170293Sjeff if (nts == ts) 573170293Sjeff break; 574170293Sjeff if (ts != nts) 575170293Sjeff panic("runq_remove_idx: ts %p not on rqindex %d", 576170293Sjeff ts, pri); 577170293Sjeff } 578164936Sjulian TAILQ_REMOVE(rqh, ts, ts_procq); 57972376Sjake if (TAILQ_EMPTY(rqh)) { 580165761Sjeff CTR0(KTR_RUNQ, "runq_remove_idx: empty"); 58172376Sjake runq_clrbit(rq, pri); 582165761Sjeff if (idx != NULL && *idx == pri) 583165761Sjeff *idx = (pri + 1) % RQ_NQS; 58472376Sjake } 58572376Sjake} 58699072Sjulian 587134791Sjulian/****** functions that are temporarily here ***********/ 588134791Sjulian#include <vm/uma.h> 589134791Sjulianextern struct mtx kse_zombie_lock; 590134791Sjulian 591134791Sjulian/* 592134791Sjulian * Allocate scheduler specific per-process resources. 593164936Sjulian * The thread and proc have already been linked in. 594134791Sjulian * 595134791Sjulian * Called from: 596134791Sjulian * proc_init() (UMA init method) 597134791Sjulian */ 598134791Sjulianvoid 599164936Sjuliansched_newproc(struct proc *p, struct thread *td) 600134791Sjulian{ 601134791Sjulian} 602134791Sjulian 603134791Sjulian/* 604134791Sjulian * thread is being either created or recycled. 605134791Sjulian * Fix up the per-scheduler resources associated with it. 606134791Sjulian * Called from: 607134791Sjulian * sched_fork_thread() 608134791Sjulian * thread_dtor() (*may go away) 609134791Sjulian * thread_init() (*may go away) 610134791Sjulian */ 611134791Sjulianvoid 612134791Sjuliansched_newthread(struct thread *td) 613134791Sjulian{ 614164936Sjulian struct td_sched *ts; 615134791Sjulian 616164936Sjulian ts = (struct td_sched *) (td + 1); 617164936Sjulian bzero(ts, sizeof(*ts)); 618164936Sjulian td->td_sched = ts; 619164936Sjulian ts->ts_thread = td; 620134791Sjulian} 621134791Sjulian 622134791Sjulian#endif /* KERN_SWITCH_INCLUDE */ 623