1/*-
2 * Copyright (c) 1982, 1986, 1990, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the University of
21 * California, Berkeley and its contributors.
22 * 4. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95
| 1/*-
2 * Copyright (c) 1982, 1986, 1990, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the University of
21 * California, Berkeley and its contributors.
22 * 4. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95
|
40 */
41
42#include "opt_ddb.h"
43#include "opt_ktrace.h"
44#ifdef __i386__
45#include "opt_swtch.h"
46#endif
47
48#include <sys/param.h>
49#include <sys/systm.h>
50#include <sys/condvar.h>
51#include <sys/kernel.h>
52#include <sys/ktr.h>
53#include <sys/lock.h>
54#include <sys/mutex.h>
55#include <sys/proc.h>
56#include <sys/resourcevar.h>
57#include <sys/sched.h>
58#include <sys/signalvar.h>
59#include <sys/smp.h>
60#include <sys/sx.h>
61#include <sys/sysctl.h>
62#include <sys/sysproto.h>
63#include <sys/vmmeter.h>
64#ifdef DDB
65#include <ddb/ddb.h>
66#endif
67#ifdef KTRACE
68#include <sys/uio.h>
69#include <sys/ktrace.h>
70#endif
71
72#include <machine/cpu.h>
73#ifdef SWTCH_OPTIM_STATS
74#include <machine/md_var.h>
75#endif
76
77static void sched_setup(void *dummy);
78SYSINIT(sched_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, sched_setup, NULL)
79
80int hogticks;
81int lbolt;
82
83static struct callout loadav_callout;
84static struct callout lbolt_callout;
85
86struct loadavg averunnable =
87 { {0, 0, 0}, FSCALE }; /* load average, of runnable procs */
88/*
89 * Constants for averages over 1, 5, and 15 minutes
90 * when sampling at 5 second intervals.
91 */
92static fixpt_t cexp[3] = {
93 0.9200444146293232 * FSCALE, /* exp(-1/12) */
94 0.9834714538216174 * FSCALE, /* exp(-1/60) */
95 0.9944598480048967 * FSCALE, /* exp(-1/180) */
96};
97
98/* kernel uses `FSCALE', userland (SHOULD) use kern.fscale */
99static int fscale __unused = FSCALE;
100SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, 0, FSCALE, "");
101
102static void endtsleep(void *);
103static void loadav(void *arg);
104static void lboltcb(void *arg);
105
106/*
107 * We're only looking at 7 bits of the address; everything is
108 * aligned to 4, lots of things are aligned to greater powers
109 * of 2. Shift right by 8, i.e. drop the bottom 256 worth.
110 */
111#define TABLESIZE 128
112static TAILQ_HEAD(slpquehead, thread) slpque[TABLESIZE];
113#define LOOKUP(x) (((intptr_t)(x) >> 8) & (TABLESIZE - 1))
114
115void
116sleepinit(void)
117{
118 int i;
119
120 hogticks = (hz / 10) * 2; /* Default only. */
121 for (i = 0; i < TABLESIZE; i++)
122 TAILQ_INIT(&slpque[i]);
123}
124
125/*
126 * General sleep call. Suspends the current process until a wakeup is
127 * performed on the specified identifier. The process will then be made
128 * runnable with the specified priority. Sleeps at most timo/hz seconds
129 * (0 means no timeout). If pri includes PCATCH flag, signals are checked
130 * before and after sleeping, else signals are not checked. Returns 0 if
131 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a
132 * signal needs to be delivered, ERESTART is returned if the current system
133 * call should be restarted if possible, and EINTR is returned if the system
134 * call should be interrupted by the signal (return EINTR).
135 *
136 * The mutex argument is exited before the caller is suspended, and
137 * entered before msleep returns. If priority includes the PDROP
138 * flag the mutex is not entered before returning.
139 */
140
141int
142msleep(ident, mtx, priority, wmesg, timo)
143 void *ident;
144 struct mtx *mtx;
145 int priority, timo;
146 const char *wmesg;
147{
148 struct thread *td = curthread;
149 struct proc *p = td->td_proc;
150 int sig, catch = priority & PCATCH;
151 int rval = 0;
152 WITNESS_SAVE_DECL(mtx);
153
154#ifdef KTRACE
155 if (KTRPOINT(td, KTR_CSW))
156 ktrcsw(1, 0);
157#endif
158 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mtx->mtx_object,
159 "Sleeping on \"%s\"", wmesg);
160 KASSERT(timo != 0 || mtx_owned(&Giant) || mtx != NULL,
161 ("sleeping without a mutex"));
162 /*
163 * If we are capable of async syscalls and there isn't already
164 * another one ready to return, start a new thread
165 * and queue it as ready to run. Note that there is danger here
166 * because we need to make sure that we don't sleep allocating
167 * the thread (recursion here might be bad).
168 */
169 mtx_lock_spin(&sched_lock);
170 if (p->p_flag & P_THREADED || p->p_numthreads > 1) {
171 /*
172 * Just don't bother if we are exiting
173 * and not the exiting thread or thread was marked as
174 * interrupted.
175 */
176 if (catch &&
177 (((p->p_flag & P_WEXIT) && (p->p_singlethread != td)) ||
178 (td->td_flags & TDF_INTERRUPT))) {
179 td->td_flags &= ~TDF_INTERRUPT;
180 mtx_unlock_spin(&sched_lock);
181 return (EINTR);
182 }
183 }
184 if (cold ) {
185 /*
186 * During autoconfiguration, just give interrupts
187 * a chance, then just return.
188 * Don't run any other procs or panic below,
189 * in case this is the idle process and already asleep.
190 */
191 if (mtx != NULL && priority & PDROP)
192 mtx_unlock(mtx);
193 mtx_unlock_spin(&sched_lock);
194 return (0);
195 }
196
197 DROP_GIANT();
198
199 if (mtx != NULL) {
200 mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED);
201 WITNESS_SAVE(&mtx->mtx_object, mtx);
202 mtx_unlock(mtx);
203 if (priority & PDROP)
204 mtx = NULL;
205 }
206
207 KASSERT(p != NULL, ("msleep1"));
208 KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep"));
209
210 CTR5(KTR_PROC, "msleep: thread %p (pid %d, %s) on %s (%p)",
211 td, p->p_pid, p->p_comm, wmesg, ident);
212
213 td->td_wchan = ident;
214 td->td_wmesg = wmesg;
215 TAILQ_INSERT_TAIL(&slpque[LOOKUP(ident)], td, td_slpq);
216 TD_SET_ON_SLEEPQ(td);
217 if (timo)
218 callout_reset(&td->td_slpcallout, timo, endtsleep, td);
219 /*
220 * We put ourselves on the sleep queue and start our timeout
221 * before calling thread_suspend_check, as we could stop there, and
222 * a wakeup or a SIGCONT (or both) could occur while we were stopped.
223 * without resuming us, thus we must be ready for sleep
224 * when cursig is called. If the wakeup happens while we're
225 * stopped, td->td_wchan will be 0 upon return from cursig.
226 */
227 if (catch) {
228 CTR3(KTR_PROC, "msleep caught: thread %p (pid %d, %s)", td,
229 p->p_pid, p->p_comm);
230 td->td_flags |= TDF_SINTR;
231 mtx_unlock_spin(&sched_lock);
232 PROC_LOCK(p);
233 mtx_lock(&p->p_sigacts->ps_mtx);
234 sig = cursig(td);
235 mtx_unlock(&p->p_sigacts->ps_mtx);
236 if (sig == 0 && thread_suspend_check(1))
237 sig = SIGSTOP;
238 mtx_lock_spin(&sched_lock);
239 PROC_UNLOCK(p);
240 if (sig != 0) {
241 if (TD_ON_SLEEPQ(td))
242 unsleep(td);
243 } else if (!TD_ON_SLEEPQ(td))
244 catch = 0;
245 } else
246 sig = 0;
247
248 /*
249 * Let the scheduler know we're about to voluntarily go to sleep.
250 */
251 sched_sleep(td, priority & PRIMASK);
252
253 if (TD_ON_SLEEPQ(td)) {
254 p->p_stats->p_ru.ru_nvcsw++;
255 TD_SET_SLEEPING(td);
256 mi_switch();
257 }
258 /*
259 * We're awake from voluntary sleep.
260 */
261 CTR3(KTR_PROC, "msleep resume: thread %p (pid %d, %s)", td, p->p_pid,
262 p->p_comm);
263 KASSERT(TD_IS_RUNNING(td), ("running but not TDS_RUNNING"));
264 td->td_flags &= ~TDF_SINTR;
265 if (td->td_flags & TDF_TIMEOUT) {
266 td->td_flags &= ~TDF_TIMEOUT;
267 if (sig == 0)
268 rval = EWOULDBLOCK;
269 } else if (td->td_flags & TDF_TIMOFAIL) {
270 td->td_flags &= ~TDF_TIMOFAIL;
271 } else if (timo && callout_stop(&td->td_slpcallout) == 0) {
272 /*
273 * This isn't supposed to be pretty. If we are here, then
274 * the endtsleep() callout is currently executing on another
275 * CPU and is either spinning on the sched_lock or will be
276 * soon. If we don't synchronize here, there is a chance
277 * that this process may msleep() again before the callout
278 * has a chance to run and the callout may end up waking up
279 * the wrong msleep(). Yuck.
280 */
281 TD_SET_SLEEPING(td);
282 p->p_stats->p_ru.ru_nivcsw++;
283 mi_switch();
284 td->td_flags &= ~TDF_TIMOFAIL;
285 }
286 if ((td->td_flags & TDF_INTERRUPT) && (priority & PCATCH) &&
287 (rval == 0)) {
288 td->td_flags &= ~TDF_INTERRUPT;
289 rval = EINTR;
290 }
291 mtx_unlock_spin(&sched_lock);
292
293 if (rval == 0 && catch) {
294 PROC_LOCK(p);
295 /* XXX: shouldn't we always be calling cursig() */
296 mtx_lock(&p->p_sigacts->ps_mtx);
297 if (sig != 0 || (sig = cursig(td))) {
298 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig))
299 rval = EINTR;
300 else
301 rval = ERESTART;
302 }
303 mtx_unlock(&p->p_sigacts->ps_mtx);
304 PROC_UNLOCK(p);
305 }
306#ifdef KTRACE
307 if (KTRPOINT(td, KTR_CSW))
308 ktrcsw(0, 0);
309#endif
310 PICKUP_GIANT();
311 if (mtx != NULL) {
312 mtx_lock(mtx);
313 WITNESS_RESTORE(&mtx->mtx_object, mtx);
314 }
315 return (rval);
316}
317
318/*
319 * Implement timeout for msleep()
320 *
321 * If process hasn't been awakened (wchan non-zero),
322 * set timeout flag and undo the sleep. If proc
323 * is stopped, just unsleep so it will remain stopped.
324 * MP-safe, called without the Giant mutex.
325 */
326static void
327endtsleep(arg)
328 void *arg;
329{
330 register struct thread *td = arg;
331
332 CTR3(KTR_PROC, "endtsleep: thread %p (pid %d, %s)",
333 td, td->td_proc->p_pid, td->td_proc->p_comm);
334 mtx_lock_spin(&sched_lock);
335 /*
336 * This is the other half of the synchronization with msleep()
337 * described above. If the TDS_TIMEOUT flag is set, we lost the
338 * race and just need to put the process back on the runqueue.
339 */
340 if (TD_ON_SLEEPQ(td)) {
341 TAILQ_REMOVE(&slpque[LOOKUP(td->td_wchan)], td, td_slpq);
342 TD_CLR_ON_SLEEPQ(td);
343 td->td_flags |= TDF_TIMEOUT;
344 td->td_wmesg = NULL;
345 } else {
346 td->td_flags |= TDF_TIMOFAIL;
347 }
348 TD_CLR_SLEEPING(td);
349 setrunnable(td);
350 mtx_unlock_spin(&sched_lock);
351}
352
353/*
354 * Abort a thread, as if an interrupt had occured. Only abort
355 * interruptable waits (unfortunatly it isn't only safe to abort others).
356 * This is about identical to cv_abort().
357 * Think about merging them?
358 * Also, whatever the signal code does...
359 */
360void
361abortsleep(struct thread *td)
362{
363
364 mtx_assert(&sched_lock, MA_OWNED);
365 /*
366 * If the TDF_TIMEOUT flag is set, just leave. A
367 * timeout is scheduled anyhow.
368 */
369 if ((td->td_flags & (TDF_TIMEOUT | TDF_SINTR)) == TDF_SINTR) {
370 if (TD_ON_SLEEPQ(td)) {
371 unsleep(td);
372 TD_CLR_SLEEPING(td);
373 setrunnable(td);
374 }
375 }
376}
377
378/*
379 * Remove a process from its wait queue
380 */
381void
382unsleep(struct thread *td)
383{
384
385 mtx_lock_spin(&sched_lock);
386 if (TD_ON_SLEEPQ(td)) {
387 TAILQ_REMOVE(&slpque[LOOKUP(td->td_wchan)], td, td_slpq);
388 TD_CLR_ON_SLEEPQ(td);
389 td->td_wmesg = NULL;
390 }
391 mtx_unlock_spin(&sched_lock);
392}
393
394/*
395 * Make all processes sleeping on the specified identifier runnable.
396 */
397void
398wakeup(ident)
399 register void *ident;
400{
401 register struct slpquehead *qp;
402 register struct thread *td;
403 struct thread *ntd;
404 struct proc *p;
405
406 mtx_lock_spin(&sched_lock);
407 qp = &slpque[LOOKUP(ident)];
408restart:
409 for (td = TAILQ_FIRST(qp); td != NULL; td = ntd) {
410 ntd = TAILQ_NEXT(td, td_slpq);
411 if (td->td_wchan == ident) {
412 unsleep(td);
413 TD_CLR_SLEEPING(td);
414 setrunnable(td);
415 p = td->td_proc;
416 CTR3(KTR_PROC,"wakeup: thread %p (pid %d, %s)",
417 td, p->p_pid, p->p_comm);
418 goto restart;
419 }
420 }
421 mtx_unlock_spin(&sched_lock);
422}
423
424/*
425 * Make a process sleeping on the specified identifier runnable.
426 * May wake more than one process if a target process is currently
427 * swapped out.
428 */
429void
430wakeup_one(ident)
431 register void *ident;
432{
433 register struct slpquehead *qp;
434 register struct thread *td;
435 register struct proc *p;
436 struct thread *ntd;
437
438 mtx_lock_spin(&sched_lock);
439 qp = &slpque[LOOKUP(ident)];
440 for (td = TAILQ_FIRST(qp); td != NULL; td = ntd) {
441 ntd = TAILQ_NEXT(td, td_slpq);
442 if (td->td_wchan == ident) {
443 unsleep(td);
444 TD_CLR_SLEEPING(td);
445 setrunnable(td);
446 p = td->td_proc;
447 CTR3(KTR_PROC,"wakeup1: thread %p (pid %d, %s)",
448 td, p->p_pid, p->p_comm);
449 break;
450 }
451 }
452 mtx_unlock_spin(&sched_lock);
453}
454
455/*
456 * The machine independent parts of mi_switch().
457 */
458void
459mi_switch(void)
460{
461 struct bintime new_switchtime;
462 struct thread *td;
| 40 */
41
42#include "opt_ddb.h"
43#include "opt_ktrace.h"
44#ifdef __i386__
45#include "opt_swtch.h"
46#endif
47
48#include <sys/param.h>
49#include <sys/systm.h>
50#include <sys/condvar.h>
51#include <sys/kernel.h>
52#include <sys/ktr.h>
53#include <sys/lock.h>
54#include <sys/mutex.h>
55#include <sys/proc.h>
56#include <sys/resourcevar.h>
57#include <sys/sched.h>
58#include <sys/signalvar.h>
59#include <sys/smp.h>
60#include <sys/sx.h>
61#include <sys/sysctl.h>
62#include <sys/sysproto.h>
63#include <sys/vmmeter.h>
64#ifdef DDB
65#include <ddb/ddb.h>
66#endif
67#ifdef KTRACE
68#include <sys/uio.h>
69#include <sys/ktrace.h>
70#endif
71
72#include <machine/cpu.h>
73#ifdef SWTCH_OPTIM_STATS
74#include <machine/md_var.h>
75#endif
76
77static void sched_setup(void *dummy);
78SYSINIT(sched_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, sched_setup, NULL)
79
80int hogticks;
81int lbolt;
82
83static struct callout loadav_callout;
84static struct callout lbolt_callout;
85
86struct loadavg averunnable =
87 { {0, 0, 0}, FSCALE }; /* load average, of runnable procs */
88/*
89 * Constants for averages over 1, 5, and 15 minutes
90 * when sampling at 5 second intervals.
91 */
92static fixpt_t cexp[3] = {
93 0.9200444146293232 * FSCALE, /* exp(-1/12) */
94 0.9834714538216174 * FSCALE, /* exp(-1/60) */
95 0.9944598480048967 * FSCALE, /* exp(-1/180) */
96};
97
98/* kernel uses `FSCALE', userland (SHOULD) use kern.fscale */
99static int fscale __unused = FSCALE;
100SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, 0, FSCALE, "");
101
102static void endtsleep(void *);
103static void loadav(void *arg);
104static void lboltcb(void *arg);
105
106/*
107 * We're only looking at 7 bits of the address; everything is
108 * aligned to 4, lots of things are aligned to greater powers
109 * of 2. Shift right by 8, i.e. drop the bottom 256 worth.
110 */
111#define TABLESIZE 128
112static TAILQ_HEAD(slpquehead, thread) slpque[TABLESIZE];
113#define LOOKUP(x) (((intptr_t)(x) >> 8) & (TABLESIZE - 1))
114
115void
116sleepinit(void)
117{
118 int i;
119
120 hogticks = (hz / 10) * 2; /* Default only. */
121 for (i = 0; i < TABLESIZE; i++)
122 TAILQ_INIT(&slpque[i]);
123}
124
125/*
126 * General sleep call. Suspends the current process until a wakeup is
127 * performed on the specified identifier. The process will then be made
128 * runnable with the specified priority. Sleeps at most timo/hz seconds
129 * (0 means no timeout). If pri includes PCATCH flag, signals are checked
130 * before and after sleeping, else signals are not checked. Returns 0 if
131 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a
132 * signal needs to be delivered, ERESTART is returned if the current system
133 * call should be restarted if possible, and EINTR is returned if the system
134 * call should be interrupted by the signal (return EINTR).
135 *
136 * The mutex argument is exited before the caller is suspended, and
137 * entered before msleep returns. If priority includes the PDROP
138 * flag the mutex is not entered before returning.
139 */
140
141int
142msleep(ident, mtx, priority, wmesg, timo)
143 void *ident;
144 struct mtx *mtx;
145 int priority, timo;
146 const char *wmesg;
147{
148 struct thread *td = curthread;
149 struct proc *p = td->td_proc;
150 int sig, catch = priority & PCATCH;
151 int rval = 0;
152 WITNESS_SAVE_DECL(mtx);
153
154#ifdef KTRACE
155 if (KTRPOINT(td, KTR_CSW))
156 ktrcsw(1, 0);
157#endif
158 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, &mtx->mtx_object,
159 "Sleeping on \"%s\"", wmesg);
160 KASSERT(timo != 0 || mtx_owned(&Giant) || mtx != NULL,
161 ("sleeping without a mutex"));
162 /*
163 * If we are capable of async syscalls and there isn't already
164 * another one ready to return, start a new thread
165 * and queue it as ready to run. Note that there is danger here
166 * because we need to make sure that we don't sleep allocating
167 * the thread (recursion here might be bad).
168 */
169 mtx_lock_spin(&sched_lock);
170 if (p->p_flag & P_THREADED || p->p_numthreads > 1) {
171 /*
172 * Just don't bother if we are exiting
173 * and not the exiting thread or thread was marked as
174 * interrupted.
175 */
176 if (catch &&
177 (((p->p_flag & P_WEXIT) && (p->p_singlethread != td)) ||
178 (td->td_flags & TDF_INTERRUPT))) {
179 td->td_flags &= ~TDF_INTERRUPT;
180 mtx_unlock_spin(&sched_lock);
181 return (EINTR);
182 }
183 }
184 if (cold ) {
185 /*
186 * During autoconfiguration, just give interrupts
187 * a chance, then just return.
188 * Don't run any other procs or panic below,
189 * in case this is the idle process and already asleep.
190 */
191 if (mtx != NULL && priority & PDROP)
192 mtx_unlock(mtx);
193 mtx_unlock_spin(&sched_lock);
194 return (0);
195 }
196
197 DROP_GIANT();
198
199 if (mtx != NULL) {
200 mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED);
201 WITNESS_SAVE(&mtx->mtx_object, mtx);
202 mtx_unlock(mtx);
203 if (priority & PDROP)
204 mtx = NULL;
205 }
206
207 KASSERT(p != NULL, ("msleep1"));
208 KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep"));
209
210 CTR5(KTR_PROC, "msleep: thread %p (pid %d, %s) on %s (%p)",
211 td, p->p_pid, p->p_comm, wmesg, ident);
212
213 td->td_wchan = ident;
214 td->td_wmesg = wmesg;
215 TAILQ_INSERT_TAIL(&slpque[LOOKUP(ident)], td, td_slpq);
216 TD_SET_ON_SLEEPQ(td);
217 if (timo)
218 callout_reset(&td->td_slpcallout, timo, endtsleep, td);
219 /*
220 * We put ourselves on the sleep queue and start our timeout
221 * before calling thread_suspend_check, as we could stop there, and
222 * a wakeup or a SIGCONT (or both) could occur while we were stopped.
223 * without resuming us, thus we must be ready for sleep
224 * when cursig is called. If the wakeup happens while we're
225 * stopped, td->td_wchan will be 0 upon return from cursig.
226 */
227 if (catch) {
228 CTR3(KTR_PROC, "msleep caught: thread %p (pid %d, %s)", td,
229 p->p_pid, p->p_comm);
230 td->td_flags |= TDF_SINTR;
231 mtx_unlock_spin(&sched_lock);
232 PROC_LOCK(p);
233 mtx_lock(&p->p_sigacts->ps_mtx);
234 sig = cursig(td);
235 mtx_unlock(&p->p_sigacts->ps_mtx);
236 if (sig == 0 && thread_suspend_check(1))
237 sig = SIGSTOP;
238 mtx_lock_spin(&sched_lock);
239 PROC_UNLOCK(p);
240 if (sig != 0) {
241 if (TD_ON_SLEEPQ(td))
242 unsleep(td);
243 } else if (!TD_ON_SLEEPQ(td))
244 catch = 0;
245 } else
246 sig = 0;
247
248 /*
249 * Let the scheduler know we're about to voluntarily go to sleep.
250 */
251 sched_sleep(td, priority & PRIMASK);
252
253 if (TD_ON_SLEEPQ(td)) {
254 p->p_stats->p_ru.ru_nvcsw++;
255 TD_SET_SLEEPING(td);
256 mi_switch();
257 }
258 /*
259 * We're awake from voluntary sleep.
260 */
261 CTR3(KTR_PROC, "msleep resume: thread %p (pid %d, %s)", td, p->p_pid,
262 p->p_comm);
263 KASSERT(TD_IS_RUNNING(td), ("running but not TDS_RUNNING"));
264 td->td_flags &= ~TDF_SINTR;
265 if (td->td_flags & TDF_TIMEOUT) {
266 td->td_flags &= ~TDF_TIMEOUT;
267 if (sig == 0)
268 rval = EWOULDBLOCK;
269 } else if (td->td_flags & TDF_TIMOFAIL) {
270 td->td_flags &= ~TDF_TIMOFAIL;
271 } else if (timo && callout_stop(&td->td_slpcallout) == 0) {
272 /*
273 * This isn't supposed to be pretty. If we are here, then
274 * the endtsleep() callout is currently executing on another
275 * CPU and is either spinning on the sched_lock or will be
276 * soon. If we don't synchronize here, there is a chance
277 * that this process may msleep() again before the callout
278 * has a chance to run and the callout may end up waking up
279 * the wrong msleep(). Yuck.
280 */
281 TD_SET_SLEEPING(td);
282 p->p_stats->p_ru.ru_nivcsw++;
283 mi_switch();
284 td->td_flags &= ~TDF_TIMOFAIL;
285 }
286 if ((td->td_flags & TDF_INTERRUPT) && (priority & PCATCH) &&
287 (rval == 0)) {
288 td->td_flags &= ~TDF_INTERRUPT;
289 rval = EINTR;
290 }
291 mtx_unlock_spin(&sched_lock);
292
293 if (rval == 0 && catch) {
294 PROC_LOCK(p);
295 /* XXX: shouldn't we always be calling cursig() */
296 mtx_lock(&p->p_sigacts->ps_mtx);
297 if (sig != 0 || (sig = cursig(td))) {
298 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig))
299 rval = EINTR;
300 else
301 rval = ERESTART;
302 }
303 mtx_unlock(&p->p_sigacts->ps_mtx);
304 PROC_UNLOCK(p);
305 }
306#ifdef KTRACE
307 if (KTRPOINT(td, KTR_CSW))
308 ktrcsw(0, 0);
309#endif
310 PICKUP_GIANT();
311 if (mtx != NULL) {
312 mtx_lock(mtx);
313 WITNESS_RESTORE(&mtx->mtx_object, mtx);
314 }
315 return (rval);
316}
317
318/*
319 * Implement timeout for msleep()
320 *
321 * If process hasn't been awakened (wchan non-zero),
322 * set timeout flag and undo the sleep. If proc
323 * is stopped, just unsleep so it will remain stopped.
324 * MP-safe, called without the Giant mutex.
325 */
326static void
327endtsleep(arg)
328 void *arg;
329{
330 register struct thread *td = arg;
331
332 CTR3(KTR_PROC, "endtsleep: thread %p (pid %d, %s)",
333 td, td->td_proc->p_pid, td->td_proc->p_comm);
334 mtx_lock_spin(&sched_lock);
335 /*
336 * This is the other half of the synchronization with msleep()
337 * described above. If the TDS_TIMEOUT flag is set, we lost the
338 * race and just need to put the process back on the runqueue.
339 */
340 if (TD_ON_SLEEPQ(td)) {
341 TAILQ_REMOVE(&slpque[LOOKUP(td->td_wchan)], td, td_slpq);
342 TD_CLR_ON_SLEEPQ(td);
343 td->td_flags |= TDF_TIMEOUT;
344 td->td_wmesg = NULL;
345 } else {
346 td->td_flags |= TDF_TIMOFAIL;
347 }
348 TD_CLR_SLEEPING(td);
349 setrunnable(td);
350 mtx_unlock_spin(&sched_lock);
351}
352
353/*
354 * Abort a thread, as if an interrupt had occured. Only abort
355 * interruptable waits (unfortunatly it isn't only safe to abort others).
356 * This is about identical to cv_abort().
357 * Think about merging them?
358 * Also, whatever the signal code does...
359 */
360void
361abortsleep(struct thread *td)
362{
363
364 mtx_assert(&sched_lock, MA_OWNED);
365 /*
366 * If the TDF_TIMEOUT flag is set, just leave. A
367 * timeout is scheduled anyhow.
368 */
369 if ((td->td_flags & (TDF_TIMEOUT | TDF_SINTR)) == TDF_SINTR) {
370 if (TD_ON_SLEEPQ(td)) {
371 unsleep(td);
372 TD_CLR_SLEEPING(td);
373 setrunnable(td);
374 }
375 }
376}
377
378/*
379 * Remove a process from its wait queue
380 */
381void
382unsleep(struct thread *td)
383{
384
385 mtx_lock_spin(&sched_lock);
386 if (TD_ON_SLEEPQ(td)) {
387 TAILQ_REMOVE(&slpque[LOOKUP(td->td_wchan)], td, td_slpq);
388 TD_CLR_ON_SLEEPQ(td);
389 td->td_wmesg = NULL;
390 }
391 mtx_unlock_spin(&sched_lock);
392}
393
394/*
395 * Make all processes sleeping on the specified identifier runnable.
396 */
397void
398wakeup(ident)
399 register void *ident;
400{
401 register struct slpquehead *qp;
402 register struct thread *td;
403 struct thread *ntd;
404 struct proc *p;
405
406 mtx_lock_spin(&sched_lock);
407 qp = &slpque[LOOKUP(ident)];
408restart:
409 for (td = TAILQ_FIRST(qp); td != NULL; td = ntd) {
410 ntd = TAILQ_NEXT(td, td_slpq);
411 if (td->td_wchan == ident) {
412 unsleep(td);
413 TD_CLR_SLEEPING(td);
414 setrunnable(td);
415 p = td->td_proc;
416 CTR3(KTR_PROC,"wakeup: thread %p (pid %d, %s)",
417 td, p->p_pid, p->p_comm);
418 goto restart;
419 }
420 }
421 mtx_unlock_spin(&sched_lock);
422}
423
424/*
425 * Make a process sleeping on the specified identifier runnable.
426 * May wake more than one process if a target process is currently
427 * swapped out.
428 */
429void
430wakeup_one(ident)
431 register void *ident;
432{
433 register struct slpquehead *qp;
434 register struct thread *td;
435 register struct proc *p;
436 struct thread *ntd;
437
438 mtx_lock_spin(&sched_lock);
439 qp = &slpque[LOOKUP(ident)];
440 for (td = TAILQ_FIRST(qp); td != NULL; td = ntd) {
441 ntd = TAILQ_NEXT(td, td_slpq);
442 if (td->td_wchan == ident) {
443 unsleep(td);
444 TD_CLR_SLEEPING(td);
445 setrunnable(td);
446 p = td->td_proc;
447 CTR3(KTR_PROC,"wakeup1: thread %p (pid %d, %s)",
448 td, p->p_pid, p->p_comm);
449 break;
450 }
451 }
452 mtx_unlock_spin(&sched_lock);
453}
454
455/*
456 * The machine independent parts of mi_switch().
457 */
458void
459mi_switch(void)
460{
461 struct bintime new_switchtime;
462 struct thread *td;
|
522 newtd = choosethread();
523 if (td != newtd)
524 cpu_switch(td, newtd); /* SHAZAM!! */
525#ifdef SWTCH_OPTIM_STATS
526 else
527 stupid_switch++;
528#endif
529#else
530 cpu_switch(); /* SHAZAM!!*/
531#endif
532
533 sched_lock.mtx_recurse = sched_nest;
534 sched_lock.mtx_lock = (uintptr_t)td;
535 sched_switchin(td);
536
537 /*
538 * Start setting up stats etc. for the incoming thread.
539 * Similar code in fork_exit() is returned to by cpu_switch()
540 * in the case of a new thread/process.
541 */
542 CTR3(KTR_PROC, "mi_switch: new thread %p (pid %d, %s)", td, p->p_pid,
543 p->p_comm);
544 if (PCPU_GET(switchtime.sec) == 0)
545 binuptime(PCPU_PTR(switchtime));
546 PCPU_SET(switchticks, ticks);
547
548 /*
549 * Call the switchin function while still holding the scheduler lock
550 * (used by the idlezero code and the general page-zeroing code)
551 */
552 if (td->td_switchin)
553 td->td_switchin();
554
555 /*
556 * If the last thread was exiting, finish cleaning it up.
557 */
558 if ((td = PCPU_GET(deadthread))) {
559 PCPU_SET(deadthread, NULL);
560 thread_stash(td);
561 }
562}
563
564/*
565 * Change process state to be runnable,
566 * placing it on the run queue if it is in memory,
567 * and awakening the swapper if it isn't in memory.
568 */
569void
570setrunnable(struct thread *td)
571{
572 struct proc *p = td->td_proc;
573
574 mtx_assert(&sched_lock, MA_OWNED);
575 switch (p->p_state) {
576 case PRS_ZOMBIE:
577 panic("setrunnable(1)");
578 default:
579 break;
580 }
581 switch (td->td_state) {
582 case TDS_RUNNING:
583 case TDS_RUNQ:
584 return;
585 case TDS_INHIBITED:
586 /*
587 * If we are only inhibited because we are swapped out
588 * then arange to swap in this process. Otherwise just return.
589 */
590 if (td->td_inhibitors != TDI_SWAPPED)
591 return;
592 case TDS_CAN_RUN:
593 break;
594 default:
595 printf("state is 0x%x", td->td_state);
596 panic("setrunnable(2)");
597 }
598 if ((p->p_sflag & PS_INMEM) == 0) {
599 if ((p->p_sflag & PS_SWAPPINGIN) == 0) {
600 p->p_sflag |= PS_SWAPINREQ;
601 wakeup(&proc0);
602 }
603 } else
604 sched_wakeup(td);
605}
606
607/*
608 * Compute a tenex style load average of a quantity on
609 * 1, 5 and 15 minute intervals.
610 * XXXKSE Needs complete rewrite when correct info is available.
611 * Completely Bogus.. only works with 1:1 (but compiles ok now :-)
612 */
613static void
614loadav(void *arg)
615{
616 int i, nrun;
617 struct loadavg *avg;
618 struct proc *p;
619 struct thread *td;
620
621 avg = &averunnable;
622 sx_slock(&allproc_lock);
623 nrun = 0;
624 FOREACH_PROC_IN_SYSTEM(p) {
625 FOREACH_THREAD_IN_PROC(p, td) {
626 switch (td->td_state) {
627 case TDS_RUNQ:
628 case TDS_RUNNING:
629 if ((p->p_flag & P_NOLOAD) != 0)
630 goto nextproc;
631 nrun++; /* XXXKSE */
632 default:
633 break;
634 }
635nextproc:
636 continue;
637 }
638 }
639 sx_sunlock(&allproc_lock);
640 for (i = 0; i < 3; i++)
641 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
642 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
643
644 /*
645 * Schedule the next update to occur after 5 seconds, but add a
646 * random variation to avoid synchronisation with processes that
647 * run at regular intervals.
648 */
649 callout_reset(&loadav_callout, hz * 4 + (int)(random() % (hz * 2 + 1)),
650 loadav, NULL);
651}
652
653static void
654lboltcb(void *arg)
655{
656 wakeup(&lbolt);
657 callout_reset(&lbolt_callout, hz, lboltcb, NULL);
658}
659
660/* ARGSUSED */
661static void
662sched_setup(dummy)
663 void *dummy;
664{
665 callout_init(&loadav_callout, 0);
666 callout_init(&lbolt_callout, 1);
667
668 /* Kick off timeout driven events by calling first time. */
669 loadav(NULL);
670 lboltcb(NULL);
671}
672
673/*
674 * General purpose yield system call
675 */
676int
677yield(struct thread *td, struct yield_args *uap)
678{
679 struct ksegrp *kg = td->td_ksegrp;
680
681 mtx_assert(&Giant, MA_NOTOWNED);
682 mtx_lock_spin(&sched_lock);
683 kg->kg_proc->p_stats->p_ru.ru_nvcsw++;
684 sched_prio(td, PRI_MAX_TIMESHARE);
685 mi_switch();
686 mtx_unlock_spin(&sched_lock);
687 td->td_retval[0] = 0;
688
689 return (0);
690}
691
| 522 newtd = choosethread();
523 if (td != newtd)
524 cpu_switch(td, newtd); /* SHAZAM!! */
525#ifdef SWTCH_OPTIM_STATS
526 else
527 stupid_switch++;
528#endif
529#else
530 cpu_switch(); /* SHAZAM!!*/
531#endif
532
533 sched_lock.mtx_recurse = sched_nest;
534 sched_lock.mtx_lock = (uintptr_t)td;
535 sched_switchin(td);
536
537 /*
538 * Start setting up stats etc. for the incoming thread.
539 * Similar code in fork_exit() is returned to by cpu_switch()
540 * in the case of a new thread/process.
541 */
542 CTR3(KTR_PROC, "mi_switch: new thread %p (pid %d, %s)", td, p->p_pid,
543 p->p_comm);
544 if (PCPU_GET(switchtime.sec) == 0)
545 binuptime(PCPU_PTR(switchtime));
546 PCPU_SET(switchticks, ticks);
547
548 /*
549 * Call the switchin function while still holding the scheduler lock
550 * (used by the idlezero code and the general page-zeroing code)
551 */
552 if (td->td_switchin)
553 td->td_switchin();
554
555 /*
556 * If the last thread was exiting, finish cleaning it up.
557 */
558 if ((td = PCPU_GET(deadthread))) {
559 PCPU_SET(deadthread, NULL);
560 thread_stash(td);
561 }
562}
563
564/*
565 * Change process state to be runnable,
566 * placing it on the run queue if it is in memory,
567 * and awakening the swapper if it isn't in memory.
568 */
569void
570setrunnable(struct thread *td)
571{
572 struct proc *p = td->td_proc;
573
574 mtx_assert(&sched_lock, MA_OWNED);
575 switch (p->p_state) {
576 case PRS_ZOMBIE:
577 panic("setrunnable(1)");
578 default:
579 break;
580 }
581 switch (td->td_state) {
582 case TDS_RUNNING:
583 case TDS_RUNQ:
584 return;
585 case TDS_INHIBITED:
586 /*
587 * If we are only inhibited because we are swapped out
588 * then arange to swap in this process. Otherwise just return.
589 */
590 if (td->td_inhibitors != TDI_SWAPPED)
591 return;
592 case TDS_CAN_RUN:
593 break;
594 default:
595 printf("state is 0x%x", td->td_state);
596 panic("setrunnable(2)");
597 }
598 if ((p->p_sflag & PS_INMEM) == 0) {
599 if ((p->p_sflag & PS_SWAPPINGIN) == 0) {
600 p->p_sflag |= PS_SWAPINREQ;
601 wakeup(&proc0);
602 }
603 } else
604 sched_wakeup(td);
605}
606
607/*
608 * Compute a tenex style load average of a quantity on
609 * 1, 5 and 15 minute intervals.
610 * XXXKSE Needs complete rewrite when correct info is available.
611 * Completely Bogus.. only works with 1:1 (but compiles ok now :-)
612 */
613static void
614loadav(void *arg)
615{
616 int i, nrun;
617 struct loadavg *avg;
618 struct proc *p;
619 struct thread *td;
620
621 avg = &averunnable;
622 sx_slock(&allproc_lock);
623 nrun = 0;
624 FOREACH_PROC_IN_SYSTEM(p) {
625 FOREACH_THREAD_IN_PROC(p, td) {
626 switch (td->td_state) {
627 case TDS_RUNQ:
628 case TDS_RUNNING:
629 if ((p->p_flag & P_NOLOAD) != 0)
630 goto nextproc;
631 nrun++; /* XXXKSE */
632 default:
633 break;
634 }
635nextproc:
636 continue;
637 }
638 }
639 sx_sunlock(&allproc_lock);
640 for (i = 0; i < 3; i++)
641 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
642 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
643
644 /*
645 * Schedule the next update to occur after 5 seconds, but add a
646 * random variation to avoid synchronisation with processes that
647 * run at regular intervals.
648 */
649 callout_reset(&loadav_callout, hz * 4 + (int)(random() % (hz * 2 + 1)),
650 loadav, NULL);
651}
652
653static void
654lboltcb(void *arg)
655{
656 wakeup(&lbolt);
657 callout_reset(&lbolt_callout, hz, lboltcb, NULL);
658}
659
660/* ARGSUSED */
661static void
662sched_setup(dummy)
663 void *dummy;
664{
665 callout_init(&loadav_callout, 0);
666 callout_init(&lbolt_callout, 1);
667
668 /* Kick off timeout driven events by calling first time. */
669 loadav(NULL);
670 lboltcb(NULL);
671}
672
673/*
674 * General purpose yield system call
675 */
676int
677yield(struct thread *td, struct yield_args *uap)
678{
679 struct ksegrp *kg = td->td_ksegrp;
680
681 mtx_assert(&Giant, MA_NOTOWNED);
682 mtx_lock_spin(&sched_lock);
683 kg->kg_proc->p_stats->p_ru.ru_nvcsw++;
684 sched_prio(td, PRI_MAX_TIMESHARE);
685 mi_switch();
686 mtx_unlock_spin(&sched_lock);
687 td->td_retval[0] = 0;
688
689 return (0);
690}
691
|