1/*
2 * Copyright (C) 2003 Daniel M. Eischen <deischen@freebsd.org>
3 * Copyright (C) 2002 Jonathon Mini <mini@freebsd.org>
4 * Copyright (c) 1995-1998 John Birrell <jb@cimlogic.com.au>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by John Birrell.
18 * 4. Neither the name of the author nor the names of any co-contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 */
35#include <sys/cdefs.h>
36__FBSDID("$FreeBSD: head/lib/libkse/thread/thr_kern.c 115173 2003-05-19 23:04:50Z deischen $");
37
38#include <sys/types.h>
39#include <sys/kse.h>
40#include <sys/signalvar.h>
41#include <sys/queue.h>
42#include <machine/atomic.h>
43
44#include <assert.h>
45#include <errno.h>
46#include <signal.h>
47#include <stdlib.h>
48#include <string.h>
49#include <time.h>
50#include <ucontext.h>
51#include <unistd.h>
52
53#include "atomic_ops.h"
54#include "thr_private.h"
55#include "pthread_md.h"
56#include "libc_private.h"
57
58/*#define DEBUG_THREAD_KERN */
59#ifdef DEBUG_THREAD_KERN
60#define DBG_MSG stdout_debug
61#else
62#define DBG_MSG(x...)
63#endif
64
65/*
66 * Define a high water mark for the maximum number of threads that
67 * will be cached. Once this level is reached, any extra threads
68 * will be free()'d.
69 *
70 * XXX - It doesn't make sense to worry about the maximum number of
71 * KSEs that we can cache because the system will limit us to
72 * something *much* less than the maximum number of threads
73 * that we can have. Disregarding KSEs in their own group,
74 * the maximum number of KSEs is the number of processors in
75 * the system.
76 */
77#define MAX_CACHED_THREADS 100
78#define KSE_STACKSIZE 16384
79
80#define KSE_SET_MBOX(kse, thrd) \
81 (kse)->k_mbx.km_curthread = &(thrd)->tmbx
82
83#define KSE_SET_EXITED(kse) (kse)->k_flags |= KF_EXITED
84
85/*
86 * Macros for manipulating the run queues. The priority queue
87 * routines use the thread's pqe link and also handle the setting
88 * and clearing of the thread's THR_FLAGS_IN_RUNQ flag.
89 */
90#define KSE_RUNQ_INSERT_HEAD(kse, thrd) \
91 _pq_insert_head(&(kse)->k_schedq->sq_runq, thrd)
92#define KSE_RUNQ_INSERT_TAIL(kse, thrd) \
93 _pq_insert_tail(&(kse)->k_schedq->sq_runq, thrd)
94#define KSE_RUNQ_REMOVE(kse, thrd) \
95 _pq_remove(&(kse)->k_schedq->sq_runq, thrd)
96#define KSE_RUNQ_FIRST(kse) _pq_first(&(kse)->k_schedq->sq_runq)
97
98#define KSE_RUNQ_THREADS(kse) ((kse)->k_schedq->sq_runq.pq_threads)
99
100/*
101 * We've got to keep track of everything that is allocated, not only
102 * to have a speedy free list, but also so they can be deallocated
103 * after a fork().
104 */
105static TAILQ_HEAD(, kse) active_kseq;
106static TAILQ_HEAD(, kse) free_kseq;
107static TAILQ_HEAD(, kse_group) free_kse_groupq;
108static TAILQ_HEAD(, kse_group) active_kse_groupq;
109static TAILQ_HEAD(, kse_group) gc_ksegq;
110static struct lock kse_lock; /* also used for kseg queue */
111static int free_kse_count = 0;
112static int free_kseg_count = 0;
113static TAILQ_HEAD(, pthread) free_threadq;
114static struct lock thread_lock;
115static int free_thread_count = 0;
116static int inited = 0;
117static int active_kse_count = 0;
118static int active_kseg_count = 0;
119
120#ifdef DEBUG_THREAD_KERN
121static void dump_queues(struct kse *curkse);
122#endif
123static void kse_check_completed(struct kse *kse);
124static void kse_check_waitq(struct kse *kse);
125static void kse_check_signals(struct kse *kse);
126static void kse_fini(struct kse *curkse);
127static void kse_reinit(struct kse *kse);
128static void kse_sched_multi(struct kse *curkse);
129#ifdef NOT_YET
130static void kse_sched_single(struct kse *curkse);
131#endif
132static void kse_switchout_thread(struct kse *kse, struct pthread *thread);
133static void kse_wait(struct kse *kse, struct pthread *td_wait);
134static void kse_free_unlocked(struct kse *kse);
135static void kseg_free_unlocked(struct kse_group *kseg);
136static void kseg_init(struct kse_group *kseg);
137static void kseg_reinit(struct kse_group *kseg);
138static void kse_waitq_insert(struct pthread *thread);
139static void kse_wakeup_multi(struct kse *curkse);
140static void kse_wakeup_one(struct pthread *thread);
141static void thr_cleanup(struct kse *kse, struct pthread *curthread);
142static void thr_resume_wrapper(int unused_1, siginfo_t *unused_2,
143 ucontext_t *ucp);
144static void thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
145 struct pthread_sigframe *psf);
146static int thr_timedout(struct pthread *thread, struct timespec *curtime);
147
148/*
149 * This is called after a fork().
150 * No locks need to be taken here since we are guaranteed to be
151 * single threaded.
152 */
153void
154_kse_single_thread(struct pthread *curthread)
155{
156 struct kse *kse, *kse_next;
157 struct kse_group *kseg, *kseg_next;
158 struct pthread *thread, *thread_next;
159 kse_critical_t crit;
160 int i;
161
162 /*
163 * Disable upcalls and clear the threaded flag.
164 * XXX - I don't think we need to disable upcalls after a fork().
165 * but it doesn't hurt.
166 */
167 crit = _kse_critical_enter();
168 __isthreaded = 0;
169
170 /*
171 * Enter a loop to remove and free all threads other than
172 * the running thread from the active thread list:
173 */
174 for (thread = TAILQ_FIRST(&_thread_list); thread != NULL;
175 thread = thread_next) {
176 /*
177 * Advance to the next thread before the destroying
178 * the current thread.
179 */
180 thread_next = TAILQ_NEXT(thread, tle);
181
182 /*
183 * Remove this thread from the list (the current
184 * thread will be removed but re-added by libpthread
185 * initialization.
186 */
187 TAILQ_REMOVE(&_thread_list, thread, tle);
188 /* Make sure this isn't the running thread: */
189 if (thread != curthread) {
190 _thr_stack_free(&thread->attr);
191 if (thread->specific != NULL)
192 free(thread->specific);
193 for (i = 0; i < MAX_THR_LOCKLEVEL; i++) {
194 _lockuser_destroy(&thread->lockusers[i]);
195 }
196 _lock_destroy(&thread->lock);
197 free(thread);
198 }
199 }
200
201 TAILQ_INIT(&curthread->mutexq); /* initialize mutex queue */
202 curthread->joiner = NULL; /* no joining threads yet */
203 sigemptyset(&curthread->sigpend); /* clear pending signals */
204 if (curthread->specific != NULL) {
205 free(curthread->specific);
206 curthread->specific = NULL;
207 curthread->specific_data_count = 0;
208 }
209
210 /* Free the free KSEs: */
211 while ((kse = TAILQ_FIRST(&free_kseq)) != NULL) {
212 TAILQ_REMOVE(&free_kseq, kse, k_qe);
213 _ksd_destroy(&kse->k_ksd);
214 if (kse->k_stack.ss_sp != NULL)
215 free(kse->k_stack.ss_sp);
216 free(kse);
217 }
218 free_kse_count = 0;
219
220 /* Free the active KSEs: */
221 for (kse = TAILQ_FIRST(&active_kseq); kse != NULL; kse = kse_next) {
222 kse_next = TAILQ_NEXT(kse, k_qe);
223 TAILQ_REMOVE(&active_kseq, kse, k_qe);
224 for (i = 0; i < MAX_KSE_LOCKLEVEL; i++) {
225 _lockuser_destroy(&kse->k_lockusers[i]);
226 }
227 if (kse->k_stack.ss_sp != NULL)
228 free(kse->k_stack.ss_sp);
229 _lock_destroy(&kse->k_lock);
230 free(kse);
231 }
232 active_kse_count = 0;
233
234 /* Free the free KSEGs: */
235 while ((kseg = TAILQ_FIRST(&free_kse_groupq)) != NULL) {
236 TAILQ_REMOVE(&free_kse_groupq, kseg, kg_qe);
237 _lock_destroy(&kseg->kg_lock);
238 _pq_free(&kseg->kg_schedq.sq_runq);
239 free(kseg);
240 }
241 free_kseg_count = 0;
242
243 /* Free the active KSEGs: */
244 for (kseg = TAILQ_FIRST(&active_kse_groupq);
245 kseg != NULL; kseg = kseg_next) {
246 kseg_next = TAILQ_NEXT(kseg, kg_qe);
247 TAILQ_REMOVE(&active_kse_groupq, kseg, kg_qe);
248 _lock_destroy(&kseg->kg_lock);
249 _pq_free(&kseg->kg_schedq.sq_runq);
250 free(kseg);
251 }
252 active_kseg_count = 0;
253
254 /* Free the free threads. */
255 while ((thread = TAILQ_FIRST(&free_threadq)) != NULL) {
256 TAILQ_REMOVE(&free_threadq, thread, tle);
257 if (thread->specific != NULL)
258 free(thread->specific);
259 for (i = 0; i < MAX_THR_LOCKLEVEL; i++) {
260 _lockuser_destroy(&thread->lockusers[i]);
261 }
262 _lock_destroy(&thread->lock);
263 free(thread);
264 }
265 free_thread_count = 0;
266
267 /* Free the to-be-gc'd threads. */
268 while ((thread = TAILQ_FIRST(&_thread_gc_list)) != NULL) {
269 TAILQ_REMOVE(&_thread_gc_list, thread, gcle);
270 for (i = 0; i < MAX_THR_LOCKLEVEL; i++) {
271 _lockuser_destroy(&thread->lockusers[i]);
272 }
273 _lock_destroy(&thread->lock);
274 free(thread);
275 }
276 TAILQ_INIT(&gc_ksegq);
277 _gc_count = 0;
278
279 if (inited != 0) {
280 /*
281 * Destroy these locks; they'll be recreated to assure they
282 * are in the unlocked state.
283 */
284 _lock_destroy(&kse_lock);
285 _lock_destroy(&thread_lock);
286 _lock_destroy(&_thread_list_lock);
287 inited = 0;
288 }
289
290 /*
291 * After a fork(), the leftover thread goes back to being
292 * scope process.
293 */
294 curthread->attr.flags &= ~PTHREAD_SCOPE_SYSTEM;
295 curthread->attr.flags |= PTHREAD_SCOPE_PROCESS;
296
297 /*
298 * After a fork, we are still operating on the thread's original
299 * stack. Don't clear the THR_FLAGS_USER from the thread's
300 * attribute flags.
301 */
302
303 /* Initialize the threads library. */
304 curthread->kse = NULL;
305 curthread->kseg = NULL;
306 _kse_initial = NULL;
307 _libpthread_init(curthread);
308}
309
310/*
311 * This is used to initialize housekeeping and to initialize the
312 * KSD for the KSE.
313 */
314void
315_kse_init(void)
316{
317 if (inited == 0) {
318 TAILQ_INIT(&active_kseq);
319 TAILQ_INIT(&active_kse_groupq);
320 TAILQ_INIT(&free_kseq);
321 TAILQ_INIT(&free_kse_groupq);
322 TAILQ_INIT(&free_threadq);
323 TAILQ_INIT(&gc_ksegq);
324 if (_lock_init(&kse_lock, LCK_ADAPTIVE,
325 _kse_lock_wait, _kse_lock_wakeup) != 0)
326 PANIC("Unable to initialize free KSE queue lock");
327 if (_lock_init(&thread_lock, LCK_ADAPTIVE,
328 _kse_lock_wait, _kse_lock_wakeup) != 0)
329 PANIC("Unable to initialize free thread queue lock");
330 if (_lock_init(&_thread_list_lock, LCK_ADAPTIVE,
331 _kse_lock_wait, _kse_lock_wakeup) != 0)
332 PANIC("Unable to initialize thread list lock");
333 active_kse_count = 0;
334 active_kseg_count = 0;
335 _gc_count = 0;
336 inited = 1;
337 }
338}
339
340int
341_kse_isthreaded(void)
342{
343 return (__isthreaded != 0);
344}
345
346/*
347 * This is called when the first thread (other than the initial
348 * thread) is created.
349 */
350int
351_kse_setthreaded(int threaded)
352{
353 if ((threaded != 0) && (__isthreaded == 0)) {
354 /*
355 * Locking functions in libc are required when there are
356 * threads other than the initial thread.
357 */
358 __isthreaded = 1;
359
360 /*
361 * Tell the kernel to create a KSE for the initial thread
362 * and enable upcalls in it.
363 */
364 _kse_initial->k_flags |= KF_STARTED;
365 if (kse_create(&_kse_initial->k_mbx, 0) != 0) {
366 _kse_initial->k_flags &= ~KF_STARTED;
367 __isthreaded = 0;
368 /* may abort() */
369 DBG_MSG("kse_create failed\n");
370 return (-1);
371 }
372 KSE_SET_MBOX(_kse_initial, _thr_initial);
373 _thr_setmaxconcurrency();
374 }
375 return (0);
376}
377
378/*
379 * Lock wait and wakeup handlers for KSE locks. These are only used by
380 * KSEs, and should never be used by threads. KSE locks include the
381 * KSE group lock (used for locking the scheduling queue) and the
382 * kse_lock defined above.
383 *
384 * When a KSE lock attempt blocks, the entire KSE blocks allowing another
385 * KSE to run. For the most part, it doesn't make much sense to try and
386 * schedule another thread because you need to lock the scheduling queue
387 * in order to do that. And since the KSE lock is used to lock the scheduling
388 * queue, you would just end up blocking again.
389 */
390void
391_kse_lock_wait(struct lock *lock, struct lockuser *lu)
392{
393 struct kse *curkse = (struct kse *)_LCK_GET_PRIVATE(lu);
394 struct timespec ts;
395 int saved_flags;
396
397 if (curkse->k_mbx.km_curthread != NULL)
398 PANIC("kse_lock_wait does not disable upcall.\n");
399 /*
400 * Enter a loop to wait until we get the lock.
401 */
402 ts.tv_sec = 0;
403 ts.tv_nsec = 1000000; /* 1 sec */
404 while (_LCK_BUSY(lu)) {
405 /*
406 * Yield the kse and wait to be notified when the lock
407 * is granted.
408 */
409 saved_flags = curkse->k_mbx.km_flags;
410 curkse->k_mbx.km_flags |= KMF_NOUPCALL | KMF_NOCOMPLETED;
411 kse_release(&ts);
412 curkse->k_mbx.km_flags = saved_flags;
413 }
414}
415
416void
417_kse_lock_wakeup(struct lock *lock, struct lockuser *lu)
418{
419 struct kse *curkse;
420 struct kse *kse;
421 struct kse_mailbox *mbx;
422
423 curkse = _get_curkse();
424 kse = (struct kse *)_LCK_GET_PRIVATE(lu);
425
426 if (kse == curkse)
427 PANIC("KSE trying to wake itself up in lock");
428 else {
429 mbx = &kse->k_mbx;
430 _lock_grant(lock, lu);
431 /*
432 * Notify the owning kse that it has the lock.
433 * It is safe to pass invalid address to kse_wakeup
434 * even if the mailbox is not in kernel at all,
435 * and waking up a wrong kse is also harmless.
436 */
437 kse_wakeup(mbx);
438 }
439}
440
441/*
442 * Thread wait and wakeup handlers for thread locks. These are only used
443 * by threads, never by KSEs. Thread locks include the per-thread lock
444 * (defined in its structure), and condition variable and mutex locks.
445 */
446void
447_thr_lock_wait(struct lock *lock, struct lockuser *lu)
448{
449 struct pthread *curthread = (struct pthread *)lu->lu_private;
450
451 do {
452 THR_SCHED_LOCK(curthread, curthread);
453 THR_SET_STATE(curthread, PS_LOCKWAIT);
454 THR_SCHED_UNLOCK(curthread, curthread);
455 _thr_sched_switch(curthread);
456 } while _LCK_BUSY(lu);
457}
458
459void
460_thr_lock_wakeup(struct lock *lock, struct lockuser *lu)
461{
462 struct pthread *thread;
463 struct pthread *curthread;
464
465 curthread = _get_curthread();
466 thread = (struct pthread *)_LCK_GET_PRIVATE(lu);
467
468 THR_SCHED_LOCK(curthread, thread);
469 _lock_grant(lock, lu);
470 _thr_setrunnable_unlocked(thread);
471 THR_SCHED_UNLOCK(curthread, thread);
472}
473
474kse_critical_t
475_kse_critical_enter(void)
476{
477 kse_critical_t crit;
478
479 crit = _ksd_readandclear_tmbx;
480 return (crit);
481}
482
483void
484_kse_critical_leave(kse_critical_t crit)
485{
486 struct pthread *curthread;
487
488 _ksd_set_tmbx(crit);
489 if ((crit != NULL) && ((curthread = _get_curthread()) != NULL))
490 THR_YIELD_CHECK(curthread);
491}
492
493int
494_kse_in_critical(void)
495{
496 return (_ksd_get_tmbx() == NULL);
497}
498
499void
500_thr_critical_enter(struct pthread *thread)
501{
502 thread->critical_count++;
503}
504
505void
506_thr_critical_leave(struct pthread *thread)
507{
508 thread->critical_count--;
509 THR_YIELD_CHECK(thread);
510}
511
512void
513_thr_sched_switch(struct pthread *curthread)
514{
515 struct kse *curkse;
516
517 (void)_kse_critical_enter();
518 curkse = _get_curkse();
519 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
520 _thr_sched_switch_unlocked(curthread);
521}
522
523/*
524 * XXX - We may need to take the scheduling lock before calling
525 * this, or perhaps take the lock within here before
526 * doing anything else.
527 */
528void
529_thr_sched_switch_unlocked(struct pthread *curthread)
530{
531 struct pthread *td;
532 struct pthread_sigframe psf;
533 struct kse *curkse;
534 int ret;
535 volatile int uts_once;
536 volatile int resume_once = 0;
537
538 /* We're in the scheduler, 5 by 5: */
539 curkse = _get_curkse();
540
541 curthread->need_switchout = 1; /* The thread yielded on its own. */
542 curthread->critical_yield = 0; /* No need to yield anymore. */
543 curthread->slice_usec = -1; /* Restart the time slice. */
544
545 /* Thread can unlock the scheduler lock. */
546 curthread->lock_switch = 1;
547
548 /*
549 * The signal frame is allocated off the stack because
550 * a thread can be interrupted by other signals while
551 * it is running down pending signals.
552 */
553 sigemptyset(&psf.psf_sigset);
554 curthread->curframe = &psf;
555
556 /*
557 * Enter the scheduler if any one of the following is true:
558 *
559 * o The current thread is dead; it's stack needs to be
560 * cleaned up and it can't be done while operating on
561 * it.
562 * o There are no runnable threads.
563 * o The next thread to run won't unlock the scheduler
564 * lock. A side note: the current thread may be run
565 * instead of the next thread in the run queue, but
566 * we don't bother checking for that.
567 */
568 if ((curthread->state == PS_DEAD) ||
569 (((td = KSE_RUNQ_FIRST(curkse)) == NULL) &&
570 (curthread->state != PS_RUNNING)) ||
571 ((td != NULL) && (td->lock_switch == 0)))
572 _thread_enter_uts(&curthread->tmbx, &curkse->k_mbx);
573 else {
574 uts_once = 0;
575 THR_GETCONTEXT(&curthread->tmbx.tm_context);
576 if (uts_once == 0) {
577 uts_once = 1;
578
579 /* Switchout the current thread. */
580 kse_switchout_thread(curkse, curthread);
581
582 /* Choose another thread to run. */
583 td = KSE_RUNQ_FIRST(curkse);
584 KSE_RUNQ_REMOVE(curkse, td);
585 curkse->k_curthread = td;
586
587 /*
588 * Make sure the current thread's kse points to
589 * this kse.
590 */
591 td->kse = curkse;
592
593 /*
594 * Reset accounting.
595 */
596 td->tmbx.tm_uticks = 0;
597 td->tmbx.tm_sticks = 0;
598
599 /*
600 * Reset the time slice if this thread is running
601 * for the first time or running again after using
602 * its full time slice allocation.
603 */
604 if (td->slice_usec == -1)
605 td->slice_usec = 0;
606
607 /* Mark the thread active. */
608 td->active = 1;
609
610 /* Remove the frame reference. */
611 td->curframe = NULL;
612
613 /*
614 * Continue the thread at its current frame:
615 */
616 ret = _thread_switch(&td->tmbx, NULL);
617 /* This point should not be reached. */
618 if (ret != 0)
619 PANIC("Bad return from _thread_switch");
620 PANIC("Thread has returned from _thread_switch");
621 }
622 }
623
624 if (curthread->lock_switch != 0) {
625 /*
626 * Unlock the scheduling queue and leave the
627 * critical region.
628 */
629 /* Don't trust this after a switch! */
630 curkse = _get_curkse();
631
632 curthread->lock_switch = 0;
633 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
634 _kse_critical_leave(&curthread->tmbx);
635 }
636 /*
637 * This thread is being resumed; check for cancellations.
638 */
639 if ((resume_once == 0) && (!THR_IN_CRITICAL(curthread))) {
640 resume_once = 1;
641 thr_resume_check(curthread, &curthread->tmbx.tm_context, &psf);
642 }
643
644 THR_ACTIVATE_LAST_LOCK(curthread);
645}
646
647/*
648 * This is the scheduler for a KSE which runs a scope system thread.
649 * The multi-thread KSE scheduler should also work for a single threaded
650 * KSE, but we use a separate scheduler so that it can be fine-tuned
651 * to be more efficient (and perhaps not need a separate stack for
652 * the KSE, allowing it to use the thread's stack).
653 *
654 * XXX - This probably needs some work.
655 */
656#ifdef NOT_YET
657static void
658kse_sched_single(struct kse *curkse)
659{
660 struct pthread *curthread = curkse->k_curthread;
661 struct pthread *td_wait;
662 struct timespec ts;
663 int level;
664
665 if (curthread->active == 0) {
666 if (curthread->state != PS_RUNNING) {
667 /* Check to see if the thread has timed out. */
668 KSE_GET_TOD(curkse, &ts);
669 if (thr_timedout(curthread, &ts) != 0) {
670 curthread->timeout = 1;
671 curthread->state = PS_RUNNING;
672 }
673 }
674 }
675
676 /* This thread no longer needs to yield the CPU: */
677 curthread->critical_yield = 0;
678 curthread->need_switchout = 0;
679
680 /*
681 * Lock the scheduling queue.
682 *
683 * There is no scheduling queue for single threaded KSEs,
684 * but we need a lock for protection regardless.
685 */
686 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
687
688 /*
689 * This has to do the job of kse_switchout_thread(), only
690 * for a single threaded KSE/KSEG.
691 */
692
693 switch (curthread->state) {
694 case PS_DEAD:
695 /* Unlock the scheduling queue and exit the KSE. */
696 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
697 kse_fini(curkse); /* does not return */
698 break;
699
700 case PS_COND_WAIT:
701 case PS_SLEEP_WAIT:
702 /* Only insert threads that can timeout: */
703 if (curthread->wakeup_time.tv_sec != -1) {
704 /* Insert into the waiting queue: */
705 KSE_WAITQ_INSERT(curkse, curthread);
706 }
707 break;
708
709 case PS_LOCKWAIT:
710 level = curthread->locklevel - 1;
711 if (_LCK_BUSY(&curthread->lockusers[level]))
712 KSE_WAITQ_INSERT(curkse, curthread);
713 else
714 THR_SET_STATE(curthread, PS_RUNNING);
715 break;
716
717 case PS_JOIN:
718 case PS_MUTEX_WAIT:
719 case PS_RUNNING:
720 case PS_SIGSUSPEND:
721 case PS_SIGWAIT:
722 case PS_SUSPENDED:
723 case PS_DEADLOCK:
724 default:
725 /*
726 * These states don't timeout and don't need
727 * to be in the waiting queue.
728 */
729 break;
730 }
731 while (curthread->state != PS_RUNNING) {
732 curthread->active = 0;
733 td_wait = KSE_WAITQ_FIRST(curkse);
734
735 kse_wait(curkse, td_wait);
736
737 if (td_wait != NULL) {
738 KSE_GET_TOD(curkse, &ts);
739 if (thr_timedout(curthread, &ts)) {
740 /* Indicate the thread timedout: */
741 td_wait->timeout = 1;
742
743 /* Make the thread runnable. */
744 THR_SET_STATE(td_wait, PS_RUNNING);
745 KSE_WAITQ_REMOVE(curkse, td_wait);
746 }
747 }
748 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
749 kse_check_signals(curkse);
750 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
751 }
752
753 /* Remove the frame reference. */
754 curthread->curframe = NULL;
755
756 /* Unlock the scheduling queue. */
757 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
758
759 /*
760 * Continue the thread at its current frame:
761 */
762 DBG_MSG("Continuing bound thread %p\n", curthread);
763 _thread_switch(&curthread->tmbx, &curkse->k_mbx.km_curthread);
764 PANIC("Thread has returned from _thread_switch");
765}
766#endif
767
768#ifdef DEBUG_THREAD_KERN
769static void
770dump_queues(struct kse *curkse)
771{
772 struct pthread *thread;
773
774 DBG_MSG("Threads in waiting queue:\n");
775 TAILQ_FOREACH(thread, &curkse->k_kseg->kg_schedq.sq_waitq, pqe) {
776 DBG_MSG(" thread %p, state %d, blocked %d\n",
777 thread, thread->state, thread->blocked);
778 }
779}
780#endif
781
782/*
783 * This is the scheduler for a KSE which runs multiple threads.
784 */
785static void
786kse_sched_multi(struct kse *curkse)
787{
788 struct pthread *curthread, *td_wait;
789 struct pthread_sigframe *curframe;
790 int ret;
791
792 THR_ASSERT(curkse->k_mbx.km_curthread == NULL,
793 "Mailbox not null in kse_sched_multi");
794
795 /* Check for first time initialization: */
796 if ((curkse->k_flags & KF_INITIALIZED) == 0) {
797 /* Setup this KSEs specific data. */
798 _ksd_setprivate(&curkse->k_ksd);
799 _set_curkse(curkse);
800
801 /* Set this before grabbing the context. */
802 curkse->k_flags |= KF_INITIALIZED;
803 }
804
805 /* This may have returned from a kse_release(). */
806 if (KSE_WAITING(curkse)) {
807 DBG_MSG("Entered upcall when KSE is waiting.");
808 KSE_CLEAR_WAIT(curkse);
809 }
810
811 /* Lock the scheduling lock. */
812 curthread = curkse->k_curthread;
813 if ((curthread == NULL) || (curthread->need_switchout == 0)) {
814 /* This is an upcall; take the scheduler lock. */
815 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
816 }
817
818 /*
819 * If the current thread was completed in another KSE, then
820 * it will be in the run queue. Don't mark it as being blocked.
821 */
822 if ((curthread != NULL) &&
823 ((curthread->flags & THR_FLAGS_IN_RUNQ) == 0) &&
824 (curthread->need_switchout == 0)) {
825 /*
826 * Assume the current thread is blocked; when the
827 * completed threads are checked and if the current
828 * thread is among the completed, the blocked flag
829 * will be cleared.
830 */
831 curthread->blocked = 1;
832 }
833
834 /* Check for any unblocked threads in the kernel. */
835 kse_check_completed(curkse);
836
837 /*
838 * Check for threads that have timed-out.
839 */
840 kse_check_waitq(curkse);
841
842 /*
843 * Switchout the current thread, if necessary, as the last step
844 * so that it is inserted into the run queue (if it's runnable)
845 * _after_ any other threads that were added to it above.
846 */
847 if (curthread == NULL)
848 ; /* Nothing to do here. */
849 else if ((curthread->need_switchout == 0) &&
850 (curthread->blocked == 0) && (THR_IN_CRITICAL(curthread))) {
851 /*
852 * Resume the thread and tell it to yield when
853 * it leaves the critical region.
854 */
855 curthread->critical_yield = 1;
856 curthread->active = 1;
857 if ((curthread->flags & THR_FLAGS_IN_RUNQ) != 0)
858 KSE_RUNQ_REMOVE(curkse, curthread);
859 curkse->k_curthread = curthread;
860 curthread->kse = curkse;
861 DBG_MSG("Continuing thread %p in critical region\n",
862 curthread);
863 kse_wakeup_multi(curkse);
864 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
865 ret = _thread_switch(&curthread->tmbx,
866 &curkse->k_mbx.km_curthread);
867 if (ret != 0)
868 PANIC("Can't resume thread in critical region\n");
869 }
870 else if ((curthread->flags & THR_FLAGS_IN_RUNQ) == 0)
871 kse_switchout_thread(curkse, curthread);
872 curkse->k_curthread = NULL;
873
874 kse_wakeup_multi(curkse);
875
876 /* This has to be done without the scheduling lock held. */
877 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
878 kse_check_signals(curkse);
879 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
880
881#ifdef DEBUG_THREAD_KERN
882 dump_queues(curkse);
883#endif
884
885 /* Check if there are no threads ready to run: */
886 while (((curthread = KSE_RUNQ_FIRST(curkse)) == NULL) &&
887 (curkse->k_kseg->kg_threadcount != 0)) {
888 /*
889 * Wait for a thread to become active or until there are
890 * no more threads.
891 */
892 td_wait = KSE_WAITQ_FIRST(curkse);
893 kse_wait(curkse, td_wait);
894 kse_check_completed(curkse);
895 kse_check_waitq(curkse);
896 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
897 kse_check_signals(curkse);
898 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
899 }
900
901 /* Check for no more threads: */
902 if (curkse->k_kseg->kg_threadcount == 0) {
903 /*
904 * Normally this shouldn't return, but it will if there
905 * are other KSEs running that create new threads that
906 * are assigned to this KSE[G]. For instance, if a scope
907 * system thread were to create a scope process thread
908 * and this kse[g] is the initial kse[g], then that newly
909 * created thread would be assigned to us (the initial
910 * kse[g]).
911 */
912 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
913 kse_fini(curkse);
914 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
915 curthread = KSE_RUNQ_FIRST(curkse);
916 }
917
918 THR_ASSERT(curthread != NULL,
919 "Return from kse_wait/fini without thread.");
920 THR_ASSERT(curthread->state != PS_DEAD,
921 "Trying to resume dead thread!");
922 KSE_RUNQ_REMOVE(curkse, curthread);
923
924 /*
925 * Make the selected thread the current thread.
926 */
927 curkse->k_curthread = curthread;
928
929 /*
930 * Make sure the current thread's kse points to this kse.
931 */
932 curthread->kse = curkse;
933
934 /*
935 * Reset accounting.
936 */
937 curthread->tmbx.tm_uticks = 0;
938 curthread->tmbx.tm_sticks = 0;
939
940 /*
941 * Reset the time slice if this thread is running for the first
942 * time or running again after using its full time slice allocation.
943 */
944 if (curthread->slice_usec == -1)
945 curthread->slice_usec = 0;
946
947 /* Mark the thread active. */
948 curthread->active = 1;
949
950 /* Remove the frame reference. */
951 curframe = curthread->curframe;
952 curthread->curframe = NULL;
953
954 kse_wakeup_multi(curkse);
955
956 /*
957 * The thread's current signal frame will only be NULL if it
958 * is being resumed after being blocked in the kernel. In
959 * this case, and if the thread needs to run down pending
960 * signals or needs a cancellation check, we need to add a
961 * signal frame to the thread's context.
962 */
963#ifdef NOT_YET
964 if ((curframe == NULL) && ((curthread->have_signals != 0) ||
965 (((curthread->cancelflags & THR_AT_CANCEL_POINT) == 0) &&
966 ((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0))))
967 signalcontext(&curthread->tmbx.tm_context, 0,
968 (__sighandler_t *)thr_resume_wrapper);
969#else
970 if ((curframe == NULL) && (curthread->have_signals != 0))
971 signalcontext(&curthread->tmbx.tm_context, 0,
972 (__sighandler_t *)thr_resume_wrapper);
973#endif
974 /*
975 * Continue the thread at its current frame:
976 */
977 if (curthread->lock_switch != 0) {
978 /*
979 * This thread came from a scheduler switch; it will
980 * unlock the scheduler lock and set the mailbox.
981 */
982 ret = _thread_switch(&curthread->tmbx, NULL);
983 } else {
984 /* This thread won't unlock the scheduler lock. */
985 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
986 ret = _thread_switch(&curthread->tmbx,
987 &curkse->k_mbx.km_curthread);
988 }
989 if (ret != 0)
990 PANIC("Thread has returned from _thread_switch");
991
992 /* This point should not be reached. */
993 PANIC("Thread has returned from _thread_switch");
994}
995
996static void
997kse_check_signals(struct kse *curkse)
998{
999 sigset_t sigset;
1000 int i;
1001
1002 /* Deliver posted signals. */
1003 for (i = 0; i < _SIG_WORDS; i++) {
1004 atomic_swap_int(&curkse->k_mbx.km_sigscaught.__bits[i],
1005 0, &sigset.__bits[i]);
1006 }
1007 if (SIGNOTEMPTY(sigset)) {
1008 /*
1009 * Dispatch each signal.
1010 *
1011 * XXX - There is no siginfo for any of these.
1012 * I think there should be, especially for
1013 * signals from other processes (si_pid, si_uid).
1014 */
1015 for (i = 1; i < NSIG; i++) {
1016 if (sigismember(&sigset, i) != 0) {
1017 DBG_MSG("Dispatching signal %d\n", i);
1018 _thr_sig_dispatch(curkse, i,
1019 NULL /* no siginfo */);
1020 }
1021 }
1022 sigemptyset(&sigset);
1023 __sys_sigprocmask(SIG_SETMASK, &sigset, NULL);
1024 }
1025}
1026
1027static void
1028thr_resume_wrapper(int unused_1, siginfo_t *unused_2, ucontext_t *ucp)
1029{
1030 struct pthread *curthread = _get_curthread();
1031
1032 thr_resume_check(curthread, ucp, NULL);
1033}
1034
1035static void
1036thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
1037 struct pthread_sigframe *psf)
1038{
1039 /* Check signals before cancellations. */
1040 while (curthread->have_signals != 0) {
1041 /* Clear the pending flag. */
1042 curthread->have_signals = 0;
1043
1044 /*
1045 * It's perfectly valid, though not portable, for
1046 * signal handlers to munge their interrupted context
1047 * and expect to return to it. Ensure we use the
1048 * correct context when running down signals.
1049 */
1050 _thr_sig_rundown(curthread, ucp, psf);
1051 }
1052
1053#ifdef NOT_YET
1054 if (((curthread->cancelflags & THR_AT_CANCEL_POINT) == 0) &&
1055 ((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0))
1056 pthread_testcancel();
1057#endif
1058}
1059
1060/*
1061 * Clean up a thread. This must be called with the thread's KSE
1062 * scheduling lock held. The thread must be a thread from the
1063 * KSE's group.
1064 */
1065static void
1066thr_cleanup(struct kse *curkse, struct pthread *thread)
1067{
1068 struct pthread *joiner;
1069
1070 if ((joiner = thread->joiner) != NULL) {
1071 thread->joiner = NULL;
1072 if ((joiner->state == PS_JOIN) &&
1073 (joiner->join_status.thread == thread)) {
1074 joiner->join_status.thread = NULL;
1075
1076 /* Set the return status for the joining thread: */
1077 joiner->join_status.ret = thread->ret;
1078
1079 /* Make the thread runnable. */
1080 if (joiner->kseg == curkse->k_kseg)
1081 _thr_setrunnable_unlocked(joiner);
1082 else {
1083 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
1084 KSE_SCHED_LOCK(curkse, joiner->kseg);
1085 _thr_setrunnable_unlocked(joiner);
1086 KSE_SCHED_UNLOCK(curkse, joiner->kseg);
1087 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
1088 }
1089 }
1090 thread->attr.flags |= PTHREAD_DETACHED;
1091 }
1092
1093 if ((thread->attr.flags & PTHREAD_SCOPE_PROCESS) == 0) {
1094 /*
1095 * Remove the thread from the KSEG's list of threads.
1096 */
1097 KSEG_THRQ_REMOVE(thread->kseg, thread);
1098 /*
1099 * Migrate the thread to the main KSE so that this
1100 * KSE and KSEG can be cleaned when their last thread
1101 * exits.
1102 */
1103 thread->kseg = _kse_initial->k_kseg;
1104 thread->kse = _kse_initial;
1105 }
1106 thread->flags |= THR_FLAGS_GC_SAFE;
1107
1108 /*
1109 * We can't hold the thread list lock while holding the
1110 * scheduler lock.
1111 */
1112 KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
1113 DBG_MSG("Adding thread %p to GC list\n", thread);
1114 KSE_LOCK_ACQUIRE(curkse, &_thread_list_lock);
1115 THR_GCLIST_ADD(thread);
1116 KSE_LOCK_RELEASE(curkse, &_thread_list_lock);
1117 KSE_SCHED_LOCK(curkse, curkse->k_kseg);
1118}
1119
1120void
1121_thr_gc(struct pthread *curthread)
1122{
1123 struct pthread *td, *td_next;
1124 kse_critical_t crit;
1125 TAILQ_HEAD(, pthread) worklist;
1126
1127 TAILQ_INIT(&worklist);
1128 crit = _kse_critical_enter();
1129 KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
1130
1131 /* Check the threads waiting for GC. */
1132 for (td = TAILQ_FIRST(&_thread_gc_list); td != NULL; td = td_next) {
1133 td_next = TAILQ_NEXT(td, gcle);
1134 if ((td->flags & THR_FLAGS_GC_SAFE) == 0)
1135 continue;
1136#ifdef NOT_YET
1137 else if (((td->attr.flags & PTHREAD_SCOPE_PROCESS) != 0) &&
1138 (td->kse->k_mbx.km_flags == 0)) {
1139 /*
1140 * The thread and KSE are operating on the same
1141 * stack. Wait for the KSE to exit before freeing
1142 * the thread's stack as well as everything else.
1143 */
1144 continue;
1145 }
1146#endif
1147 /*
1148 * Remove the thread from the GC list. If the thread
1149 * isn't yet detached, it will get added back to the
1150 * GC list at a later time.
1151 */
1152 THR_GCLIST_REMOVE(td);
1153 DBG_MSG("Freeing thread %p stack\n", td);
1154 /*
1155 * We can free the thread stack since it's no longer
1156 * in use.
1157 */
1158 _thr_stack_free(&td->attr);
1159 if (((td->attr.flags & PTHREAD_DETACHED) != 0) &&
1160 (td->refcount == 0)) {
1161 /*
1162 * The thread has detached and is no longer
1163 * referenced. It is safe to remove all
1164 * remnants of the thread.
1165 */
1166 THR_LIST_REMOVE(td);
1167 TAILQ_INSERT_HEAD(&worklist, td, gcle);
1168 }
1169 }
1170 KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
1171 _kse_critical_leave(crit);
1172
1173 while ((td = TAILQ_FIRST(&worklist)) != NULL) {
1174 TAILQ_REMOVE(&worklist, td, gcle);
1175
1176 if ((td->attr.flags & PTHREAD_SCOPE_PROCESS) != 0) {
1177 crit = _kse_critical_enter();
1178 KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock);
1179 kse_free_unlocked(td->kse);
1180 kseg_free_unlocked(td->kseg);
1181 KSE_LOCK_RELEASE(curthread->kse, &kse_lock);
1182 _kse_critical_leave(crit);
1183 }
1184 DBG_MSG("Freeing thread %p\n", td);
1185 _thr_free(curthread, td);
1186 }
1187}
1188
1189
1190/*
1191 * Only new threads that are running or suspended may be scheduled.
1192 */
1193int
1194_thr_schedule_add(struct pthread *curthread, struct pthread *newthread)
1195{
1196 struct kse *curkse;
1197 kse_critical_t crit;
1198 int need_start;
1199 int ret;
1200
1201 /*
1202 * If this is the first time creating a thread, make sure
1203 * the mailbox is set for the current thread.
1204 */
1205 if ((newthread->attr.flags & PTHREAD_SCOPE_SYSTEM) != 0) {
1206#ifdef NOT_YET
1207 /* We use the thread's stack as the KSE's stack. */
1208 new_thread->kse->k_mbx.km_stack.ss_sp =
1209 new_thread->attr.stackaddr_attr;
1210 new_thread->kse->k_mbx.km_stack.ss_size =
1211 new_thread->attr.stacksize_attr;
1212#endif
1213 /*
1214 * No need to lock the scheduling queue since the
1215 * KSE/KSEG pair have not yet been started.
1216 */
1217 KSEG_THRQ_ADD(newthread->kseg, newthread);
1218 TAILQ_INSERT_TAIL(&newthread->kseg->kg_kseq, newthread->kse,
1219 k_kgqe);
1220 newthread->kseg->kg_ksecount = 1;
1221 if (newthread->state == PS_RUNNING)
1222 THR_RUNQ_INSERT_TAIL(newthread);
1223 newthread->kse->k_curthread = NULL;
1224 newthread->kse->k_mbx.km_flags = 0;
1225 newthread->kse->k_mbx.km_func = (kse_func_t *)kse_sched_multi;
1226 newthread->kse->k_mbx.km_quantum = 0;
1227
1228 /*
1229 * This thread needs a new KSE and KSEG.
1230 */
1231 crit = _kse_critical_enter();
1232 curkse = _get_curkse();
1233 _ksd_setprivate(&newthread->kse->k_ksd);
1234 newthread->kse->k_flags |= KF_INITIALIZED;
1235 ret = kse_create(&newthread->kse->k_mbx, 1);
1236 if (ret != 0)
1237 ret = errno;
1238 _ksd_setprivate(&curkse->k_ksd);
1239 _kse_critical_leave(crit);
1240 }
1241 else {
1242 /*
1243 * Lock the KSE and add the new thread to its list of
1244 * assigned threads. If the new thread is runnable, also
1245 * add it to the KSE's run queue.
1246 */
1247 need_start = 0;
1248 KSE_SCHED_LOCK(curthread->kse, newthread->kseg);
1249 KSEG_THRQ_ADD(newthread->kseg, newthread);
1250 if (newthread->state == PS_RUNNING)
1251 THR_RUNQ_INSERT_TAIL(newthread);
1252 if ((newthread->kse->k_flags & KF_STARTED) == 0) {
1253 /*
1254 * This KSE hasn't been started yet. Start it
1255 * outside of holding the lock.
1256 */
1257 newthread->kse->k_flags |= KF_STARTED;
1258 newthread->kse->k_mbx.km_func =
1259 (kse_func_t *)kse_sched_multi;
1260 newthread->kse->k_mbx.km_flags = 0;
1261 need_start = 1;
1262 }
1263 KSE_SCHED_UNLOCK(curthread->kse, newthread->kseg);
1264
1265 if (need_start != 0)
1266 kse_create(&newthread->kse->k_mbx, 0);
1267 else if ((newthread->state == PS_RUNNING) &&
1268 KSE_IS_IDLE(newthread->kse)) {
1269 /*
1270 * The thread is being scheduled on another KSEG.
1271 */
1272 kse_wakeup_one(newthread);
1273 }
1274 ret = 0;
1275 }
1276 return (ret);
1277}
1278
1279void
1280kse_waitq_insert(struct pthread *thread)
1281{
1282 struct pthread *td;
1283
1284 if (thread->wakeup_time.tv_sec == -1)
1285 TAILQ_INSERT_TAIL(&thread->kse->k_schedq->sq_waitq, thread,
1286 pqe);
1287 else {
1288 td = TAILQ_FIRST(&thread->kse->k_schedq->sq_waitq);
1289 while ((td != NULL) && (td->wakeup_time.tv_sec != -1) &&
1290 ((td->wakeup_time.tv_sec < thread->wakeup_time.tv_sec) ||
1291 ((td->wakeup_time.tv_sec == thread->wakeup_time.tv_sec) &&
1292 (td->wakeup_time.tv_nsec <= thread->wakeup_time.tv_nsec))))
1293 td = TAILQ_NEXT(td, pqe);
1294 if (td == NULL)
1295 TAILQ_INSERT_TAIL(&thread->kse->k_schedq->sq_waitq,
1296 thread, pqe);
1297 else
1298 TAILQ_INSERT_BEFORE(td, thread, pqe);
1299 }
1300 thread->flags |= THR_FLAGS_IN_WAITQ;
1301}
1302
1303/*
1304 * This must be called with the scheduling lock held.
1305 */
1306static void
1307kse_check_completed(struct kse *kse)
1308{
1309 struct pthread *thread;
1310 struct kse_thr_mailbox *completed;
1311
1312 if ((completed = kse->k_mbx.km_completed) != NULL) {
1313 kse->k_mbx.km_completed = NULL;
1314 while (completed != NULL) {
1315 thread = completed->tm_udata;
1316 DBG_MSG("Found completed thread %p, name %s\n",
1317 thread,
1318 (thread->name == NULL) ? "none" : thread->name);
1319 thread->blocked = 0;
1320 if (thread != kse->k_curthread) {
1321 if ((thread->flags & THR_FLAGS_SUSPENDED) != 0)
1322 THR_SET_STATE(thread, PS_SUSPENDED);
1323 else
1324 KSE_RUNQ_INSERT_TAIL(kse, thread);
1325 if ((thread->kse != kse) &&
1326 (thread->kse->k_curthread == thread)) {
1327 thread->kse->k_curthread = NULL;
1328 thread->active = 0;
1329 }
1330 }
1331 completed = completed->tm_next;
1332 }
1333 }
1334}
1335
1336/*
1337 * This must be called with the scheduling lock held.
1338 */
1339static void
1340kse_check_waitq(struct kse *kse)
1341{
1342 struct pthread *pthread;
1343 struct timespec ts;
1344
1345 KSE_GET_TOD(kse, &ts);
1346
1347 /*
1348 * Wake up threads that have timedout. This has to be
1349 * done before adding the current thread to the run queue
1350 * so that a CPU intensive thread doesn't get preference
1351 * over waiting threads.
1352 */
1353 while (((pthread = KSE_WAITQ_FIRST(kse)) != NULL) &&
1354 thr_timedout(pthread, &ts)) {
1355 /* Remove the thread from the wait queue: */
1356 KSE_WAITQ_REMOVE(kse, pthread);
1357 DBG_MSG("Found timedout thread %p in waitq\n", pthread);
1358
1359 /* Indicate the thread timedout: */
1360 pthread->timeout = 1;
1361
1362 /* Add the thread to the priority queue: */
1363 if ((pthread->flags & THR_FLAGS_SUSPENDED) != 0)
1364 THR_SET_STATE(pthread, PS_SUSPENDED);
1365 else {
1366 THR_SET_STATE(pthread, PS_RUNNING);
1367 KSE_RUNQ_INSERT_TAIL(kse, pthread);
1368 }
1369 }
1370}
1371
1372static int
1373thr_timedout(struct pthread *thread, struct timespec *curtime)
1374{
1375 if (thread->wakeup_time.tv_sec < 0)
1376 return (0);
1377 else if (thread->wakeup_time.tv_sec > curtime->tv_sec)
1378 return (0);
1379 else if ((thread->wakeup_time.tv_sec == curtime->tv_sec) &&
1380 (thread->wakeup_time.tv_nsec > curtime->tv_nsec))
1381 return (0);
1382 else
1383 return (1);
1384}
1385
1386/*
1387 * This must be called with the scheduling lock held.
1388 *
1389 * Each thread has a time slice, a wakeup time (used when it wants
1390 * to wait for a specified amount of time), a run state, and an
1391 * active flag.
1392 *
1393 * When a thread gets run by the scheduler, the active flag is
1394 * set to non-zero (1). When a thread performs an explicit yield
1395 * or schedules a state change, it enters the scheduler and the
1396 * active flag is cleared. When the active flag is still seen
1397 * set in the scheduler, that means that the thread is blocked in
1398 * the kernel (because it is cleared before entering the scheduler
1399 * in all other instances).
1400 *
1401 * The wakeup time is only set for those states that can timeout.
1402 * It is set to (-1, -1) for all other instances.
1403 *
1404 * The thread's run state, aside from being useful when debugging,
1405 * is used to place the thread in an appropriate queue. There
1406 * are 2 basic queues:
1407 *
1408 * o run queue - queue ordered by priority for all threads
1409 * that are runnable
1410 * o waiting queue - queue sorted by wakeup time for all threads
1411 * that are not otherwise runnable (not blocked
1412 * in kernel, not waiting for locks)
1413 *
1414 * The thread's time slice is used for round-robin scheduling
1415 * (the default scheduling policy). While a SCHED_RR thread
1416 * is runnable it's time slice accumulates. When it reaches
1417 * the time slice interval, it gets reset and added to the end
1418 * of the queue of threads at its priority. When a thread no
1419 * longer becomes runnable (blocks in kernel, waits, etc), its
1420 * time slice is reset.
1421 *
1422 * The job of kse_switchout_thread() is to handle all of the above.
1423 */
1424static void
1425kse_switchout_thread(struct kse *kse, struct pthread *thread)
1426{
1427 int level;
1428 int i;
1429
1430 /*
1431 * Place the currently running thread into the
1432 * appropriate queue(s).
1433 */
1434 DBG_MSG("Switching out thread %p, state %d\n", thread, thread->state);
1435
1436 THR_DEACTIVATE_LAST_LOCK(thread);
1437 if (thread->blocked != 0) {
1438 thread->active = 0;
1439 thread->need_switchout = 0;
1440 /* This thread must have blocked in the kernel. */
1441 /* thread->slice_usec = -1;*/ /* restart timeslice */
1442 /*
1443 * XXX - Check for pending signals for this thread to
1444 * see if we need to interrupt it in the kernel.
1445 */
1446 /* if (thread->check_pending != 0) */
1447 if ((thread->slice_usec != -1) &&
1448 (thread->attr.sched_policy != SCHED_FIFO))
1449 thread->slice_usec += (thread->tmbx.tm_uticks
1450 + thread->tmbx.tm_sticks) * _clock_res_usec;
1451 }
1452 else {
1453 switch (thread->state) {
1454 case PS_DEAD:
1455 /*
1456 * The scheduler is operating on a different
1457 * stack. It is safe to do garbage collecting
1458 * here.
1459 */
1460 thread->active = 0;
1461 thread->need_switchout = 0;
1462 thr_cleanup(kse, thread);
1463 return;
1464 break;
1465
1466 case PS_RUNNING:
1467 if ((thread->flags & THR_FLAGS_SUSPENDED) != 0)
1468 THR_SET_STATE(thread, PS_SUSPENDED);
1469 break;
1470
1471 case PS_COND_WAIT:
1472 case PS_SLEEP_WAIT:
1473 /* Insert into the waiting queue: */
1474 KSE_WAITQ_INSERT(kse, thread);
1475 break;
1476
1477 case PS_LOCKWAIT:
1478 /*
1479 * This state doesn't timeout.
1480 */
1481 thread->wakeup_time.tv_sec = -1;
1482 thread->wakeup_time.tv_nsec = -1;
1483 level = thread->locklevel - 1;
1484 if (_LCK_BUSY(&thread->lockusers[level]))
1485 KSE_WAITQ_INSERT(kse, thread);
1486 else
1487 THR_SET_STATE(thread, PS_RUNNING);
1488 break;
1489
1490 case PS_JOIN:
1491 case PS_MUTEX_WAIT:
1492 case PS_SIGSUSPEND:
1493 case PS_SIGWAIT:
1494 case PS_SUSPENDED:
1495 case PS_DEADLOCK:
1496 default:
1497 /*
1498 * These states don't timeout.
1499 */
1500 thread->wakeup_time.tv_sec = -1;
1501 thread->wakeup_time.tv_nsec = -1;
1502
1503 /* Insert into the waiting queue: */
1504 KSE_WAITQ_INSERT(kse, thread);
1505 break;
1506 }
1507 if (thread->state != PS_RUNNING) {
1508 /* Restart the time slice: */
1509 thread->slice_usec = -1;
1510 } else {
1511 if (thread->need_switchout != 0)
1512 /*
1513 * The thread yielded on its own;
1514 * restart the timeslice.
1515 */
1516 thread->slice_usec = -1;
1517 else if ((thread->slice_usec != -1) &&
1518 (thread->attr.sched_policy != SCHED_FIFO)) {
1519 thread->slice_usec += (thread->tmbx.tm_uticks
1520 + thread->tmbx.tm_sticks) * _clock_res_usec;
1521 /* Check for time quantum exceeded: */
1522 if (thread->slice_usec > TIMESLICE_USEC)
1523 thread->slice_usec = -1;
1524 }
1525 if (thread->slice_usec == -1) {
1526 /*
1527 * The thread exceeded its time quantum or
1528 * it yielded the CPU; place it at the tail
1529 * of the queue for its priority.
1530 */
1531 KSE_RUNQ_INSERT_TAIL(kse, thread);
1532 } else {
1533 /*
1534 * The thread hasn't exceeded its interval
1535 * Place it at the head of the queue for its
1536 * priority.
1537 */
1538 KSE_RUNQ_INSERT_HEAD(kse, thread);
1539 }
1540 }
1541 }
1542 thread->active = 0;
1543 thread->need_switchout = 0;
1544 if (thread->check_pending != 0) {
1545 /* Install pending signals into the frame. */
1546 thread->check_pending = 0;
1547 for (i = 0; i < _SIG_MAXSIG; i++) {
1548 if (sigismember(&thread->sigpend, i) &&
1549 !sigismember(&thread->tmbx.tm_context.uc_sigmask, i))
1550 _thr_sig_add(thread, i, &thread->siginfo[i]);
1551 }
1552 }
1553}
1554
1555/*
1556 * This function waits for the smallest timeout value of any waiting
1557 * thread, or until it receives a message from another KSE.
1558 *
1559 * This must be called with the scheduling lock held.
1560 */
1561static void
1562kse_wait(struct kse *kse, struct pthread *td_wait)
1563{
1564 struct timespec ts, ts_sleep;
1565 int saved_flags;
1566
1567 KSE_GET_TOD(kse, &ts);
1568
1569 if ((td_wait == NULL) || (td_wait->wakeup_time.tv_sec < 0)) {
1570 /* Limit sleep to no more than 1 minute. */
1571 ts_sleep.tv_sec = 60;
1572 ts_sleep.tv_nsec = 0;
1573 } else {
1574 TIMESPEC_SUB(&ts_sleep, &td_wait->wakeup_time, &ts);
1575 if (ts_sleep.tv_sec > 60) {
1576 ts_sleep.tv_sec = 60;
1577 ts_sleep.tv_nsec = 0;
1578 }
1579 }
1580 /* Don't sleep for negative times. */
1581 if ((ts_sleep.tv_sec >= 0) && (ts_sleep.tv_nsec >= 0)) {
1582 KSE_SET_IDLE(kse);
1583 kse->k_kseg->kg_idle_kses++;
1584 KSE_SCHED_UNLOCK(kse, kse->k_kseg);
1585 saved_flags = kse->k_mbx.km_flags;
1586 kse->k_mbx.km_flags |= KMF_NOUPCALL;
1587 kse_release(&ts_sleep);
1588 kse->k_mbx.km_flags = saved_flags;
1589 KSE_SCHED_LOCK(kse, kse->k_kseg);
1590 if (KSE_IS_IDLE(kse)) {
1591 KSE_CLEAR_IDLE(kse);
1592 kse->k_kseg->kg_idle_kses--;
1593 }
1594 }
1595}
1596
1597/*
1598 * Avoid calling this kse_exit() so as not to confuse it with the
1599 * system call of the same name.
1600 */
1601static void
1602kse_fini(struct kse *kse)
1603{
1604 struct timespec ts;
1605 struct kse_group *free_kseg = NULL;
1606
1607 if ((kse->k_kseg->kg_flags & KGF_SINGLE_THREAD) != 0)
1608 kse_exit();
1609 /*
1610 * Check to see if this is one of the main kses.
1611 */
1612 else if (kse->k_kseg != _kse_initial->k_kseg) {
1613 /* Remove this KSE from the KSEG's list of KSEs. */
1614 KSE_SCHED_LOCK(kse, kse->k_kseg);
1615 TAILQ_REMOVE(&kse->k_kseg->kg_kseq, kse, k_kgqe);
1616 kse->k_kseg->kg_ksecount--;
1617 if (TAILQ_EMPTY(&kse->k_kseg->kg_kseq))
1618 free_kseg = kse->k_kseg;
1619 KSE_SCHED_UNLOCK(kse, kse->k_kseg);
1620
1621 /*
1622 * Add this KSE to the list of free KSEs along with
1623 * the KSEG if is now orphaned.
1624 */
1625#ifdef NOT_YET
1626 KSE_LOCK_ACQUIRE(kse, &kse_lock);
1627 if (free_kseg != NULL)
1628 kseg_free_unlocked(free_kseg);
1629 kse_free_unlocked(kse);
1630 KSE_LOCK_RELEASE(kse, &kse_lock);
1631#endif
1632 kse_exit();
1633 /* Never returns. */
1634 } else {
1635 /*
1636 * Wait for the last KSE/thread to exit, or for more
1637 * threads to be created (it is possible for additional
1638 * scope process threads to be created after the main
1639 * thread exits).
1640 */
1641 ts.tv_sec = 120;
1642 ts.tv_nsec = 0;
1643 KSE_SET_WAIT(kse);
1644 KSE_SCHED_LOCK(kse, kse->k_kseg);
1645 if ((active_kse_count > 1) &&
1646 (kse->k_kseg->kg_threadcount == 0)) {
1647 KSE_SCHED_UNLOCK(kse, kse->k_kseg);
1648 kse_release(&ts);
1649 /* The above never returns. */
1650 }
1651 else
1652 KSE_SCHED_UNLOCK(kse, kse->k_kseg);
1653
1654 /* There are no more threads; exit this process: */
1655 if (kse->k_kseg->kg_threadcount == 0) {
1656 /* kse_exit(); */
1657 __isthreaded = 0;
1658 exit(0);
1659 }
1660 }
1661}
1662
1663void
1664_thr_set_timeout(const struct timespec *timeout)
1665{
1666 struct pthread *curthread = _get_curthread();
1667 struct timespec ts;
1668
1669 /* Reset the timeout flag for the running thread: */
1670 curthread->timeout = 0;
1671
1672 /* Check if the thread is to wait forever: */
1673 if (timeout == NULL) {
1674 /*
1675 * Set the wakeup time to something that can be recognised as
1676 * different to an actual time of day:
1677 */
1678 curthread->wakeup_time.tv_sec = -1;
1679 curthread->wakeup_time.tv_nsec = -1;
1680 }
1681 /* Check if no waiting is required: */
1682 else if ((timeout->tv_sec == 0) && (timeout->tv_nsec == 0)) {
1683 /* Set the wake up time to 'immediately': */
1684 curthread->wakeup_time.tv_sec = 0;
1685 curthread->wakeup_time.tv_nsec = 0;
1686 } else {
1687 /* Calculate the time for the current thread to wakeup: */
1688 KSE_GET_TOD(curthread->kse, &ts);
1689 TIMESPEC_ADD(&curthread->wakeup_time, &ts, timeout);
1690 }
1691}
1692
1693void
1694_thr_panic_exit(char *file, int line, char *msg)
1695{
1696 char buf[256];
1697
1698 snprintf(buf, sizeof(buf), "(%s:%d) %s\n", file, line, msg);
1699 __sys_write(2, buf, strlen(buf));
1700 abort();
1701}
1702
1703void
1704_thr_setrunnable(struct pthread *curthread, struct pthread *thread)
1705{
1706 kse_critical_t crit;
1707
1708 crit = _kse_critical_enter();
1709 KSE_SCHED_LOCK(curthread->kse, thread->kseg);
1710 _thr_setrunnable_unlocked(thread);
1711 KSE_SCHED_UNLOCK(curthread->kse, thread->kseg);
1712 _kse_critical_leave(crit);
1713}
1714
1715void
1716_thr_setrunnable_unlocked(struct pthread *thread)
1717{
1718 if ((thread->kseg->kg_flags & KGF_SINGLE_THREAD) != 0) {
1719 /* No silly queues for these threads. */
1720 if ((thread->flags & THR_FLAGS_SUSPENDED) != 0)
1721 THR_SET_STATE(thread, PS_SUSPENDED);
1722 else
1723 THR_SET_STATE(thread, PS_RUNNING);
1724 } else if (thread->state != PS_RUNNING) {
1725 if ((thread->flags & THR_FLAGS_IN_WAITQ) != 0)
1726 KSE_WAITQ_REMOVE(thread->kse, thread);
1727 if ((thread->flags & THR_FLAGS_SUSPENDED) != 0)
1728 THR_SET_STATE(thread, PS_SUSPENDED);
1729 else {
1730 THR_SET_STATE(thread, PS_RUNNING);
1731 if ((thread->blocked == 0) && (thread->active == 0) &&
1732 (thread->flags & THR_FLAGS_IN_RUNQ) == 0)
1733 THR_RUNQ_INSERT_TAIL(thread);
1734 }
1735 }
1736 /*
1737 * XXX - Threads are not yet assigned to specific KSEs; they are
1738 * assigned to the KSEG. So the fact that a thread's KSE is
1739 * waiting doesn't necessarily mean that it will be the KSE
1740 * that runs the thread after the lock is granted. But we
1741 * don't know if the other KSEs within the same KSEG are
1742 * also in a waiting state or not so we err on the side of
1743 * caution and wakeup the thread's last known KSE. We
1744 * ensure that the threads KSE doesn't change while it's
1745 * scheduling lock is held so it is safe to reference it
1746 * (the KSE). If the KSE wakes up and doesn't find any more
1747 * work it will again go back to waiting so no harm is done.
1748 */
1749 kse_wakeup_one(thread);
1750}
1751
1752static void
1753kse_wakeup_one(struct pthread *thread)
1754{
1755 struct kse *ke;
1756
1757 if (KSE_IS_IDLE(thread->kse)) {
1758 KSE_CLEAR_IDLE(thread->kse);
1759 thread->kseg->kg_idle_kses--;
1760 KSE_WAKEUP(thread->kse);
1761 } else {
1762 TAILQ_FOREACH(ke, &thread->kseg->kg_kseq, k_kgqe) {
1763 if (KSE_IS_IDLE(ke)) {
1764 KSE_CLEAR_IDLE(ke);
1765 ke->k_kseg->kg_idle_kses--;
1766 KSE_WAKEUP(ke);
1767 return;
1768 }
1769 }
1770 }
1771}
1772
1773static void
1774kse_wakeup_multi(struct kse *curkse)
1775{
1776 struct kse *ke;
1777 int tmp;
1778
1779 if ((tmp = KSE_RUNQ_THREADS(curkse)) && curkse->k_kseg->kg_idle_kses) {
1780 TAILQ_FOREACH(ke, &curkse->k_kseg->kg_kseq, k_kgqe) {
1781 if (KSE_IS_IDLE(ke)) {
1782 KSE_CLEAR_IDLE(ke);
1783 ke->k_kseg->kg_idle_kses--;
1784 KSE_WAKEUP(ke);
1785 if (--tmp == 0)
1786 break;
1787 }
1788 }
1789 }
1790}
1791
1792struct pthread *
1793_get_curthread(void)
1794{
1795 return (_ksd_curthread);
1796}
1797
1798/* This assumes the caller has disabled upcalls. */
1799struct kse *
1800_get_curkse(void)
1801{
1802 return (_ksd_curkse);
1803}
1804
1805void
1806_set_curkse(struct kse *kse)
1807{
1808 _ksd_setprivate(&kse->k_ksd);
1809}
1810
1811/*
1812 * Allocate a new KSEG.
1813 *
1814 * We allow the current thread to be NULL in the case that this
1815 * is the first time a KSEG is being created (library initialization).
1816 * In this case, we don't need to (and can't) take any locks.
1817 */
1818struct kse_group *
1819_kseg_alloc(struct pthread *curthread)
1820{
1821 struct kse_group *kseg = NULL;
1822 kse_critical_t crit;
1823
1824 if ((curthread != NULL) && (free_kseg_count > 0)) {
1825 /* Use the kse lock for the kseg queue. */
1826 crit = _kse_critical_enter();
1827 KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock);
1828 if ((kseg = TAILQ_FIRST(&free_kse_groupq)) != NULL) {
1829 TAILQ_REMOVE(&free_kse_groupq, kseg, kg_qe);
1830 free_kseg_count--;
1831 active_kseg_count++;
1832 TAILQ_INSERT_TAIL(&active_kse_groupq, kseg, kg_qe);
1833 }
1834 KSE_LOCK_RELEASE(curthread->kse, &kse_lock);
1835 _kse_critical_leave(crit);
1836 if (kseg)
1837 kseg_reinit(kseg);
1838 }
1839
1840 /*
1841 * If requested, attempt to allocate a new KSE group only if the
1842 * KSE allocation was successful and a KSE group wasn't found in
1843 * the free list.
1844 */
1845 if ((kseg == NULL) &&
1846 ((kseg = (struct kse_group *)malloc(sizeof(*kseg))) != NULL)) {
1847 if (_pq_alloc(&kseg->kg_schedq.sq_runq,
1848 THR_MIN_PRIORITY, THR_LAST_PRIORITY) != 0) {
1849 free(kseg);
1850 kseg = NULL;
1851 } else {
1852 kseg_init(kseg);
1853 /* Add the KSEG to the list of active KSEGs. */
1854 if (curthread != NULL) {
1855 crit = _kse_critical_enter();
1856 KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock);
1857 active_kseg_count++;
1858 TAILQ_INSERT_TAIL(&active_kse_groupq,
1859 kseg, kg_qe);
1860 KSE_LOCK_RELEASE(curthread->kse, &kse_lock);
1861 _kse_critical_leave(crit);
1862 } else {
1863 active_kseg_count++;
1864 TAILQ_INSERT_TAIL(&active_kse_groupq,
1865 kseg, kg_qe);
1866 }
1867 }
1868 }
1869 return (kseg);
1870}
1871
1872/*
1873 * This must be called with the kse lock held and when there are
1874 * no more threads that reference it.
1875 */
1876static void
1877kseg_free_unlocked(struct kse_group *kseg)
1878{
1879 TAILQ_REMOVE(&active_kse_groupq, kseg, kg_qe);
1880 TAILQ_INSERT_HEAD(&free_kse_groupq, kseg, kg_qe);
1881 free_kseg_count++;
1882 active_kseg_count--;
1883}
1884
1885void
1886_kseg_free(struct kse_group *kseg)
1887{
1888 struct kse *curkse;
1889 kse_critical_t crit;
1890
1891 crit = _kse_critical_enter();
1892 curkse = _get_curkse();
1893 KSE_LOCK_ACQUIRE(curkse, &kse_lock);
1894 kseg_free_unlocked(kseg);
1895 KSE_LOCK_RELEASE(curkse, &kse_lock);
1896 _kse_critical_leave(crit);
1897}
1898
1899/*
1900 * Allocate a new KSE.
1901 *
1902 * We allow the current thread to be NULL in the case that this
1903 * is the first time a KSE is being created (library initialization).
1904 * In this case, we don't need to (and can't) take any locks.
1905 */
1906struct kse *
1907_kse_alloc(struct pthread *curthread)
1908{
1909 struct kse *kse = NULL;
1910 kse_critical_t crit;
1911 int need_ksd = 0;
1912 int i;
1913
1914 if ((curthread != NULL) && (free_kse_count > 0)) {
1915 crit = _kse_critical_enter();
1916 KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock);
1917 /* Search for a finished KSE. */
1918 kse = TAILQ_FIRST(&free_kseq);
1919#ifdef NOT_YET
1920#define KEMBX_DONE 0x04
1921 while ((kse != NULL) &&
1922 ((kse->k_mbx.km_flags & KEMBX_DONE) == 0)) {
1923 kse = TAILQ_NEXT(kse, k_qe);
1924 }
1925#undef KEMBX_DONE
1926#endif
1927 if (kse != NULL) {
1928 TAILQ_REMOVE(&free_kseq, kse, k_qe);
1929 free_kse_count--;
1930 TAILQ_INSERT_TAIL(&active_kseq, kse, k_qe);
1931 active_kse_count++;
1932 }
1933 KSE_LOCK_RELEASE(curthread->kse, &kse_lock);
1934 _kse_critical_leave(crit);
1935 if (kse != NULL)
1936 kse_reinit(kse);
1937 }
1938 if ((kse == NULL) &&
1939 ((kse = (struct kse *)malloc(sizeof(*kse))) != NULL)) {
1940 bzero(kse, sizeof(*kse));
1941
1942 /* Initialize the lockusers. */
1943 for (i = 0; i < MAX_KSE_LOCKLEVEL; i++) {
1944 _lockuser_init(&kse->k_lockusers[i], (void *)kse);
1945 _LCK_SET_PRIVATE2(&kse->k_lockusers[i], NULL);
1946 }
1947 /* _lock_init(kse->k_lock, ...) */
1948
1949 /* We had to malloc a kse; mark it as needing a new ID.*/
1950 need_ksd = 1;
1951
1952 /*
1953 * Create the KSE context.
1954 *
1955 * XXX - For now this is done here in the allocation.
1956 * In the future, we may want to have it done
1957 * outside the allocation so that scope system
1958 * threads (one thread per KSE) are not required
1959 * to have a stack for an unneeded kse upcall.
1960 */
1961 kse->k_mbx.km_func = (kse_func_t *)kse_sched_multi;
1962 kse->k_mbx.km_stack.ss_sp = (char *)malloc(KSE_STACKSIZE);
1963 kse->k_mbx.km_stack.ss_size = KSE_STACKSIZE;
1964 kse->k_mbx.km_udata = (void *)kse;
1965 kse->k_mbx.km_quantum = 20000;
1966 /*
1967 * We need to keep a copy of the stack in case it
1968 * doesn't get used; a KSE running a scope system
1969 * thread will use that thread's stack.
1970 */
1971 kse->k_stack.ss_sp = kse->k_mbx.km_stack.ss_sp;
1972 kse->k_stack.ss_size = kse->k_mbx.km_stack.ss_size;
1973 if (kse->k_mbx.km_stack.ss_sp == NULL) {
1974 for (i = 0; i < MAX_KSE_LOCKLEVEL; i++) {
1975 _lockuser_destroy(&kse->k_lockusers[i]);
1976 }
1977 /* _lock_destroy(&kse->k_lock); */
1978 free(kse);
1979 kse = NULL;
1980 }
1981 }
1982 if ((kse != NULL) && (need_ksd != 0)) {
1983 /* This KSE needs initialization. */
1984 if (curthread != NULL) {
1985 crit = _kse_critical_enter();
1986 KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock);
1987 }
1988 /* Initialize KSD inside of the lock. */
1989 if (_ksd_create(&kse->k_ksd, (void *)kse, sizeof(*kse)) != 0) {
1990 if (curthread != NULL) {
1991 KSE_LOCK_RELEASE(curthread->kse, &kse_lock);
1992 _kse_critical_leave(crit);
1993 }
1994 free(kse->k_mbx.km_stack.ss_sp);
1995 for (i = 0; i < MAX_KSE_LOCKLEVEL; i++) {
1996 _lockuser_destroy(&kse->k_lockusers[i]);
1997 }
1998 free(kse);
1999 return (NULL);
2000 }
2001 kse->k_flags = 0;
2002 TAILQ_INSERT_TAIL(&active_kseq, kse, k_qe);
2003 active_kse_count++;
2004 if (curthread != NULL) {
2005 KSE_LOCK_RELEASE(curthread->kse, &kse_lock);
2006 _kse_critical_leave(crit);
2007 }
2008 }
2009 return (kse);
2010}
2011
2012static void
2013kse_reinit(struct kse *kse)
2014{
2015 bzero(&kse->k_mbx, sizeof(struct kse_mailbox));
2016 kse->k_curthread = 0;
2017 kse->k_kseg = 0;
2018 kse->k_schedq = 0;
2019 kse->k_locklevel = 0;
2020 sigemptyset(&kse->k_sigmask);
2021 bzero(&kse->k_sigq, sizeof(kse->k_sigq));
2022 kse->k_check_sigq = 0;
2023 kse->k_flags = 0;
2024 kse->k_waiting = 0;
2025 kse->k_error = 0;
2026 kse->k_cpu = 0;
2027 kse->k_done = 0;
2028}
2029
2030void
2031kse_free_unlocked(struct kse *kse)
2032{
2033 TAILQ_REMOVE(&active_kseq, kse, k_qe);
2034 active_kse_count--;
2035 kse->k_kseg = NULL;
2036 kse->k_mbx.km_quantum = 20000;
2037 kse->k_flags &= ~KF_INITIALIZED;
2038 TAILQ_INSERT_HEAD(&free_kseq, kse, k_qe);
2039 free_kse_count++;
2040}
2041
2042void
2043_kse_free(struct pthread *curthread, struct kse *kse)
2044{
2045 kse_critical_t crit;
2046
2047 if (curthread == NULL)
2048 kse_free_unlocked(kse);
2049 else {
2050 crit = _kse_critical_enter();
2051 KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock);
2052 kse_free_unlocked(kse);
2053 KSE_LOCK_RELEASE(curthread->kse, &kse_lock);
2054 _kse_critical_leave(crit);
2055 }
2056}
2057
2058static void
2059kseg_init(struct kse_group *kseg)
2060{
2061 kseg_reinit(kseg);
2062 _lock_init(&kseg->kg_lock, LCK_ADAPTIVE, _kse_lock_wait,
2063 _kse_lock_wakeup);
2064}
2065
2066static void
2067kseg_reinit(struct kse_group *kseg)
2068{
2069 TAILQ_INIT(&kseg->kg_kseq);
2070 TAILQ_INIT(&kseg->kg_threadq);
2071 TAILQ_INIT(&kseg->kg_schedq.sq_waitq);
2072 kseg->kg_threadcount = 0;
2073 kseg->kg_ksecount = 0;
2074 kseg->kg_idle_kses = 0;
2075 kseg->kg_flags = 0;
2076}
2077
2078struct pthread *
2079_thr_alloc(struct pthread *curthread)
2080{
2081 kse_critical_t crit;
2082 void *p;
2083 struct pthread *thread = NULL;
2084
2085 if (curthread != NULL) {
2086 if (GC_NEEDED())
2087 _thr_gc(curthread);
2088 if (free_thread_count > 0) {
2089 crit = _kse_critical_enter();
2090 KSE_LOCK_ACQUIRE(curthread->kse, &thread_lock);
2091 if ((thread = TAILQ_FIRST(&free_threadq)) != NULL) {
2092 TAILQ_REMOVE(&free_threadq, thread, tle);
2093 free_thread_count--;
2094 }
2095 KSE_LOCK_RELEASE(curthread->kse, &thread_lock);
2096 _kse_critical_leave(crit);
2097 }
2098 }
2099 if (thread == NULL) {
2100 p = malloc(sizeof(struct pthread) + THR_ALIGNBYTES);
2101 if (p != NULL) {
2102 thread = (struct pthread *)THR_ALIGN(p);
2103 thread->alloc_addr = p;
2104 }
2105 }
2106 return (thread);
2107}
2108
2109void
2110_thr_free(struct pthread *curthread, struct pthread *thread)
2111{
2112 kse_critical_t crit;
2113 int i;
2114
2115 DBG_MSG("Freeing thread %p\n", thread);
2116 if ((curthread == NULL) || (free_thread_count >= MAX_CACHED_THREADS)) {
2117 for (i = 0; i < MAX_THR_LOCKLEVEL; i++) {
2118 _lockuser_destroy(&thread->lockusers[i]);
2119 }
2120 _lock_destroy(&thread->lock);
2121 free(thread->alloc_addr);
2122 }
2123 else {
2124 crit = _kse_critical_enter();
2125 KSE_LOCK_ACQUIRE(curthread->kse, &thread_lock);
2126 TAILQ_INSERT_HEAD(&free_threadq, thread, tle);
2127 free_thread_count++;
2128 KSE_LOCK_RELEASE(curthread->kse, &thread_lock);
2129 _kse_critical_leave(crit);
2130 }
2131}