Deleted Added
sdiff udiff text old ( 136439 ) new ( 136445 )
full compact
subr_sleepqueue.c (136439) subr_sleepqueue.c (136445)
1/*
2 * Copyright (c) 2004 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 48 unchanged lines hidden (view full) ---

57 * pre-existing abuse of that API. The same lock must also be held when
58 * awakening threads, though that is currently only enforced for condition
59 * variables.
60 */
61
62#include "opt_sleepqueue_profiling.h"
63
64#include <sys/cdefs.h>
1/*
2 * Copyright (c) 2004 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 48 unchanged lines hidden (view full) ---

57 * pre-existing abuse of that API. The same lock must also be held when
58 * awakening threads, though that is currently only enforced for condition
59 * variables.
60 */
61
62#include "opt_sleepqueue_profiling.h"
63
64#include <sys/cdefs.h>
65__FBSDID("$FreeBSD: head/sys/kern/subr_sleepqueue.c 136439 2004-10-12 16:31:23Z ups $");
65__FBSDID("$FreeBSD: head/sys/kern/subr_sleepqueue.c 136445 2004-10-12 18:36:20Z jhb $");
66
67#include <sys/param.h>
68#include <sys/systm.h>
69#include <sys/lock.h>
70#include <sys/kernel.h>
71#include <sys/ktr.h>
72#include <sys/malloc.h>
73#include <sys/mutex.h>

--- 34 unchanged lines hidden (view full) ---

108 * Locking key:
109 * c - sleep queue chain lock
110 */
111struct sleepqueue {
112 TAILQ_HEAD(, thread) sq_blocked; /* (c) Blocked threads. */
113 LIST_ENTRY(sleepqueue) sq_hash; /* (c) Chain and free list. */
114 LIST_HEAD(, sleepqueue) sq_free; /* (c) Free queues. */
115 void *sq_wchan; /* (c) Wait channel. */
66
67#include <sys/param.h>
68#include <sys/systm.h>
69#include <sys/lock.h>
70#include <sys/kernel.h>
71#include <sys/ktr.h>
72#include <sys/malloc.h>
73#include <sys/mutex.h>

--- 34 unchanged lines hidden (view full) ---

108 * Locking key:
109 * c - sleep queue chain lock
110 */
111struct sleepqueue {
112 TAILQ_HEAD(, thread) sq_blocked; /* (c) Blocked threads. */
113 LIST_ENTRY(sleepqueue) sq_hash; /* (c) Chain and free list. */
114 LIST_HEAD(, sleepqueue) sq_free; /* (c) Free queues. */
115 void *sq_wchan; /* (c) Wait channel. */
116 int sq_type; /* (c) Queue type. */
117#ifdef INVARIANTS
116#ifdef INVARIANTS
117 int sq_type; /* (c) Queue type. */
118 struct mtx *sq_lock; /* (c) Associated lock. */
119#endif
120};
121
122struct sleepqueue_chain {
123 LIST_HEAD(, sleepqueue) sc_queues; /* List of sleep queues. */
124 struct mtx sc_lock; /* Spin lock for this chain. */
125#ifdef SLEEPQUEUE_PROFILING

--- 77 unchanged lines hidden (view full) ---

203{
204
205 MPASS(sq != NULL);
206 MPASS(TAILQ_EMPTY(&sq->sq_blocked));
207 free(sq, M_SLEEPQUEUE);
208}
209
210/*
118 struct mtx *sq_lock; /* (c) Associated lock. */
119#endif
120};
121
122struct sleepqueue_chain {
123 LIST_HEAD(, sleepqueue) sc_queues; /* List of sleep queues. */
124 struct mtx sc_lock; /* Spin lock for this chain. */
125#ifdef SLEEPQUEUE_PROFILING

--- 77 unchanged lines hidden (view full) ---

203{
204
205 MPASS(sq != NULL);
206 MPASS(TAILQ_EMPTY(&sq->sq_blocked));
207 free(sq, M_SLEEPQUEUE);
208}
209
210/*
211 * Lock the sleep queue chain associated with the specified wait channel.
212 */
213void
214sleepq_lock(void *wchan)
215{
216 struct sleepqueue_chain *sc;
217
218 sc = SC_LOOKUP(wchan);
219 mtx_lock_spin(&sc->sc_lock);
220}
221
222/*
211 * Look up the sleep queue associated with a given wait channel in the hash
223 * Look up the sleep queue associated with a given wait channel in the hash
212 * table locking the associated sleep queue chain. Return holdind the sleep
213 * queue chain lock. If no queue is found in the table, NULL is returned.
224 * table locking the associated sleep queue chain. If no queue is found in
225 * the table, NULL is returned.
214 */
215struct sleepqueue *
216sleepq_lookup(void *wchan)
217{
218 struct sleepqueue_chain *sc;
219 struct sleepqueue *sq;
220
221 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
222 sc = SC_LOOKUP(wchan);
226 */
227struct sleepqueue *
228sleepq_lookup(void *wchan)
229{
230 struct sleepqueue_chain *sc;
231 struct sleepqueue *sq;
232
233 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
234 sc = SC_LOOKUP(wchan);
223 mtx_lock_spin(&sc->sc_lock);
235 mtx_assert(&sc->sc_lock, MA_OWNED);
224 LIST_FOREACH(sq, &sc->sc_queues, sq_hash)
225 if (sq->sq_wchan == wchan)
226 return (sq);
227 return (NULL);
228}
229
230/*
231 * Unlock the sleep queue chain associated with a given wait channel.

--- 9 unchanged lines hidden (view full) ---

241
242/*
243 * Places the current thread on the sleepqueue for the specified wait
244 * channel. If INVARIANTS is enabled, then it associates the passed in
245 * lock with the sleepq to make sure it is held when that sleep queue is
246 * woken up.
247 */
248void
236 LIST_FOREACH(sq, &sc->sc_queues, sq_hash)
237 if (sq->sq_wchan == wchan)
238 return (sq);
239 return (NULL);
240}
241
242/*
243 * Unlock the sleep queue chain associated with a given wait channel.

--- 9 unchanged lines hidden (view full) ---

253
254/*
255 * Places the current thread on the sleepqueue for the specified wait
256 * channel. If INVARIANTS is enabled, then it associates the passed in
257 * lock with the sleepq to make sure it is held when that sleep queue is
258 * woken up.
259 */
260void
249sleepq_add(struct sleepqueue *sq, void *wchan, struct mtx *lock,
250 const char *wmesg, int flags)
261sleepq_add(void *wchan, struct mtx *lock, const char *wmesg, int flags)
251{
252 struct sleepqueue_chain *sc;
262{
263 struct sleepqueue_chain *sc;
264 struct sleepqueue *sq;
253 struct thread *td, *td1;
254
255 td = curthread;
256 sc = SC_LOOKUP(wchan);
257 mtx_assert(&sc->sc_lock, MA_OWNED);
258 MPASS(td->td_sleepqueue != NULL);
259 MPASS(wchan != NULL);
260
265 struct thread *td, *td1;
266
267 td = curthread;
268 sc = SC_LOOKUP(wchan);
269 mtx_assert(&sc->sc_lock, MA_OWNED);
270 MPASS(td->td_sleepqueue != NULL);
271 MPASS(wchan != NULL);
272
261 /* If the passed in sleep queue is NULL, use this thread's queue. */
273 /* Look up the sleep queue associated with the wait channel 'wchan'. */
274 sq = sleepq_lookup(wchan);
275
276 /*
277 * If the wait channel does not already have a sleep queue, use
278 * this thread's sleep queue. Otherwise, insert the current thread
279 * into the sleep queue already in use by this wait channel.
280 */
262 if (sq == NULL) {
263#ifdef SLEEPQUEUE_PROFILING
264 sc->sc_depth++;
265 if (sc->sc_depth > sc->sc_max_depth) {
266 sc->sc_max_depth = sc->sc_depth;
267 if (sc->sc_max_depth > sleepq_max_depth)
268 sleepq_max_depth = sc->sc_max_depth;
269 }
270#endif
271 sq = td->td_sleepqueue;
272 LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
273 KASSERT(TAILQ_EMPTY(&sq->sq_blocked),
274 ("thread's sleep queue has a non-empty queue"));
275 KASSERT(LIST_EMPTY(&sq->sq_free),
276 ("thread's sleep queue has a non-empty free list"));
277 KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
278 sq->sq_wchan = wchan;
279#ifdef INVARIANTS
280 sq->sq_lock = lock;
281 if (sq == NULL) {
282#ifdef SLEEPQUEUE_PROFILING
283 sc->sc_depth++;
284 if (sc->sc_depth > sc->sc_max_depth) {
285 sc->sc_max_depth = sc->sc_depth;
286 if (sc->sc_max_depth > sleepq_max_depth)
287 sleepq_max_depth = sc->sc_max_depth;
288 }
289#endif
290 sq = td->td_sleepqueue;
291 LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
292 KASSERT(TAILQ_EMPTY(&sq->sq_blocked),
293 ("thread's sleep queue has a non-empty queue"));
294 KASSERT(LIST_EMPTY(&sq->sq_free),
295 ("thread's sleep queue has a non-empty free list"));
296 KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
297 sq->sq_wchan = wchan;
298#ifdef INVARIANTS
299 sq->sq_lock = lock;
281#endif
282 sq->sq_type = flags & SLEEPQ_TYPE;
300 sq->sq_type = flags & SLEEPQ_TYPE;
301#endif
283 TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
284 } else {
285 MPASS(wchan == sq->sq_wchan);
286 MPASS(lock == sq->sq_lock);
302 TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
303 } else {
304 MPASS(wchan == sq->sq_wchan);
305 MPASS(lock == sq->sq_lock);
306 MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);
287 TAILQ_FOREACH(td1, &sq->sq_blocked, td_slpq)
288 if (td1->td_priority > td->td_priority)
289 break;
290 if (td1 != NULL)
291 TAILQ_INSERT_BEFORE(td1, td, td_slpq);
292 else
293 TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
294 LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);

--- 68 unchanged lines hidden (view full) ---

363 PROC_UNLOCK(p);
364
365 /*
366 * If there were pending signals and this thread is still on
367 * the sleep queue, remove it from the sleep queue. If the
368 * thread was removed from the sleep queue while we were blocked
369 * above, then clear TDF_SINTR before returning.
370 */
307 TAILQ_FOREACH(td1, &sq->sq_blocked, td_slpq)
308 if (td1->td_priority > td->td_priority)
309 break;
310 if (td1 != NULL)
311 TAILQ_INSERT_BEFORE(td1, td, td_slpq);
312 else
313 TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
314 LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);

--- 68 unchanged lines hidden (view full) ---

383 PROC_UNLOCK(p);
384
385 /*
386 * If there were pending signals and this thread is still on
387 * the sleep queue, remove it from the sleep queue. If the
388 * thread was removed from the sleep queue while we were blocked
389 * above, then clear TDF_SINTR before returning.
390 */
391 sleepq_lock(wchan);
371 sq = sleepq_lookup(wchan);
372 mtx_lock_spin(&sched_lock);
373 if (TD_ON_SLEEPQ(td) && (sig != 0 || do_upcall != 0)) {
374 mtx_unlock_spin(&sched_lock);
375 sleepq_remove_thread(sq, td);
376 } else {
377 if (!TD_ON_SLEEPQ(td) && sig == 0)
378 td->td_flags &= ~TDF_SINTR;

--- 281 unchanged lines hidden (view full) ---

660 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
661 sq = sleepq_lookup(wchan);
662 if (sq == NULL) {
663 sleepq_release(wchan);
664 return;
665 }
666 KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
667 ("%s: mismatch between sleep/wakeup and cv_*", __func__));
392 sq = sleepq_lookup(wchan);
393 mtx_lock_spin(&sched_lock);
394 if (TD_ON_SLEEPQ(td) && (sig != 0 || do_upcall != 0)) {
395 mtx_unlock_spin(&sched_lock);
396 sleepq_remove_thread(sq, td);
397 } else {
398 if (!TD_ON_SLEEPQ(td) && sig == 0)
399 td->td_flags &= ~TDF_SINTR;

--- 281 unchanged lines hidden (view full) ---

681 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
682 sq = sleepq_lookup(wchan);
683 if (sq == NULL) {
684 sleepq_release(wchan);
685 return;
686 }
687 KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
688 ("%s: mismatch between sleep/wakeup and cv_*", __func__));
668 /* XXX: Do for all sleep queues eventually. */
669 if (flags & SLEEPQ_CONDVAR)
670 mtx_assert(sq->sq_lock, MA_OWNED);
671
672 /* Remove first thread from queue and awaken it. */
673 td = TAILQ_FIRST(&sq->sq_blocked);
674 sleepq_remove_thread(sq, td);
675 sleepq_release(wchan);
676 sleepq_resume_thread(td, pri);
677}
678

--- 11 unchanged lines hidden (view full) ---

690 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
691 sq = sleepq_lookup(wchan);
692 if (sq == NULL) {
693 sleepq_release(wchan);
694 return;
695 }
696 KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
697 ("%s: mismatch between sleep/wakeup and cv_*", __func__));
689
690 /* Remove first thread from queue and awaken it. */
691 td = TAILQ_FIRST(&sq->sq_blocked);
692 sleepq_remove_thread(sq, td);
693 sleepq_release(wchan);
694 sleepq_resume_thread(td, pri);
695}
696

--- 11 unchanged lines hidden (view full) ---

708 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
709 sq = sleepq_lookup(wchan);
710 if (sq == NULL) {
711 sleepq_release(wchan);
712 return;
713 }
714 KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
715 ("%s: mismatch between sleep/wakeup and cv_*", __func__));
698 /* XXX: Do for all sleep queues eventually. */
699 if (flags & SLEEPQ_CONDVAR)
700 mtx_assert(sq->sq_lock, MA_OWNED);
701
702 /* Move blocked threads from the sleep queue to a temporary list. */
703 TAILQ_INIT(&list);
704 while (!TAILQ_EMPTY(&sq->sq_blocked)) {
705 td = TAILQ_FIRST(&sq->sq_blocked);
706 sleepq_remove_thread(sq, td);
707 TAILQ_INSERT_TAIL(&list, td, td_slpq);
708 }

--- 25 unchanged lines hidden (view full) ---

734 /*
735 * First, see if the thread is asleep and get the wait channel if
736 * it is.
737 */
738 mtx_lock_spin(&sched_lock);
739 if (TD_ON_SLEEPQ(td)) {
740 wchan = td->td_wchan;
741 mtx_unlock_spin(&sched_lock);
716
717 /* Move blocked threads from the sleep queue to a temporary list. */
718 TAILQ_INIT(&list);
719 while (!TAILQ_EMPTY(&sq->sq_blocked)) {
720 td = TAILQ_FIRST(&sq->sq_blocked);
721 sleepq_remove_thread(sq, td);
722 TAILQ_INSERT_TAIL(&list, td, td_slpq);
723 }

--- 25 unchanged lines hidden (view full) ---

749 /*
750 * First, see if the thread is asleep and get the wait channel if
751 * it is.
752 */
753 mtx_lock_spin(&sched_lock);
754 if (TD_ON_SLEEPQ(td)) {
755 wchan = td->td_wchan;
756 mtx_unlock_spin(&sched_lock);
757 sleepq_lock(wchan);
742 sq = sleepq_lookup(wchan);
743 mtx_lock_spin(&sched_lock);
744 } else {
745 wchan = NULL;
746 sq = NULL;
747 }
748
749 /*

--- 47 unchanged lines hidden (view full) ---

797 struct sleepqueue *sq;
798
799 /*
800 * Look up the sleep queue for this wait channel, then re-check
801 * that the thread is asleep on that channel, if it is not, then
802 * bail.
803 */
804 MPASS(wchan != NULL);
758 sq = sleepq_lookup(wchan);
759 mtx_lock_spin(&sched_lock);
760 } else {
761 wchan = NULL;
762 sq = NULL;
763 }
764
765 /*

--- 47 unchanged lines hidden (view full) ---

813 struct sleepqueue *sq;
814
815 /*
816 * Look up the sleep queue for this wait channel, then re-check
817 * that the thread is asleep on that channel, if it is not, then
818 * bail.
819 */
820 MPASS(wchan != NULL);
821 sleepq_lock(wchan);
805 sq = sleepq_lookup(wchan);
806 mtx_lock_spin(&sched_lock);
807 if (!TD_ON_SLEEPQ(td) || td->td_wchan != wchan) {
808 mtx_unlock_spin(&sched_lock);
809 sleepq_release(wchan);
810 return;
811 }
812 mtx_unlock_spin(&sched_lock);

--- 38 unchanged lines hidden --- 		822 sq = sleepq_lookup(wchan);
823 mtx_lock_spin(&sched_lock);
824 if (!TD_ON_SLEEPQ(td) || td->td_wchan != wchan) {
825 mtx_unlock_spin(&sched_lock);
826 sleepq_release(wchan);
827 return;
828 }
829 mtx_unlock_spin(&sched_lock);

--- 38 unchanged lines hidden ---