| subr_sleepqueue.c (176258) | subr_sleepqueue.c (177085) |
|---|---|
| 1/*- 2 * Copyright (c) 2004 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 46 unchanged lines hidden (view full) --- 55 * must consistently use the same lock to synchronize with a wait channel, 56 * though this check is currently only a warning for sleep/wakeup due to 57 * pre-existing abuse of that API. The same lock must also be held when 58 * awakening threads, though that is currently only enforced for condition 59 * variables. 60 */ 61 62#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2004 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 46 unchanged lines hidden (view full) --- 55 * must consistently use the same lock to synchronize with a wait channel, 56 * though this check is currently only a warning for sleep/wakeup due to 57 * pre-existing abuse of that API. The same lock must also be held when 58 * awakening threads, though that is currently only enforced for condition 59 * variables. 60 */ 61 62#include <sys/cdefs.h> |
| 63__FBSDID("$FreeBSD: head/sys/kern/subr_sleepqueue.c 176258 2008-02-13 23:36:56Z jhb $"); | 63__FBSDID("$FreeBSD: head/sys/kern/subr_sleepqueue.c 177085 2008-03-12 06:31:06Z jeff $"); |
| 64 65#include "opt_sleepqueue_profiling.h" 66#include "opt_ddb.h" 67#include "opt_sched.h" 68 69#include <sys/param.h> 70#include <sys/systm.h> 71#include <sys/lock.h> --- 72 unchanged lines hidden (view full) --- 144 0, "maxmimum depth achieved of a single chain"); 145#endif 146static struct sleepqueue_chain sleepq_chains[SC_TABLESIZE]; 147static uma_zone_t sleepq_zone; 148 149/* 150 * Prototypes for non-exported routines. 151 */ | 64 65#include "opt_sleepqueue_profiling.h" 66#include "opt_ddb.h" 67#include "opt_sched.h" 68 69#include <sys/param.h> 70#include <sys/systm.h> 71#include <sys/lock.h> --- 72 unchanged lines hidden (view full) --- 144 0, "maxmimum depth achieved of a single chain"); 145#endif 146static struct sleepqueue_chain sleepq_chains[SC_TABLESIZE]; 147static uma_zone_t sleepq_zone; 148 149/* 150 * Prototypes for non-exported routines. 151 */ |
| 152static int sleepq_catch_signals(void *wchan); | 152static int sleepq_catch_signals(void *wchan, int pri); |
| 153static int sleepq_check_signals(void); 154static int sleepq_check_timeout(void); 155#ifdef INVARIANTS 156static void sleepq_dtor(void *mem, int size, void *arg); 157#endif 158static int sleepq_init(void *mem, int size, int flags); 159static void sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, 160 int pri); | 153static int sleepq_check_signals(void); 154static int sleepq_check_timeout(void); 155#ifdef INVARIANTS 156static void sleepq_dtor(void *mem, int size, void *arg); 157#endif 158static int sleepq_init(void *mem, int size, int flags); 159static void sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, 160 int pri); |
| 161static void sleepq_switch(void *wchan); | 161static void sleepq_switch(void *wchan, int pri); |
| 162static void sleepq_timeout(void *arg); 163 164/* 165 * Early initialization of sleep queues that is called from the sleepinit() 166 * SYSINIT. 167 */ 168void 169init_sleepqueues(void) --- 192 unchanged lines hidden (view full) --- 362 363/* 364 * Marks the pending sleep of the current thread as interruptible and 365 * makes an initial check for pending signals before putting a thread 366 * to sleep. Enters and exits with the thread lock held. Thread lock 367 * may have transitioned from the sleepq lock to a run lock. 368 */ 369static int | 162static void sleepq_timeout(void *arg); 163 164/* 165 * Early initialization of sleep queues that is called from the sleepinit() 166 * SYSINIT. 167 */ 168void 169init_sleepqueues(void) --- 192 unchanged lines hidden (view full) --- 362 363/* 364 * Marks the pending sleep of the current thread as interruptible and 365 * makes an initial check for pending signals before putting a thread 366 * to sleep. Enters and exits with the thread lock held. Thread lock 367 * may have transitioned from the sleepq lock to a run lock. 368 */ 369static int |
| 370sleepq_catch_signals(void *wchan) | 370sleepq_catch_signals(void *wchan, int pri) |
| 371{ 372 struct sleepqueue_chain *sc; 373 struct sleepqueue *sq; 374 struct thread *td; 375 struct proc *p; 376 struct sigacts *ps; 377 int sig, ret; 378 --- 27 unchanged lines hidden (view full) --- 406 * Lock sleepq chain before unlocking proc 407 * without this, we could lose a race. 408 */ 409 mtx_lock_spin(&sc->sc_lock); 410 PROC_UNLOCK(p); 411 thread_lock(td); 412 if (ret == 0) { 413 if (!(td->td_flags & TDF_INTERRUPT)) { | 371{ 372 struct sleepqueue_chain *sc; 373 struct sleepqueue *sq; 374 struct thread *td; 375 struct proc *p; 376 struct sigacts *ps; 377 int sig, ret; 378 --- 27 unchanged lines hidden (view full) --- 406 * Lock sleepq chain before unlocking proc 407 * without this, we could lose a race. 408 */ 409 mtx_lock_spin(&sc->sc_lock); 410 PROC_UNLOCK(p); 411 thread_lock(td); 412 if (ret == 0) { 413 if (!(td->td_flags & TDF_INTERRUPT)) { |
| 414 sleepq_switch(wchan); | 414 sleepq_switch(wchan, pri); |
| 415 return (0); 416 } 417 /* KSE threads tried unblocking us. */ 418 ret = td->td_intrval; 419 MPASS(ret == EINTR || ret == ERESTART || ret == EWOULDBLOCK); 420 } 421 /* 422 * There were pending signals and this thread is still 423 * on the sleep queue, remove it from the sleep queue. 424 */ 425 if (TD_ON_SLEEPQ(td)) { 426 sq = sleepq_lookup(wchan); | 415 return (0); 416 } 417 /* KSE threads tried unblocking us. */ 418 ret = td->td_intrval; 419 MPASS(ret == EINTR || ret == ERESTART || ret == EWOULDBLOCK); 420 } 421 /* 422 * There were pending signals and this thread is still 423 * on the sleep queue, remove it from the sleep queue. 424 */ 425 if (TD_ON_SLEEPQ(td)) { 426 sq = sleepq_lookup(wchan); |
| 427 sleepq_resume_thread(sq, td, -1); | 427 sleepq_resume_thread(sq, td, 0); |
| 428 } 429 mtx_unlock_spin(&sc->sc_lock); 430 MPASS(td->td_lock != &sc->sc_lock); 431 return (ret); 432} 433 434/* 435 * Switches to another thread if we are still asleep on a sleep queue. 436 * Returns with thread lock. 437 */ 438static void | 428 } 429 mtx_unlock_spin(&sc->sc_lock); 430 MPASS(td->td_lock != &sc->sc_lock); 431 return (ret); 432} 433 434/* 435 * Switches to another thread if we are still asleep on a sleep queue. 436 * Returns with thread lock. 437 */ 438static void |
| 439sleepq_switch(void *wchan) | 439sleepq_switch(void *wchan, int pri) |
| 440{ 441 struct sleepqueue_chain *sc; 442 struct sleepqueue *sq; 443 struct thread *td; 444 445 td = curthread; 446 sc = SC_LOOKUP(wchan); 447 mtx_assert(&sc->sc_lock, MA_OWNED); --- 11 unchanged lines hidden (view full) --- 459 /* 460 * If TDF_TIMEOUT is set, then our sleep has been timed out 461 * already but we are still on the sleep queue, so dequeue the 462 * thread and return. 463 */ 464 if (td->td_flags & TDF_TIMEOUT) { 465 MPASS(TD_ON_SLEEPQ(td)); 466 sq = sleepq_lookup(wchan); | 440{ 441 struct sleepqueue_chain *sc; 442 struct sleepqueue *sq; 443 struct thread *td; 444 445 td = curthread; 446 sc = SC_LOOKUP(wchan); 447 mtx_assert(&sc->sc_lock, MA_OWNED); --- 11 unchanged lines hidden (view full) --- 459 /* 460 * If TDF_TIMEOUT is set, then our sleep has been timed out 461 * already but we are still on the sleep queue, so dequeue the 462 * thread and return. 463 */ 464 if (td->td_flags & TDF_TIMEOUT) { 465 MPASS(TD_ON_SLEEPQ(td)); 466 sq = sleepq_lookup(wchan); |
| 467 sleepq_resume_thread(sq, td, -1); | 467 sleepq_resume_thread(sq, td, 0); |
| 468 mtx_unlock_spin(&sc->sc_lock); 469 return; 470 } 471 | 468 mtx_unlock_spin(&sc->sc_lock); 469 return; 470 } 471 |
| 472 thread_lock_set(td, &sc->sc_lock); 473 | |
| 474 MPASS(td->td_sleepqueue == NULL); | 472 MPASS(td->td_sleepqueue == NULL); |
| 475 sched_sleep(td); | 473 sched_sleep(td, pri); 474 thread_lock_set(td, &sc->sc_lock); |
| 476 TD_SET_SLEEPING(td); 477 SCHED_STAT_INC(switch_sleepq); 478 mi_switch(SW_VOL, NULL); 479 KASSERT(TD_IS_RUNNING(td), ("running but not TDS_RUNNING")); 480 CTR3(KTR_PROC, "sleepq resume: thread %p (pid %ld, %s)", 481 (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name); 482} 483 --- 62 unchanged lines hidden (view full) --- 546 547 return (0); 548} 549 550/* 551 * Block the current thread until it is awakened from its sleep queue. 552 */ 553void | 475 TD_SET_SLEEPING(td); 476 SCHED_STAT_INC(switch_sleepq); 477 mi_switch(SW_VOL, NULL); 478 KASSERT(TD_IS_RUNNING(td), ("running but not TDS_RUNNING")); 479 CTR3(KTR_PROC, "sleepq resume: thread %p (pid %ld, %s)", 480 (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name); 481} 482 --- 62 unchanged lines hidden (view full) --- 545 546 return (0); 547} 548 549/* 550 * Block the current thread until it is awakened from its sleep queue. 551 */ 552void |
| 554sleepq_wait(void *wchan) | 553sleepq_wait(void *wchan, int pri) |
| 555{ 556 struct thread *td; 557 558 td = curthread; 559 MPASS(!(td->td_flags & TDF_SINTR)); 560 thread_lock(td); | 554{ 555 struct thread *td; 556 557 td = curthread; 558 MPASS(!(td->td_flags & TDF_SINTR)); 559 thread_lock(td); |
| 561 sleepq_switch(wchan); | 560 sleepq_switch(wchan, pri); |
| 562 thread_unlock(td); 563} 564 565/* 566 * Block the current thread until it is awakened from its sleep queue 567 * or it is interrupted by a signal. 568 */ 569int | 561 thread_unlock(td); 562} 563 564/* 565 * Block the current thread until it is awakened from its sleep queue 566 * or it is interrupted by a signal. 567 */ 568int |
| 570sleepq_wait_sig(void *wchan) | 569sleepq_wait_sig(void *wchan, int pri) |
| 571{ 572 int rcatch; 573 int rval; 574 | 570{ 571 int rcatch; 572 int rval; 573 |
| 575 rcatch = sleepq_catch_signals(wchan); | 574 rcatch = sleepq_catch_signals(wchan, pri); |
| 576 rval = sleepq_check_signals(); 577 thread_unlock(curthread); 578 if (rcatch) 579 return (rcatch); 580 return (rval); 581} 582 583/* 584 * Block the current thread until it is awakened from its sleep queue 585 * or it times out while waiting. 586 */ 587int | 575 rval = sleepq_check_signals(); 576 thread_unlock(curthread); 577 if (rcatch) 578 return (rcatch); 579 return (rval); 580} 581 582/* 583 * Block the current thread until it is awakened from its sleep queue 584 * or it times out while waiting. 585 */ 586int |
| 588sleepq_timedwait(void *wchan) | 587sleepq_timedwait(void *wchan, int pri) |
| 589{ 590 struct thread *td; 591 int rval; 592 593 td = curthread; 594 MPASS(!(td->td_flags & TDF_SINTR)); 595 thread_lock(td); | 588{ 589 struct thread *td; 590 int rval; 591 592 td = curthread; 593 MPASS(!(td->td_flags & TDF_SINTR)); 594 thread_lock(td); |
| 596 sleepq_switch(wchan); | 595 sleepq_switch(wchan, pri); |
| 597 rval = sleepq_check_timeout(); 598 thread_unlock(td); 599 600 return (rval); 601} 602 603/* 604 * Block the current thread until it is awakened from its sleep queue, 605 * it is interrupted by a signal, or it times out waiting to be awakened. 606 */ 607int | 596 rval = sleepq_check_timeout(); 597 thread_unlock(td); 598 599 return (rval); 600} 601 602/* 603 * Block the current thread until it is awakened from its sleep queue, 604 * it is interrupted by a signal, or it times out waiting to be awakened. 605 */ 606int |
| 608sleepq_timedwait_sig(void *wchan) | 607sleepq_timedwait_sig(void *wchan, int pri) |
| 609{ 610 int rcatch, rvalt, rvals; 611 | 608{ 609 int rcatch, rvalt, rvals; 610 |
| 612 rcatch = sleepq_catch_signals(wchan); | 611 rcatch = sleepq_catch_signals(wchan, pri); |
| 613 rvalt = sleepq_check_timeout(); 614 rvals = sleepq_check_signals(); 615 thread_unlock(curthread); 616 if (rcatch) 617 return (rcatch); 618 if (rvals) 619 return (rvals); 620 return (rvalt); --- 47 unchanged lines hidden (view full) --- 668 * the sleeping flag if it isn't set though, so we just always 669 * do it. However, we can't assert that it is set. 670 */ 671 CTR3(KTR_PROC, "sleepq_wakeup: thread %p (pid %ld, %s)", 672 (void *)td, (long)td->td_proc->p_pid, td->td_name); 673 TD_CLR_SLEEPING(td); 674 675 /* Adjust priority if requested. */ | 612 rvalt = sleepq_check_timeout(); 613 rvals = sleepq_check_signals(); 614 thread_unlock(curthread); 615 if (rcatch) 616 return (rcatch); 617 if (rvals) 618 return (rvals); 619 return (rvalt); --- 47 unchanged lines hidden (view full) --- 667 * the sleeping flag if it isn't set though, so we just always 668 * do it. However, we can't assert that it is set. 669 */ 670 CTR3(KTR_PROC, "sleepq_wakeup: thread %p (pid %ld, %s)", 671 (void *)td, (long)td->td_proc->p_pid, td->td_name); 672 TD_CLR_SLEEPING(td); 673 674 /* Adjust priority if requested. */ |
| 676 MPASS(pri == -1 || (pri >= PRI_MIN && pri <= PRI_MAX)); 677 if (pri != -1 && td->td_priority > pri) | 675 MPASS(pri == 0 || (pri >= PRI_MIN && pri <= PRI_MAX)); 676 if (pri != 0 && td->td_priority > pri) |
| 678 sched_prio(td, pri); 679 setrunnable(td); 680} 681 682#ifdef INVARIANTS 683/* 684 * UMA zone item deallocator. 685 */ --- 69 unchanged lines hidden (view full) --- 755{ 756 struct sleepqueue *sq; 757 struct thread *td; 758 759 CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags); 760 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); 761 MPASS((queue >= 0) && (queue < NR_SLEEPQS)); 762 sq = sleepq_lookup(wchan); | 677 sched_prio(td, pri); 678 setrunnable(td); 679} 680 681#ifdef INVARIANTS 682/* 683 * UMA zone item deallocator. 684 */ --- 69 unchanged lines hidden (view full) --- 754{ 755 struct sleepqueue *sq; 756 struct thread *td; 757 758 CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags); 759 KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); 760 MPASS((queue >= 0) && (queue < NR_SLEEPQS)); 761 sq = sleepq_lookup(wchan); |
| 763 if (sq == NULL) { 764 sleepq_release(wchan); | 762 if (sq == NULL) |
| 765 return; | 763 return; |
| 766 } | |
| 767 KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE), 768 ("%s: mismatch between sleep/wakeup and cv_*", __func__)); 769 770 /* Resume all blocked threads on the sleep queue. */ 771 while (!TAILQ_EMPTY(&sq->sq_blocked[queue])) { 772 td = TAILQ_FIRST(&sq->sq_blocked[queue]); 773 thread_lock(td); 774 sleepq_resume_thread(sq, td, pri); 775 thread_unlock(td); 776 } | 764 KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE), 765 ("%s: mismatch between sleep/wakeup and cv_*", __func__)); 766 767 /* Resume all blocked threads on the sleep queue. */ 768 while (!TAILQ_EMPTY(&sq->sq_blocked[queue])) { 769 td = TAILQ_FIRST(&sq->sq_blocked[queue]); 770 thread_lock(td); 771 sleepq_resume_thread(sq, td, pri); 772 thread_unlock(td); 773 } |
| 777 sleepq_release(wchan); | |
| 778} 779 780/* 781 * Time sleeping threads out. When the timeout expires, the thread is 782 * removed from the sleep queue and made runnable if it is still asleep. 783 */ 784static void 785sleepq_timeout(void *arg) --- 14 unchanged lines hidden (view full) --- 800 thread_lock(td); 801 if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) { 802 wchan = td->td_wchan; 803 sc = SC_LOOKUP(wchan); 804 THREAD_LOCKPTR_ASSERT(td, &sc->sc_lock); 805 sq = sleepq_lookup(wchan); 806 MPASS(sq != NULL); 807 td->td_flags |= TDF_TIMEOUT; | 774} 775 776/* 777 * Time sleeping threads out. When the timeout expires, the thread is 778 * removed from the sleep queue and made runnable if it is still asleep. 779 */ 780static void 781sleepq_timeout(void *arg) --- 14 unchanged lines hidden (view full) --- 796 thread_lock(td); 797 if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) { 798 wchan = td->td_wchan; 799 sc = SC_LOOKUP(wchan); 800 THREAD_LOCKPTR_ASSERT(td, &sc->sc_lock); 801 sq = sleepq_lookup(wchan); 802 MPASS(sq != NULL); 803 td->td_flags |= TDF_TIMEOUT; |
| 808 sleepq_resume_thread(sq, td, -1); | 804 sleepq_resume_thread(sq, td, 0); |
| 809 thread_unlock(td); 810 return; 811 } 812 813 /* 814 * If the thread is on the SLEEPQ but isn't sleeping yet, it 815 * can either be on another CPU in between sleepq_add() and 816 * one of the sleepq_*wait*() routines or it can be in --- 50 unchanged lines hidden (view full) --- 867 if (!TD_ON_SLEEPQ(td) || td->td_wchan != wchan) { 868 sleepq_release(wchan); 869 return; 870 } 871 /* Thread is asleep on sleep queue sq, so wake it up. */ 872 thread_lock(td); 873 MPASS(sq != NULL); 874 MPASS(td->td_wchan == wchan); | 805 thread_unlock(td); 806 return; 807 } 808 809 /* 810 * If the thread is on the SLEEPQ but isn't sleeping yet, it 811 * can either be on another CPU in between sleepq_add() and 812 * one of the sleepq_*wait*() routines or it can be in --- 50 unchanged lines hidden (view full) --- 863 if (!TD_ON_SLEEPQ(td) || td->td_wchan != wchan) { 864 sleepq_release(wchan); 865 return; 866 } 867 /* Thread is asleep on sleep queue sq, so wake it up. */ 868 thread_lock(td); 869 MPASS(sq != NULL); 870 MPASS(td->td_wchan == wchan); |
| 875 sleepq_resume_thread(sq, td, -1); | 871 sleepq_resume_thread(sq, td, 0); |
| 876 thread_unlock(td); 877 sleepq_release(wchan); 878} 879 880/* 881 * Abort a thread as if an interrupt had occurred. Only abort 882 * interruptible waits (unfortunately it isn't safe to abort others). 883 */ --- 27 unchanged lines hidden (view full) --- 911 if (!TD_IS_SLEEPING(td)) 912 return; 913 wchan = td->td_wchan; 914 MPASS(wchan != NULL); 915 sq = sleepq_lookup(wchan); 916 MPASS(sq != NULL); 917 918 /* Thread is asleep on sleep queue sq, so wake it up. */ | 872 thread_unlock(td); 873 sleepq_release(wchan); 874} 875 876/* 877 * Abort a thread as if an interrupt had occurred. Only abort 878 * interruptible waits (unfortunately it isn't safe to abort others). 879 */ --- 27 unchanged lines hidden (view full) --- 907 if (!TD_IS_SLEEPING(td)) 908 return; 909 wchan = td->td_wchan; 910 MPASS(wchan != NULL); 911 sq = sleepq_lookup(wchan); 912 MPASS(sq != NULL); 913 914 /* Thread is asleep on sleep queue sq, so wake it up. */ |
| 919 sleepq_resume_thread(sq, td, -1); | 915 sleepq_resume_thread(sq, td, 0); |
| 920} 921 922#ifdef DDB 923DB_SHOW_COMMAND(sleepq, db_show_sleepqueue) 924{ 925 struct sleepqueue_chain *sc; 926 struct sleepqueue *sq; 927#ifdef INVARIANTS --- 60 unchanged lines hidden --- | 916} 917 918#ifdef DDB 919DB_SHOW_COMMAND(sleepq, db_show_sleepqueue) 920{ 921 struct sleepqueue_chain *sc; 922 struct sleepqueue *sq; 923#ifdef INVARIANTS --- 60 unchanged lines hidden --- |