| subr_turnstile.c (104161) | subr_turnstile.c (104387) |
|---|---|
| 1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 13 unchanged lines hidden (view full) --- 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ | 1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 13 unchanged lines hidden (view full) --- 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ |
| 30 * $FreeBSD: head/sys/kern/subr_turnstile.c 104161 2002-09-29 23:28:58Z julian $ | 30 * $FreeBSD: head/sys/kern/subr_turnstile.c 104387 2002-10-02 20:31:47Z jhb $ |
| 31 */ 32 33/* 34 * Machine independent bits of mutex implementation. 35 */ 36 37#include "opt_adaptive_mutexes.h" 38#include "opt_ddb.h" --- 119 unchanged lines hidden (view full) --- 158 /* 159 * Adjust for any other cases. 160 */ 161 td->td_priority = pri; 162 163 /* 164 * If we aren't blocked on a mutex, we should be. 165 */ | 31 */ 32 33/* 34 * Machine independent bits of mutex implementation. 35 */ 36 37#include "opt_adaptive_mutexes.h" 38#include "opt_ddb.h" --- 119 unchanged lines hidden (view full) --- 158 /* 159 * Adjust for any other cases. 160 */ 161 td->td_priority = pri; 162 163 /* 164 * If we aren't blocked on a mutex, we should be. 165 */ |
| 166 KASSERT(TD_ON_MUTEX(td), ( | 166 KASSERT(TD_ON_LOCK(td), ( |
| 167 "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 168 td->td_proc->p_pid, td->td_proc->p_comm, td->td_state, 169 m->mtx_object.lo_name)); 170 171 /* 172 * Pick up the mutex that td is blocked on. 173 */ 174 m = td->td_blocked; 175 MPASS(m != NULL); 176 177 /* 178 * Check if the thread needs to be moved up on 179 * the blocked chain 180 */ 181 if (td == TAILQ_FIRST(&m->mtx_blocked)) { 182 continue; 183 } 184 | 167 "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 168 td->td_proc->p_pid, td->td_proc->p_comm, td->td_state, 169 m->mtx_object.lo_name)); 170 171 /* 172 * Pick up the mutex that td is blocked on. 173 */ 174 m = td->td_blocked; 175 MPASS(m != NULL); 176 177 /* 178 * Check if the thread needs to be moved up on 179 * the blocked chain 180 */ 181 if (td == TAILQ_FIRST(&m->mtx_blocked)) { 182 continue; 183 } 184 |
| 185 td1 = TAILQ_PREV(td, threadqueue, td_blkq); | 185 td1 = TAILQ_PREV(td, threadqueue, td_lockq); |
| 186 if (td1->td_priority <= pri) { 187 continue; 188 } 189 190 /* 191 * Remove thread from blocked chain and determine where 192 * it should be moved up to. Since we know that td1 has 193 * a lower priority than td, we know that at least one 194 * thread in the chain has a lower priority and that 195 * td1 will thus not be NULL after the loop. 196 */ | 186 if (td1->td_priority <= pri) { 187 continue; 188 } 189 190 /* 191 * Remove thread from blocked chain and determine where 192 * it should be moved up to. Since we know that td1 has 193 * a lower priority than td, we know that at least one 194 * thread in the chain has a lower priority and that 195 * td1 will thus not be NULL after the loop. 196 */ |
| 197 TAILQ_REMOVE(&m->mtx_blocked, td, td_blkq); 198 TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) { | 197 TAILQ_REMOVE(&m->mtx_blocked, td, td_lockq); 198 TAILQ_FOREACH(td1, &m->mtx_blocked, td_lockq) { |
| 199 MPASS(td1->td_proc->p_magic == P_MAGIC); 200 if (td1->td_priority > pri) 201 break; 202 } 203 204 MPASS(td1 != NULL); | 199 MPASS(td1->td_proc->p_magic == P_MAGIC); 200 if (td1->td_priority > pri) 201 break; 202 } 203 204 MPASS(td1 != NULL); |
| 205 TAILQ_INSERT_BEFORE(td1, td, td_blkq); | 205 TAILQ_INSERT_BEFORE(td1, td, td_lockq); |
| 206 CTR4(KTR_LOCK, 207 "propagate_priority: p %p moved before %p on [%p] %s", 208 td, td1, m, m->mtx_object.lo_name); 209 } 210} 211 212#ifdef MUTEX_PROFILING 213SYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging"); --- 372 unchanged lines hidden (view full) --- 586#endif 587 588 /* 589 * Put us on the list of threads blocked on this mutex. 590 */ 591 if (TAILQ_EMPTY(&m->mtx_blocked)) { 592 td1 = mtx_owner(m); 593 LIST_INSERT_HEAD(&td1->td_contested, m, mtx_contested); | 206 CTR4(KTR_LOCK, 207 "propagate_priority: p %p moved before %p on [%p] %s", 208 td, td1, m, m->mtx_object.lo_name); 209 } 210} 211 212#ifdef MUTEX_PROFILING 213SYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging"); --- 372 unchanged lines hidden (view full) --- 586#endif 587 588 /* 589 * Put us on the list of threads blocked on this mutex. 590 */ 591 if (TAILQ_EMPTY(&m->mtx_blocked)) { 592 td1 = mtx_owner(m); 593 LIST_INSERT_HEAD(&td1->td_contested, m, mtx_contested); |
| 594 TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); | 594 TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_lockq); |
| 595 } else { | 595 } else { |
| 596 TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) | 596 TAILQ_FOREACH(td1, &m->mtx_blocked, td_lockq) |
| 597 if (td1->td_priority > td->td_priority) 598 break; 599 if (td1) | 597 if (td1->td_priority > td->td_priority) 598 break; 599 if (td1) |
| 600 TAILQ_INSERT_BEFORE(td1, td, td_blkq); | 600 TAILQ_INSERT_BEFORE(td1, td, td_lockq); |
| 601 else | 601 else |
| 602 TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); | 602 TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_lockq); |
| 603 } 604#ifdef KTR 605 if (!cont_logged) { 606 CTR6(KTR_CONTENTION, 607 "contention: %p at %s:%d wants %s, taken by %s:%d", 608 td, file, line, m->mtx_object.lo_name, 609 WITNESS_FILE(&m->mtx_object), 610 WITNESS_LINE(&m->mtx_object)); 611 cont_logged = 1; 612 } 613#endif 614 615 /* 616 * Save who we're blocked on. 617 */ 618 td->td_blocked = m; | 603 } 604#ifdef KTR 605 if (!cont_logged) { 606 CTR6(KTR_CONTENTION, 607 "contention: %p at %s:%d wants %s, taken by %s:%d", 608 td, file, line, m->mtx_object.lo_name, 609 WITNESS_FILE(&m->mtx_object), 610 WITNESS_LINE(&m->mtx_object)); 611 cont_logged = 1; 612 } 613#endif 614 615 /* 616 * Save who we're blocked on. 617 */ 618 td->td_blocked = m; |
| 619 td->td_mtxname = m->mtx_object.lo_name; 620 TD_SET_MUTEX(td); | 619 td->td_lockname = m->mtx_object.lo_name; 620 TD_SET_LOCK(td); |
| 621 propagate_priority(td); 622 623 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 624 CTR3(KTR_LOCK, 625 "_mtx_lock_sleep: p %p blocked on [%p] %s", td, m, 626 m->mtx_object.lo_name); 627 628 td->td_proc->p_stats->p_ru.ru_nvcsw++; --- 101 unchanged lines hidden (view full) --- 730 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 731 mtx_unlock_spin(&sched_lock); 732 return; 733 } 734#endif 735 MPASS(td->td_proc->p_magic == P_MAGIC); 736 MPASS(td1->td_proc->p_magic == P_MAGIC); 737 | 621 propagate_priority(td); 622 623 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 624 CTR3(KTR_LOCK, 625 "_mtx_lock_sleep: p %p blocked on [%p] %s", td, m, 626 m->mtx_object.lo_name); 627 628 td->td_proc->p_stats->p_ru.ru_nvcsw++; --- 101 unchanged lines hidden (view full) --- 730 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 731 mtx_unlock_spin(&sched_lock); 732 return; 733 } 734#endif 735 MPASS(td->td_proc->p_magic == P_MAGIC); 736 MPASS(td1->td_proc->p_magic == P_MAGIC); 737 |
| 738 TAILQ_REMOVE(&m->mtx_blocked, td1, td_blkq); | 738 TAILQ_REMOVE(&m->mtx_blocked, td1, td_lockq); |
| 739 740 if (TAILQ_EMPTY(&m->mtx_blocked)) { 741 LIST_REMOVE(m, mtx_contested); 742 _release_lock_quick(m); 743 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 744 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 745 } else 746 atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); --- 9 unchanged lines hidden (view full) --- 756 pri = td->td_base_pri; 757 td->td_priority = pri; 758 759 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 760 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 761 m, td1); 762 763 td1->td_blocked = NULL; | 739 740 if (TAILQ_EMPTY(&m->mtx_blocked)) { 741 LIST_REMOVE(m, mtx_contested); 742 _release_lock_quick(m); 743 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 744 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 745 } else 746 atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); --- 9 unchanged lines hidden (view full) --- 756 pri = td->td_base_pri; 757 td->td_priority = pri; 758 759 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 760 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 761 m, td1); 762 763 td1->td_blocked = NULL; |
| 764 TD_CLR_MUTEX(td1); | 764 TD_CLR_LOCK(td1); |
| 765 if (!TD_CAN_RUN(td1)) { 766 mtx_unlock_spin(&sched_lock); 767 return; 768 } 769 setrunqueue(td1); 770 771 if (td->td_critnest == 1 && td1->td_priority < pri) { 772#ifdef notyet --- 251 unchanged lines hidden --- | 765 if (!TD_CAN_RUN(td1)) { 766 mtx_unlock_spin(&sched_lock); 767 return; 768 } 769 setrunqueue(td1); 770 771 if (td->td_critnest == 1 && td1->td_priority < pri) { 772#ifdef notyet --- 251 unchanged lines hidden --- |