sys/kern/subr_turnstile.c

160 static void	propagate_priority(struct thread *td);
162 		    struct thread *td);
182 propagate_priority(struct thread *td)
187 	THREAD_LOCK_ASSERT(td, MA_OWNED);
188 	pri = td->td_priority;
189 	ts = td->td_blocked;
190 	THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
199 		td = ts->ts_owner;
201 		if (td == NULL) {
210 		thread_lock_flags(td, MTX_DUPOK);
212 		MPASS(td->td_proc != NULL);
213 		MPASS(td->td_proc->p_magic == P_MAGIC);
221 		if (TD_IS_SLEEPING(td)) {
224 			    td->td_tid, td->td_proc->p_pid);
225 			kdb_backtrace_thread(td);
233 		if (td->td_priority <= pri) {
234 			thread_unlock(td);
241 		sched_lend_prio(td, pri);
247 		if (TD_IS_RUNNING(td) || TD_ON_RUNQ(td)) {
248 			MPASS(td->td_blocked == NULL);
249 			thread_unlock(td);
255 		 * For UP, we check to see if td is curthread (this shouldn't
258 		KASSERT(td != curthread, ("Deadlock detected"));
264 		KASSERT(TD_ON_LOCK(td), (
266 		    td->td_tid, td->td_name, td->td_state,
270 		 * Pick up the lock that td is blocked on.
272 		ts = td->td_blocked;
274 		THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
275 		/* Resort td on the list if needed. */
276 		if (!turnstile_adjust_thread(ts, td)) {
289 turnstile_adjust_thread(struct turnstile *ts, struct thread *td)
294 	THREAD_LOCK_ASSERT(td, MA_OWNED);
295 	MPASS(TD_ON_LOCK(td));
307 	if (td->td_turnstile != NULL)
315 	THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
316 	td1 = TAILQ_PREV(td, threadqueue, td_lockq);
317 	td2 = TAILQ_NEXT(td, td_lockq);
318 	if ((td1 != NULL && td->td_priority < td1->td_priority) ||
319 	    (td2 != NULL && td->td_priority > td2->td_priority)) {
325 		queue = td->td_tsqueue;
328 		TAILQ_REMOVE(&ts->ts_blocked[queue], td, td_lockq);
331 			if (td1->td_priority > td->td_priority)
336 			TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
338 			TAILQ_INSERT_BEFORE(td1, td, td_lockq);
342 		    "turnstile_adjust_thread: td %d put at tail on [%p] %s",
343 			    td->td_tid, ts->ts_lockobj, ts->ts_lockobj->lo_name);
346 		    "turnstile_adjust_thread: td %d moved before %d on [%p] %s",
347 			    td->td_tid, td1->td_tid, ts->ts_lockobj,
419 turnstile_adjust(struct thread *td, u_char oldpri)
423 	MPASS(TD_ON_LOCK(td));
426 	 * Pick up the lock that td is blocked on.
428 	ts = td->td_blocked;
430 	THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
434 	if (!turnstile_adjust_thread(ts, td))
442 	MPASS(td->td_tsqueue == TS_EXCLUSIVE_QUEUE ||
443 	    td->td_tsqueue == TS_SHARED_QUEUE);
444 	if (td == TAILQ_FIRST(&ts->ts_blocked[td->td_tsqueue]) &&
445 	    td->td_priority < oldpri) {
446 		propagate_priority(td);
639 	struct thread *td, *owner;
650 	td = turnstile_first_waiter(ts);
651 	MPASS(td != NULL);
652 	MPASS(td->td_proc->p_magic == P_MAGIC);
653 	THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
659 	if (td->td_priority < owner->td_priority)
660 		sched_lend_prio(owner, td->td_priority);
677 	struct thread *td, *td1;
680 	td = curthread;
693 	if (ts == td->td_turnstile) {
713 		TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
718 			if (td1->td_priority > td->td_priority)
722 			TAILQ_INSERT_BEFORE(td1, td, td_lockq);
724 			TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
727 		MPASS(td->td_turnstile != NULL);
728 		LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash);
730 	thread_lock(td);
731 	thread_lock_set(td, &ts->ts_lock);
732 	td->td_turnstile = NULL;
736 	td->td_tsqueue = queue;
737 	td->td_blocked = ts;
738 	td->td_lockname = lock->lo_name;
739 	td->td_blktick = ticks;
740 	TD_SET_LOCK(td);
742 	propagate_priority(td);
745 		CTR4(KTR_LOCK, "%s: td %d blocked on [%p] %s", __func__,
746 		    td->td_tid, lock, lock->lo_name);
750 	THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
754 		CTR4(KTR_LOCK, "%s: td %d free from blocked on [%p] %s",
755 		    __func__, td->td_tid, lock, lock->lo_name);
756 	thread_unlock(td);
767 	struct thread *td;
780 	td = TAILQ_FIRST(&ts->ts_blocked[queue]);
781 	MPASS(td->td_proc->p_magic == P_MAGIC);
783 	TAILQ_REMOVE(&ts->ts_blocked[queue], td, td_lockq);
785 	TAILQ_INSERT_TAIL(&ts->ts_pending, td, td_lockq);
805 	td->td_turnstile = ts;
819 	struct thread *td;
844 	TAILQ_FOREACH(td, &ts->ts_pending, td_lockq) {
846 			MPASS(TAILQ_NEXT(td, td_lockq) == NULL);
855 		td->td_turnstile = ts1;
869 	struct thread *td;
893 	td = curthread;
895 	thread_lock(td);
908 	LIST_FOREACH(nts, &td->td_contested, ts_link) {
914 	sched_unlend_prio(td, pri);
915 	thread_unlock(td);
924 		td = TAILQ_FIRST(&pending_threads);
925 		TAILQ_REMOVE(&pending_threads, td, td_lockq);
926 		SDT_PROBE2(sched, , , wakeup, td, td->td_proc);
927 		thread_lock(td);
928 		THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
929 		MPASS(td->td_proc->p_magic == P_MAGIC);
930 		MPASS(TD_ON_LOCK(td));
931 		TD_CLR_LOCK(td);
932 		MPASS(TD_CAN_RUN(td));
933 		td->td_blocked = NULL;
934 		td->td_lockname = NULL;
935 		td->td_blktick = 0;
937 		td->td_tsqueue = 0xff;
939 		sched_add(td, SRQ_BORING);
940 		thread_unlock(td);
952 	struct thread *td;
978 	td = curthread;
980 	thread_lock(td);
983 	LIST_FOREACH(ts, &td->td_contested, ts_link) {
989 	sched_unlend_prio(td, pri);
990 	thread_unlock(td);
1025 print_thread(struct thread *td, const char *prefix)
1028 	db_printf("%s%p (tid %d, pid %d, \"%s\")\n", prefix, td, td->td_tid,
1029 	    td->td_proc->p_pid, td->td_name);
1035 	struct thread *td;
1042 	TAILQ_FOREACH(td, queue, td_lockq) {
1043 		print_thread(td, prefix);
1099 print_lockchain(struct thread *td, const char *prefix)
1110 		db_printf("%sthread %d (pid %d, %s) ", prefix, td->td_tid,
1111 		    td->td_proc->p_pid, td->td_name);
1112 		switch (td->td_state) {
1123 			db_printf("running on CPU %d\n", td->td_oncpu);
1126 			if (TD_ON_LOCK(td)) {
1127 				ts = td->td_blocked;
1134 				td = ts->ts_owner;
1140 			db_printf("??? (%#x)\n", td->td_state);
1148 	struct thread *td;
1152 		td = db_lookup_thread(addr, true);
1154 		td = kdb_thread;
1156 	print_lockchain(td, "");
1161 	struct thread *td;
1167 		FOREACH_THREAD_IN_PROC(p, td) {
1168 			if (TD_ON_LOCK(td) && LIST_EMPTY(&td->td_contested)) {
1170 				print_lockchain(td, " ");
1184 print_sleepchain(struct thread *td, const char *prefix)
1193 		db_printf("%sthread %d (pid %d, %s) ", prefix, td->td_tid,
1194 		    td->td_proc->p_pid, td->td_name);
1195 		switch (td->td_state) {
1206 			db_printf("running on CPU %d\n", td->td_oncpu);
1209 			if (TD_ON_SLEEPQ(td)) {
1210 				if (lockmgr_chain(td, &owner) ||
1211 				    sx_chain(td, &owner)) {
1214 					td = owner;
1218 				    td->td_wchan, td->td_wmesg);
1224 			db_printf("??? (%#x)\n", td->td_state);
1232 	struct thread *td;
1236 		td = db_lookup_thread(addr, true);
1238 		td = kdb_thread;
1240 	print_sleepchain(td, "");
1246 print_waiter(struct thread *td, int indent)
1255 	print_thread(td, "thread ");
1256 	LIST_FOREACH(ts, &td->td_contested, ts_link)
1265 	struct thread *td;
1275 	TAILQ_FOREACH(td, &ts->ts_blocked[TS_EXCLUSIVE_QUEUE], td_lockq)
1276 		print_waiter(td, indent + 1);
1277 	TAILQ_FOREACH(td, &ts->ts_blocked[TS_SHARED_QUEUE], td_lockq)
1278 		print_waiter(td, indent + 1);
1279 	TAILQ_FOREACH(td, &ts->ts_pending, td_lockq)
1280 		print_waiter(td, indent + 1);