sys/kern/kern_timeout.c

177 #define	cc_exec_curr(cc, dir)		cc->cc_exec_entity[dir].cc_curr
178 #define	cc_exec_drain(cc, dir)		cc->cc_exec_entity[dir].cc_drain
179 #define	cc_exec_next(cc)		cc->cc_next
180 #define	cc_exec_cancel(cc, dir)		cc->cc_exec_entity[dir].cc_cancel
181 #define	cc_exec_waiting(cc, dir)	cc->cc_exec_entity[dir].cc_waiting
183 #define	cc_migration_func(cc, dir)	cc->cc_exec_entity[dir].ce_migration_func
184 #define	cc_migration_arg(cc, dir)	cc->cc_exec_entity[dir].ce_migration_arg
185 #define	cc_migration_cpu(cc, dir)	cc->cc_exec_entity[dir].ce_migration_cpu
186 #define	cc_migration_time(cc, dir)	cc->cc_exec_entity[dir].ce_migration_time
187 #define	cc_migration_prec(cc, dir)	cc->cc_exec_entity[dir].ce_migration_prec
198 #define	CC_LOCK(cc)	mtx_lock_spin(&(cc)->cc_lock)
199 #define	CC_UNLOCK(cc)	mtx_unlock_spin(&(cc)->cc_lock)
200 #define	CC_LOCK_ASSERT(cc)	mtx_assert(&(cc)->cc_lock, MA_OWNED)
204 static void	callout_cpu_init(struct callout_cpu *cc, int cpu);
205 static void	softclock_call_cc(struct callout *c, struct callout_cpu *cc,
234 cc_cce_cleanup(struct callout_cpu *cc, int direct)
237 	cc_exec_curr(cc, direct) = NULL;
238 	cc_exec_cancel(cc, direct) = false;
239 	cc_exec_waiting(cc, direct) = false;
241 	cc_migration_cpu(cc, direct) = CPUBLOCK;
242 	cc_migration_time(cc, direct) = 0;
243 	cc_migration_prec(cc, direct) = 0;
244 	cc_migration_func(cc, direct) = NULL;
245 	cc_migration_arg(cc, direct) = NULL;
253 cc_cce_migrating(struct callout_cpu *cc, int direct)
257 	return (cc_migration_cpu(cc, direct) != CPUBLOCK);
270 	struct callout_cpu *cc;
302 	cc = CC_CPU(timeout_cpu);
303 	cc->cc_callout = malloc(ncallout * sizeof(struct callout),
305 	callout_cpu_init(cc, timeout_cpu);
313 callout_cpu_init(struct callout_cpu *cc, int cpu)
318 	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);
319 	SLIST_INIT(&cc->cc_callfree);
320 	cc->cc_inited = 1;
321 	cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize,
324 		LIST_INIT(&cc->cc_callwheel[i]);
325 	TAILQ_INIT(&cc->cc_expireq);
326 	cc->cc_firstevent = SBT_MAX;
328 		cc_cce_cleanup(cc, i);
329 	snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),
331 	if (cc->cc_callout == NULL)	/* Only cpu0 handles timeout(9) */
334 		c = &cc->cc_callout[i];
337 		SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
348 callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu)
352 	MPASS(c != NULL && cc != NULL);
353 	CC_LOCK_ASSERT(cc);
362 	CC_UNLOCK(cc);
377 	struct callout_cpu *cc;
384 	cc = CC_CPU(timeout_cpu);
386 	if (swi_add(&clk_intr_event, name, softclock, cc, SWI_CLOCK,
387 	    INTR_MPSAFE, &cc->cc_cookie))
400 		cc = CC_CPU(cpu);
401 		cc->cc_callout = NULL;	/* Only cpu0 handles timeout(9). */
402 		callout_cpu_init(cc, cpu);
405 		if (swi_add(&ie, name, softclock, cc, SWI_CLOCK,
406 		    INTR_MPSAFE, &cc->cc_cookie))
439 	struct callout_cpu *cc;
448 	cc = CC_SELF();
449 	mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);
452 	firstb = callout_hash(cc->cc_lastscan);
453 	cc->cc_lastscan = now;
482 		sc = &cc->cc_callwheel[firstb & callwheelmask];
495 					cc_exec_next(cc) =
497 					cc->cc_bucket = firstb & callwheelmask;
499 					softclock_call_cc(tmp, cc,
504 					tmp = cc_exec_next(cc);
505 					cc_exec_next(cc) = NULL;
509 					TAILQ_INSERT_TAIL(&cc->cc_expireq,
544 	cc->cc_firstevent = last;
553 	mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
558 	if (!TAILQ_EMPTY(&cc->cc_expireq))
559 		swi_sched(cc->cc_cookie, 0);
565 	struct callout_cpu *cc;
577 		cc = CC_CPU(cpu);
578 		CC_LOCK(cc);
581 		CC_UNLOCK(cc);
583 	return (cc);
587 callout_cc_add(struct callout *c, struct callout_cpu *cc,
593 	CC_LOCK_ASSERT(cc);
594 	if (sbt < cc->cc_lastscan)
595 		sbt = cc->cc_lastscan;
609 	LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le);
610 	if (cc->cc_bucket == bucket)
611 		cc_exec_next(cc) = c;
620 	if (sbt < cc->cc_firstevent) {
621 		cc->cc_firstevent = sbt;
628 callout_cc_del(struct callout *c, struct callout_cpu *cc)
634 	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
638 softclock_call_cc(struct callout *c, struct callout_cpu *cc,
686 	cc_exec_curr(cc, direct) = c;
687 	cc_exec_cancel(cc, direct) = false;
688 	cc_exec_drain(cc, direct) = NULL;
689 	CC_UNLOCK(cc);
696 		if (cc_exec_cancel(cc, direct)) {
701 		cc_exec_cancel(cc, direct) = true;
722 	KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",
746 	KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");
751 	CC_LOCK(cc);
752 	KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr"));
753 	cc_exec_curr(cc, direct) = NULL;
754 	if (cc_exec_drain(cc, direct)) {
755 		drain = cc_exec_drain(cc, direct);
756 		cc_exec_drain(cc, direct) = NULL;
757 		CC_UNLOCK(cc);
759 		CC_LOCK(cc);
761 	if (cc_exec_waiting(cc, direct)) {
768 		if (cc_cce_migrating(cc, direct)) {
769 			cc_cce_cleanup(cc, direct);
777 		cc_exec_waiting(cc, direct) = false;
778 		CC_UNLOCK(cc);
779 		wakeup(&cc_exec_waiting(cc, direct));
780 		CC_LOCK(cc);
781 	} else if (cc_cce_migrating(cc, direct)) {
789 		new_cpu = cc_migration_cpu(cc, direct);
790 		new_time = cc_migration_time(cc, direct);
791 		new_prec = cc_migration_prec(cc, direct);
792 		new_func = cc_migration_func(cc, direct);
793 		new_arg = cc_migration_arg(cc, direct);
794 		cc_cce_cleanup(cc, direct);
806 			callout_cc_del(c, cc);
811 		new_cc = callout_cpu_switch(c, cc, new_cpu);
816 		CC_LOCK(cc);
833 		callout_cc_del(c, cc);
855 	struct callout_cpu *cc;
861 	cc = (struct callout_cpu *)arg;
862 	CC_LOCK(cc);
863 	while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {
864 		TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
865 		softclock_call_cc(c, cc,
880 	CC_UNLOCK(cc);
902 	struct callout_cpu *cc;
906 	cc = CC_CPU(timeout_cpu);
907 	CC_LOCK(cc);
909 	new = SLIST_FIRST(&cc->cc_callfree);
913 	SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);
916 	CC_UNLOCK(cc);
924 	struct callout_cpu *cc;
934 	cc = callout_lock(handle.callout);
937 	CC_UNLOCK(cc);
1019 	struct callout_cpu *cc;
1045 	cc = callout_lock(c);
1056 	if (cc_exec_curr(cc, direct) == c) {
1062 		if (c->c_lock != NULL && !cc_exec_cancel(cc, direct))
1063 			cancelled = cc_exec_cancel(cc, direct) = true;
1064 		if (cc_exec_waiting(cc, direct)) {
1072 			CC_UNLOCK(cc);
1084 			cc_migration_cpu(cc, direct) = cpu;
1085 			cc_migration_time(cc, direct) = to_sbt;
1086 			cc_migration_prec(cc, direct) = precision;
1087 			cc_migration_func(cc, direct) = ftn;
1088 			cc_migration_arg(cc, direct) = arg;
1090 			CC_UNLOCK(cc);
1097 			if (cc_exec_next(cc) == c)
1098 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1101 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1115 		if (cc_exec_curr(cc, direct) == c) {
1132 			cc_migration_cpu(cc, direct) = cpu;
1133 			cc_migration_time(cc, direct) = to_sbt;
1134 			cc_migration_prec(cc, direct) = precision;
1135 			cc_migration_func(cc, direct) = ftn;
1136 			cc_migration_arg(cc, direct) = arg;
1143 			CC_UNLOCK(cc);
1146 		cc = callout_cpu_switch(c, cc, cpu);
1150 	callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags);
1154 	CC_UNLOCK(cc);
1177 	struct callout_cpu *cc, *old_cc;
1208 	cc = callout_lock(c);
1237 	if (sq_locked != 0 && cc != old_cc) {
1239 		CC_UNLOCK(cc);
1252 	if (cc_exec_curr(cc, direct) == c) {
1269 			while (cc_exec_curr(cc, direct) == c) {
1288 					CC_UNLOCK(cc);
1290 					    &cc_exec_waiting(cc, direct));
1292 					old_cc = cc;
1302 				cc_exec_waiting(cc, direct) = true;
1304 				CC_UNLOCK(cc);
1306 				    &cc_exec_waiting(cc, direct),
1307 				    &cc->cc_lock.lock_object, "codrain",
1310 				    &cc_exec_waiting(cc, direct),
1317 				CC_LOCK(cc);
1321 			   !cc_exec_cancel(cc, direct) && (drain == NULL)) {
1332 			cc_exec_cancel(cc, direct) = true;
1335 			KASSERT(!cc_cce_migrating(cc, direct),
1340 				cc_migration_cpu(cc, direct) = CPUBLOCK;
1341 				cc_migration_time(cc, direct) = 0;
1342 				cc_migration_prec(cc, direct) = 0;
1343 				cc_migration_func(cc, direct) = NULL;
1344 				cc_migration_arg(cc, direct) = NULL;
1347 			CC_UNLOCK(cc);
1368 			cc_migration_cpu(cc, direct) = CPUBLOCK;
1369 			cc_migration_time(cc, direct) = 0;
1370 			cc_migration_prec(cc, direct) = 0;
1371 			cc_migration_func(cc, direct) = NULL;
1372 			cc_migration_arg(cc, direct) = NULL;
1377 				cc_exec_drain(cc, direct) = drain;
1379 			CC_UNLOCK(cc);
1385 			cc_exec_drain(cc, direct) = drain;
1393 		sleepq_release(&cc_exec_waiting(cc, direct));
1402 		if (cc_exec_curr(cc, direct) != c)
1404 		CC_UNLOCK(cc);
1415 			if (cc_exec_next(cc) == c)
1416 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1419 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1422 	callout_cc_del(c, cc);
1423 	CC_UNLOCK(cc);
1505 	CC_LOCK(cc);
1516 	CC_UNLOCK(cc);
1541 	struct callout_cpu *cc;
1562 		cc = CC_CPU(cpu);
1564 		cc = CC_CPU(timeout_cpu);
1566 		CC_LOCK(cc);
1568 			sc = &cc->cc_callwheel[i];
1589 		CC_UNLOCK(cc);