kernel/sched/rt.c

167 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
169 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
172 	WARN_ON_ONCE(!rt_entity_is_task(rt_se));
174 	return container_of(rt_se, struct task_struct, rt);
182 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
184 	return rt_se->rt_rq;
187 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
189 	struct rt_rq *rt_rq = rt_se->rt_rq;
196 	if (tg->rt_se)
208 		if (tg->rt_se)
209 			kfree(tg->rt_se[i]);
213 	kfree(tg->rt_se);
217 		struct sched_rt_entity *rt_se, int cpu,
228 	tg->rt_se[cpu] = rt_se;
230 	if (!rt_se)
234 		rt_se->rt_rq = &rq->rt;
236 		rt_se->rt_rq = parent->my_q;
238 	rt_se->my_q = rt_rq;
239 	rt_se->parent = parent;
240 	INIT_LIST_HEAD(&rt_se->run_list);
246 	struct sched_rt_entity *rt_se;
252 	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);
253 	if (!tg->rt_se)
265 		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
267 		if (!rt_se)
272 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
285 #define rt_entity_is_task(rt_se) (1)
287 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
289 	return container_of(rt_se, struct task_struct, rt);
297 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
299 	struct task_struct *p = rt_task_of(rt_se);
304 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
306 	struct rq *rq = rq_of_rt_se(rt_se);
440 static inline int on_rt_rq(struct sched_rt_entity *rt_se)
442 	return rt_se->on_rq;
519 #define for_each_sched_rt_entity(rt_se) \
520 	for (; rt_se; rt_se = rt_se->parent)
522 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
524 	return rt_se->my_q;
527 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
528 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
534 	struct sched_rt_entity *rt_se;
538 	rt_se = rt_rq->tg->rt_se[cpu];
541 		if (!rt_se)
543 		else if (!on_rt_rq(rt_se))
544 			enqueue_rt_entity(rt_se, 0);
553 	struct sched_rt_entity *rt_se;
556 	rt_se = rt_rq->tg->rt_se[cpu];
558 	if (!rt_se) {
563 	else if (on_rt_rq(rt_se))
564 		dequeue_rt_entity(rt_se, 0);
572 static int rt_se_boosted(struct sched_rt_entity *rt_se)
574 	struct rt_rq *rt_rq = group_rt_rq(rt_se);
580 	p = rt_task_of(rt_se);
624 #define for_each_sched_rt_entity(rt_se) \
625 	for (; rt_se; rt_se = NULL)
627 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
941 static inline int rt_se_prio(struct sched_rt_entity *rt_se)
944 	struct rt_rq *rt_rq = group_rt_rq(rt_se);
950 	return rt_task_of(rt_se)->prio;
1003 	struct sched_rt_entity *rt_se = &curr->rt;
1016 	for_each_sched_rt_entity(rt_se) {
1017 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1164 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1166 	if (rt_se_boosted(rt_se))
1174 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1176 	if (rt_se_boosted(rt_se))
1185 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1191 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
1196 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
1198 	struct rt_rq *group_rq = group_rt_rq(rt_se);
1207 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
1209 	struct rt_rq *group_rq = group_rt_rq(rt_se);
1215 	tsk = rt_task_of(rt_se);
1221 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1223 	int prio = rt_se_prio(rt_se);
1226 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
1227 	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);
1230 	inc_rt_group(rt_se, rt_rq);
1234 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1236 	WARN_ON(!rt_prio(rt_se_prio(rt_se)));
1238 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
1239 	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);
1241 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));
1242 	dec_rt_group(rt_se, rt_rq);
1246  * Change rt_se->run_list location unless SAVE && !MOVE
1258 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)
1260 	list_del_init(&rt_se->run_list);
1262 	if (list_empty(array->queue + rt_se_prio(rt_se)))
1263 		__clear_bit(rt_se_prio(rt_se), array->bitmap);
1265 	rt_se->on_list = 0;
1269 __schedstats_from_rt_se(struct sched_rt_entity *rt_se)
1273 	if (!rt_entity_is_task(rt_se))
1277 	return &rt_task_of(rt_se)->stats;
1281 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1289 	if (rt_entity_is_task(rt_se))
1290 		p = rt_task_of(rt_se);
1292 	stats = __schedstats_from_rt_se(rt_se);
1300 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1308 	if (rt_entity_is_task(rt_se))
1309 		p = rt_task_of(rt_se);
1311 	stats = __schedstats_from_rt_se(rt_se);
1319 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
1326 		update_stats_enqueue_sleeper_rt(rt_rq, rt_se);
1330 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1338 	if (rt_entity_is_task(rt_se))
1339 		p = rt_task_of(rt_se);
1341 	stats = __schedstats_from_rt_se(rt_se);
1349 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
1357 	if (rt_entity_is_task(rt_se))
1358 		p = rt_task_of(rt_se);
1374 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1376 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1378 	struct rt_rq *group_rq = group_rt_rq(rt_se);
1379 	struct list_head *queue = array->queue + rt_se_prio(rt_se);
1388 		if (rt_se->on_list)
1389 			__delist_rt_entity(rt_se, array);
1394 		WARN_ON_ONCE(rt_se->on_list);
1396 			list_add(&rt_se->run_list, queue);
1398 			list_add_tail(&rt_se->run_list, queue);
1400 		__set_bit(rt_se_prio(rt_se), array->bitmap);
1401 		rt_se->on_list = 1;
1403 	rt_se->on_rq = 1;
1405 	inc_rt_tasks(rt_se, rt_rq);
1408 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1410 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1414 		WARN_ON_ONCE(!rt_se->on_list);
1415 		__delist_rt_entity(rt_se, array);
1417 	rt_se->on_rq = 0;
1419 	dec_rt_tasks(rt_se, rt_rq);
1426 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
1431 	for_each_sched_rt_entity(rt_se) {
1432 		rt_se->back = back;
1433 		back = rt_se;
1438 	for (rt_se = back; rt_se; rt_se = rt_se->back) {
1439 		if (on_rt_rq(rt_se))
1440 			__dequeue_rt_entity(rt_se, flags);
1446 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1448 	struct rq *rq = rq_of_rt_se(rt_se);
1450 	update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1452 	dequeue_rt_stack(rt_se, flags);
1453 	for_each_sched_rt_entity(rt_se)
1454 		__enqueue_rt_entity(rt_se, flags);
1458 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1460 	struct rq *rq = rq_of_rt_se(rt_se);
1462 	update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1464 	dequeue_rt_stack(rt_se, flags);
1466 	for_each_sched_rt_entity(rt_se) {
1467 		struct rt_rq *rt_rq = group_rt_rq(rt_se);
1470 			__enqueue_rt_entity(rt_se, flags);
1481 	struct sched_rt_entity *rt_se = &p->rt;
1484 		rt_se->timeout = 0;
1487 	update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se);
1489 	enqueue_rt_entity(rt_se, flags);
1497 	struct sched_rt_entity *rt_se = &p->rt;
1500 	dequeue_rt_entity(rt_se, flags);
1510 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
1512 	if (on_rt_rq(rt_se)) {
1514 		struct list_head *queue = array->queue + rt_se_prio(rt_se);
1517 			list_move(&rt_se->run_list, queue);
1519 			list_move_tail(&rt_se->run_list, queue);
1525 	struct sched_rt_entity *rt_se = &p->rt;
1528 	for_each_sched_rt_entity(rt_se) {
1529 		rt_rq = rt_rq_of_se(rt_se);
1530 		requeue_rt_entity(rt_rq, rt_se, head);
1689 	struct sched_rt_entity *rt_se = &p->rt;
1694 		update_stats_wait_end_rt(rt_rq, rt_se);
1733 	struct sched_rt_entity *rt_se;
1737 		rt_se = pick_next_rt_entity(rt_rq);
1738 		if (unlikely(!rt_se))
1740 		rt_rq = group_rt_rq(rt_se);
1743 	return rt_task_of(rt_se);
1770 	struct sched_rt_entity *rt_se = &p->rt;
1774 		update_stats_wait_start_rt(rt_rq, rt_se);
2589 	struct sched_rt_entity *rt_se = &p->rt;
2612 	for_each_sched_rt_entity(rt_se) {
2613 		if (rt_se->run_list.prev != rt_se->run_list.next) {