Lines Matching defs:bfqg
96 /* bfqg stats flags */
138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
141 struct bfqg_stats *stats = &bfqg->stats;
145 if (bfqg == curr_bfqg)
166 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)
168 bfq_stat_add(&bfqg->stats.dequeue, 1);
171 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
173 struct bfqg_stats *stats = &bfqg->stats;
190 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
192 struct bfqg_stats *stats = &bfqg->stats;
204 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)
206 struct bfqg_stats *stats = &bfqg->stats;
212 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)
214 struct bfqg_stats *stats = &bfqg->stats;
222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
225 blkg_rwstat_add(&bfqg->stats.queued, opf, 1);
226 bfqg_stats_end_empty_time(&bfqg->stats);
227 if (!(bfqq == bfqg->bfqd->in_service_queue))
228 bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
231 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf)
233 blkg_rwstat_add(&bfqg->stats.queued, opf, -1);
236 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf)
238 blkg_rwstat_add(&bfqg->stats.merged, opf, 1);
241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
244 struct bfqg_stats *stats = &bfqg->stats;
257 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { }
258 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { }
259 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
261 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }
262 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }
279 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)
281 return pd_to_blkg(&bfqg->pd);
296 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)
298 struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;
317 static void bfqg_get(struct bfq_group *bfqg)
319 refcount_inc(&bfqg->ref);
322 static void bfqg_put(struct bfq_group *bfqg)
324 if (refcount_dec_and_test(&bfqg->ref))
325 kfree(bfqg);
328 static void bfqg_and_blkg_get(struct bfq_group *bfqg)
330 /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
331 bfqg_get(bfqg);
333 blkg_get(bfqg_to_blkg(bfqg));
336 void bfqg_and_blkg_put(struct bfq_group *bfqg)
338 blkg_put(bfqg_to_blkg(bfqg));
340 bfqg_put(bfqg);
345 struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);
347 if (!bfqg)
350 blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));
351 blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);
395 * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
396 * recursive stats can still account for the amount used by this bfqg after
399 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)
403 if (!bfqg) /* root_group */
406 parent = bfqg_parent(bfqg);
408 lockdep_assert_held(&bfqg_to_blkg(bfqg)->q->queue_lock);
413 bfqg_stats_add_aux(&parent->stats, &bfqg->stats);
414 bfqg_stats_reset(&bfqg->stats);
417 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
427 * Make sure that bfqg and its associated blkg do not
430 bfqg_and_blkg_get(bfqg);
432 entity->parent = bfqg->my_entity; /* NULL for root group */
433 entity->sched_data = &bfqg->sched_data;
513 struct bfq_group *bfqg;
515 bfqg = kzalloc_node(sizeof(*bfqg), gfp, disk->node_id);
516 if (!bfqg)
519 if (bfqg_stats_init(&bfqg->stats, gfp)) {
520 kfree(bfqg);
525 refcount_set(&bfqg->ref, 1);
526 return &bfqg->pd;
532 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
534 struct bfq_entity *entity = &bfqg->entity;
538 entity->my_sched_data = &bfqg->sched_data;
541 bfqg->my_entity = entity; /*
545 bfqg->bfqd = bfqd;
546 bfqg->active_entities = 0;
547 bfqg->num_queues_with_pending_reqs = 0;
548 bfqg->rq_pos_tree = RB_ROOT;
553 struct bfq_group *bfqg = pd_to_bfqg(pd);
555 bfqg_stats_exit(&bfqg->stats);
556 bfqg_put(bfqg);
561 struct bfq_group *bfqg = pd_to_bfqg(pd);
563 bfqg_stats_reset(&bfqg->stats);
566 static void bfq_group_set_parent(struct bfq_group *bfqg,
571 entity = &bfqg->entity;
576 static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)
586 entity = &bfqg->entity;
602 struct bfq_group *bfqg;
609 bfqg = blkg_to_bfqg(blkg);
610 if (bfqg->pd.online) {
612 return bfqg;
622 * bfq_bfqq_move - migrate @bfqq to @bfqg.
625 * @bfqg: the group to move to.
627 * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
631 * owning @bfqg does not disappear (see comments in
636 struct bfq_group *bfqg)
646 if (old_parent == bfqg)
688 entity->parent = bfqg->my_entity;
689 entity->sched_data = &bfqg->sched_data;
690 /* pin down bfqg and its associated blkg */
691 bfqg_and_blkg_get(bfqg);
711 struct bfq_group *bfqg,
718 if (sync_bfqq->entity.sched_data != &bfqg->sched_data)
719 bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
729 if (bfqq->entity.sched_data != &bfqg->sched_data)
749 * __bfq_bic_change_cgroup - move @bic to @bfqg.
752 * @bfqg: the group to move to.
760 struct bfq_group *bfqg)
769 async_bfqq->entity.sched_data != &bfqg->sched_data) {
775 bfq_sync_bfqq_move(bfqd, sync_bfqq, bic, bfqg, act_idx);
782 struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);
785 serial_nr = bfqg_to_blkg(bfqg)->blkcg->css.serial_nr;
798 bfq_link_bfqg(bfqd, bfqg);
799 __bfq_bic_change_cgroup(bfqd, bic, bfqg);
829 * Thanks to the last fact, and to the fact that: (1) bfqg has
833 * field bfqg->pd) to get to the blkg associated with bfqg,
836 * bfqg may cause dangling references to be traversed, as
837 * bfqg->pd may not exist any more.
839 * In view of the above facts, here we cache, in the bfqg, any
844 * Finally, note that bfqg itself needs to be protected from
847 * refcounter for bfqg, to let it disappear only after no
850 blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));
899 * @bfqg: the group to move from.
904 struct bfq_group *bfqg,
914 if (bfqg->sched_data.in_service_entity)
916 bfqg->sched_data.in_service_entity,
931 struct bfq_group *bfqg = pd_to_bfqg(pd);
932 struct bfq_data *bfqd = bfqg->bfqd;
933 struct bfq_entity *entity = bfqg->my_entity;
947 st = bfqg->sched_data.service_tree + i;
961 bfq_reparent_active_queues(bfqd, bfqg, st, i);
980 bfq_put_async_queues(bfqd, bfqg);
989 bfqg_stats_xfer_dead(bfqg);
997 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
999 bfq_end_wr_async_queues(bfqd, bfqg);
1021 struct bfq_group *bfqg = pd_to_bfqg(pd);
1023 if (!bfqg->entity.dev_weight)
1025 return __blkg_prfill_u64(sf, pd, bfqg->entity.dev_weight);
1039 static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)
1043 bfqg->entity.dev_weight = dev_weight;
1050 if ((unsigned short)weight != bfqg->entity.new_weight) {
1051 bfqg->entity.new_weight = (unsigned short)weight;
1054 * stored in bfqg->entity.new_weight before
1061 * depend also on bfqg->entity.new_weight (in
1064 * of bfqg->entity.new_weight is correctly
1068 bfqg->entity.prio_changed = 1;
1088 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
1090 if (bfqg)
1091 bfq_group_set_weight(bfqg, val, 0);
1105 struct bfq_group *bfqg;
1126 bfqg = blkg_to_bfqg(ctx.blkg);
1130 bfq_group_set_weight(bfqg, bfqg->entity.weight, v);
1226 struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);
1227 u64 sum = blkg_rwstat_total(&bfqg->stats.bytes);
1262 struct bfq_group *bfqg = pd_to_bfqg(pd);
1263 u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);
1267 v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);
1323 /* statistics, covers only the tasks in the bfqg */
1366 /* the same statistics which cover the bfqg and its descendants */
1448 struct bfq_group *bfqg) {}
1450 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
1460 entity->sched_data = &bfqg->sched_data;
1480 void bfqg_and_blkg_put(struct bfq_group *bfqg) {}
1484 struct bfq_group *bfqg;
1487 bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);
1488 if (!bfqg)
1492 bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
1494 return bfqg;