linux-master/block/bfq-iosched.c

379 struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync,
383 		return bic->bfqq[1][actuator_idx];
385 	return bic->bfqq[0][actuator_idx];
390 void bic_set_bfqq(struct bfq_io_cq *bic,
395 	struct bfq_queue *old_bfqq = bic->bfqq[is_sync][actuator_idx];
399 	 * bic->bfqq and bfqq is happening here. This causes troubles
400 	 * in the following case: bic->bfqq has also been scheduled
401 	 * for a possible stable merge with bic->stable_merge_bfqq,
402 	 * and bic->stable_merge_bfqq == bfqq happens to
405 	 * bic->stable_merge_bfqq points exactly to bfqq, then bfqq
408 	 * bic->stable_merge_bfqq == bfqq.
410 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[actuator_idx];
412 	/* Clear bic pointer if bfqq is detached from this bic */
413 	if (old_bfqq && old_bfqq->bic == bic)
414 		old_bfqq->bic = NULL;
417 		bic->bfqq[1][actuator_idx] = bfqq;
419 		bic->bfqq[0][actuator_idx] = bfqq;
436 struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)
438 	return bic->icq.q->elevator->elevator_data;
447 	/* bic->icq is the first member, %NULL will convert to %NULL */
452  * bfq_bic_lookup - search into @ioc a bic associated to @bfqd.
693 	struct bfq_io_cq *bic = bfq_bic_lookup(data->q);
706 	for (act_idx = 0; bic && act_idx < bfqd->num_actuators; act_idx++) {
708 			bic_to_bfqq(bic, op_is_sync(opf), act_idx);
1134 		      struct bfq_io_cq *bic, bool bfq_already_existing)
1139 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
1837 		bfqq->bic || RQ_BIC(rq)->bfqq_data[act_idx].stably_merged;
2453 	 * queue_lock inside the bfqd->lock. We assume that the bic
2457 	struct bfq_io_cq *bic = bfq_bic_lookup(q);
2462 	if (bic) {
2467 		bfq_bic_update_cgroup(bic, bio);
2469 		bfqd->bio_bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf),
2474 	bfqd->bio_bic = bic;
2797 	 * not available any more (new_bfqq->bic == NULL).
2869 		if (new_bfqq->bic) {
2872 				&new_bfqq->bic->bfqq_data[new_a_idx];
2907 		     void *io_struct, bool request, struct bfq_io_cq *bic)
2911 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
2920 	 * currently merged with some other queue (i.e., bfqq->bic
2923 	 * merged with bic->stable_merge_bfqq. But this would be
2929 		 * bic->stable_merge_bfqq may point to some queue (for
2930 		 * stable merging) also if bic is associated with a
3038 	struct bfq_io_cq *bic = bfqq->bic;
3040 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];
3043 	 * If !bfqq->bic, the queue is already shared or its requests
3047 	if (!bic)
3129 bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
3195 	 * Merge queues (that is, let bic redirect its requests to new_bfqq)
3197 	bic_set_bfqq(bic, new_bfqq, true, bfqq->actuator_idx);
3201 	 * set new_bfqq->bic to NULL. bfqq either:
3202 	 * - does not belong to any bic any more, and hence bfqq->bic must
3206 	 *   any bic soon and bfqq->bic is already NULL (therefore the next
3209 	new_bfqq->bic = NULL;
3220 	bfqq->bic = NULL;
3254 		 * bic still points to bfqq, then it has not yet been
3257 		 * fulfilled, i.e., bic can be redirected to new_bfqq
4422 	/* mark bfqq as waiting a request only if a bic still points to it */
4902 		if (bfqq->bic && bfqq->bic->bfqq[0][act_idx] &&
4903 		    bfq_bfqq_busy(bfqq->bic->bfqq[0][act_idx]) &&
4904 		    bfqq->bic->bfqq[0][act_idx]->next_rq)
4905 			async_bfqq = bfqq->bic->bfqq[0][act_idx];
4987 		    icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic &&
5373 		if (bfqq->bic && bfqq->bfqd->burst_size > 0)
5451 static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync,
5454 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, actuator_idx);
5461 		bic_set_bfqq(bic, NULL, is_sync, actuator_idx);
5468 	struct bfq_io_cq *bic = icq_to_bic(icq);
5469 	struct bfq_data *bfqd = bic_to_bfqd(bic);
5475 	 * next loop. We rely on bic being zeroed on creation, and
5479 	struct bfq_iocq_bfqq_data *bfqq_data = bic->bfqq_data;
5494 		bfq_exit_icq_bfqq(bic, true, act_idx);
5495 		bfq_exit_icq_bfqq(bic, false, act_idx);
5507 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
5516 	ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);
5531 		bfqq->new_ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio);
5535 		bfqq->new_ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio);
5558 				       struct bfq_io_cq *bic,
5561 static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio)
5563 	struct bfq_data *bfqd = bic_to_bfqd(bic);
5565 	int ioprio = bic->icq.ioc->ioprio;
5568 	 * This condition may trigger on a newly created bic, be sure to
5571 	if (unlikely(!bfqd) || likely(bic->ioprio == ioprio))
5574 	bic->ioprio = ioprio;
5576 	bfqq = bic_to_bfqq(bic, false, bfq_actuator_index(bfqd, bio));
5580 		bfqq = bfq_get_queue(bfqd, bio, false, bic, true);
5581 		bic_set_bfqq(bic, bfqq, false, bfq_actuator_index(bfqd, bio));
5585 	bfqq = bic_to_bfqq(bic, true, bfq_actuator_index(bfqd, bio));
5587 		bfq_set_next_ioprio_data(bfqq, bic);
5591 			  struct bfq_io_cq *bic, pid_t pid, int is_sync,
5606 	if (bic)
5607 		bfq_set_next_ioprio_data(bfqq, bic);
5681 			  struct bfq_io_cq *bic,
5691 	if (new_bfqq->bic)
5692 		new_bfqq->bic->bfqq_data[a_idx].stably_merged = true;
5693 	bic->bfqq_data[a_idx].stably_merged = true;
5697 	 * bfqq->bic must be set too, for
5701 	bfqq->bic = bic;
5702 	bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq);
5755 						      struct bfq_io_cq *bic)
5806 							 bic,
5823 			bic->bfqq_data[last_bfqq_created->actuator_idx].stable_merge_bfqq =
5834 				       struct bfq_io_cq *bic,
5837 	const int ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio);
5838 	const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);
5858 		bfq_init_bfqq(bfqd, bfqq, bic, current->pid,
5889 		bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic);
5948 				       struct bfq_io_cq *bic)
5971 	if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||
6167 		 * If the bic associated with the process
6715  * that bic points to the bfq iocontext of the task.
6720 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)
6731 	bic_set_bfqq(bic, NULL, true, bfqq->actuator_idx);
6740 						   struct bfq_io_cq *bic,
6746 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx);
6747 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[act_idx];
6757 	bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split);
6759 	bic_set_bfqq(bic, bfqq, is_sync, act_idx);
6817 	 * previously allocated bic/bfqq structs.
6850 	struct bfq_io_cq *bic;
6870 	bic = icq_to_bic(rq->elv.icq);
6872 	bfq_check_ioprio_change(bic, bio);
6874 	bfq_bic_update_cgroup(bic, bio);
6876 	bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,
6882 			!bic->bfqq_data[a_idx].stably_merged) {
6885 			/* Update bic before losing reference to bfqq */
6887 				bic->bfqq_data[a_idx].saved_in_large_burst =
6890 			bfqq = bfq_split_bfqq(bic, bfqq);
6894 				bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,
6922 	bic->requests++;
6926 	rq->elv.priv[0] = bic;
6931 	 * by only this bic: we can then set bfqq->bic = bic. in
6936 		bfqq->bic = bic;
6943 			bfq_bfqq_resume_state(bfqq, bfqd, bic,