linux-master/block/mq-deadline.c

116 deadline_rb_root(struct dd_per_prio *per_prio, struct request *rq)
118 	return &per_prio->sort_list[rq_data_dir(rq)];
125 static u8 dd_rq_ioclass(struct request *rq)
127 	return IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
137 	struct request *rq, *res = NULL;
142 	rq = rb_entry_rq(node);
144 		rq = rb_entry_rq(node);
145 		if (blk_rq_pos(rq) >= pos) {
146 			res = rq;
156 deadline_add_rq_rb(struct dd_per_prio *per_prio, struct request *rq)
158 	struct rb_root *root = deadline_rb_root(per_prio, rq);
160 	elv_rb_add(root, rq);
164 deadline_del_rq_rb(struct dd_per_prio *per_prio, struct request *rq)
166 	elv_rb_del(deadline_rb_root(per_prio, rq), rq);
170  * remove rq from rbtree and fifo.
174 				    struct request *rq)
176 	list_del_init(&rq->queuelist);
181 	if (!RB_EMPTY_NODE(&rq->rb_node))
182 		deadline_del_rq_rb(per_prio, rq);
184 	elv_rqhash_del(q, rq);
185 	if (q->last_merge == rq)
221 	 * if next expires before rq, assign its expire time to rq
243 		      struct request *rq)
248 	deadline_remove_request(rq->q, per_prio, rq);
268 	struct request *rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);
270 	return time_is_before_eq_jiffies((unsigned long)rq->fifo_time);
300  * Returns true if and only if @rq started after @latest_start where
303 static bool started_after(struct deadline_data *dd, struct request *rq,
306 	unsigned long start_time = (unsigned long)rq->fifo_time;
308 	start_time -= dd->fifo_expire[rq_data_dir(rq)];
321 	struct request *rq, *next_rq;
329 		rq = list_first_entry(&per_prio->dispatch, struct request,
331 		if (started_after(dd, rq, latest_start))
333 		list_del_init(&rq->queuelist);
334 		data_dir = rq_data_dir(rq);
341 	rq = deadline_next_request(dd, per_prio, dd->last_dir);
342 	if (rq && dd->batching < dd->fifo_batch) {
344 		data_dir = rq_data_dir(rq);
393 		rq = deadline_fifo_request(dd, per_prio, data_dir);
399 		rq = next_rq;
402 	if (!rq)
409 	if (started_after(dd, rq, latest_start))
413 	 * rq is the selected appropriate request.
416 	deadline_move_request(dd, per_prio, rq);
418 	ioprio_class = dd_rq_ioclass(rq);
420 	dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq);
422 	rq->rq_flags |= RQF_STARTED;
423 	return rq;
433 	struct request *rq;
445 		rq = __dd_dispatch_request(dd, &dd->per_prio[prio],
447 		if (rq)
448 			return rq;
466 	struct request *rq;
470 	rq = dd_dispatch_prio_aged_requests(dd, now);
471 	if (rq)
479 		rq = __dd_dispatch_request(dd, &dd->per_prio[prio], now);
480 		if (rq || dd_queued(dd, prio))
487 	return rq;
605  * an existing request, store the pointer to that request into *@rq.
607 static int dd_request_merge(struct request_queue *q, struct request **rq,
625 			*rq = __rq;
657  * add rq to rbtree and fifo
659 static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
664 	const enum dd_data_dir data_dir = rq_data_dir(rq);
665 	u16 ioprio = req_get_ioprio(rq);
674 	if (!rq->elv.priv[0]) {
676 		rq->elv.priv[0] = (void *)(uintptr_t)1;
679 	if (blk_mq_sched_try_insert_merge(q, rq, free))
682 	trace_block_rq_insert(rq);
685 		list_add(&rq->queuelist, &per_prio->dispatch);
686 		rq->fifo_time = jiffies;
690 		deadline_add_rq_rb(per_prio, rq);
692 		if (rq_mergeable(rq)) {
693 			elv_rqhash_add(q, rq);
695 				q->last_merge = rq;
701 		rq->fifo_time = jiffies + dd->fifo_expire[data_dir];
703 		list_add_tail(&rq->queuelist, insert_before);
720 		struct request *rq;
722 		rq = list_first_entry(list, struct request, queuelist);
723 		list_del_init(&rq->queuelist);
724 		dd_insert_request(hctx, rq, flags, &free);
732 static void dd_prepare_request(struct request *rq)
734 	rq->elv.priv[0] = NULL;
740 static void dd_finish_request(struct request *rq)
742 	struct request_queue *q = rq->q;
744 	const u8 ioprio_class = dd_rq_ioclass(rq);
753 	if (rq->elv.priv[0])
888 	struct request *rq;						\
890 	rq = deadline_from_pos(per_prio, data_dir,			\
892 	if (rq)								\
893 		__blk_mq_debugfs_rq_show(m, rq);			\