md/dm-vdo/io-submitter.c

15 #include "data-vio.h"
19 #include "vio.h"
71  * @vio: The vio associated with the bio.
74 static void count_all_bios(struct vio *vio, struct bio *bio)
76 	struct atomic_statistics *stats = &vio->completion.vdo->stats;
78 	if (is_data_vio(vio)) {
84 	if (vio->type == VIO_TYPE_RECOVERY_JOURNAL)
86 	else if (vio->type == VIO_TYPE_BLOCK_MAP)
91  * assert_in_bio_zone() - Assert that a vio is in the correct bio zone and not in interrupt
93  * @vio: The vio to check.
95 static void assert_in_bio_zone(struct vio *vio)
98 	assert_vio_in_bio_zone(vio);
104  * @vio: The vio associated with the bio.
107 static void send_bio_to_device(struct vio *vio, struct bio *bio)
109 	struct vdo *vdo = vio->completion.vdo;
111 	assert_in_bio_zone(vio);
113 	count_all_bios(vio, bio);
119  * vdo_submit_vio() - Submits a vio's bio to the underlying block device. May block if the device
124 	struct vio *vio = as_vio(completion);
126 	send_bio_to_device(vio, vio->bio);
130  * get_bio_list() - Extract the list of bios to submit from a vio.
131  * @vio: The vio submitting I/O.
133  * The list will always contain at least one entry (the bio for the vio on which it is called), but
138 static struct bio *get_bio_list(struct vio *vio)
141 	struct io_submitter *submitter = vio->completion.vdo->io_submitter;
142 	struct bio_queue_data *bio_queue_data = &(submitter->bio_queue_data[vio->bio_zone]);
144 	assert_in_bio_zone(vio);
148 			   vio->bios_merged.head->bi_iter.bi_sector);
150 			   vio->bios_merged.tail->bi_iter.bi_sector);
151 	bio = vio->bios_merged.head;
152 	bio_list_init(&vio->bios_merged);
167 	struct vio *vio = as_vio(completion);
169 	assert_in_bio_zone(vio);
170 	for (bio = get_bio_list(vio); bio != NULL; bio = next) {
173 		send_bio_to_device((struct vio *) bio->bi_private, bio);
181  * @vio: The vio we want to merge.
187  * Return: the vio to merge to, NULL if no merging is possible.
189 static struct vio *get_mergeable_locked(struct int_map *map, struct vio *vio,
192 	struct bio *bio = vio->bio;
194 	struct vio *vio_merge;
206 	if (vio->completion.priority != vio_merge->completion.priority)
224 static int map_merged_vio(struct int_map *bio_map, struct vio *vio)
229 	bio_sector = vio->bios_merged.head->bi_iter.bi_sector;
230 	result = vdo_int_map_put(bio_map, bio_sector, vio, true, NULL);
234 	bio_sector = vio->bios_merged.tail->bi_iter.bi_sector;
235 	return vdo_int_map_put(bio_map, bio_sector, vio, true, NULL);
238 static int merge_to_prev_tail(struct int_map *bio_map, struct vio *vio,
239 			      struct vio *prev_vio)
242 	bio_list_merge(&prev_vio->bios_merged, &vio->bios_merged);
246 static int merge_to_next_head(struct int_map *bio_map, struct vio *vio,
247 			      struct vio *next_vio)
255 	bio_list_merge_head(&next_vio->bios_merged, &vio->bios_merged);
260  * try_bio_map_merge() - Attempt to merge a vio's bio with other pending I/Os.
261  * @vio: The vio to merge.
265  * Return: whether or not the vio was merged.
267 static bool try_bio_map_merge(struct vio *vio)
271 	struct bio *bio = vio->bio;
272 	struct vio *prev_vio, *next_vio;
273 	struct vdo *vdo = vio->completion.vdo;
275 		&vdo->io_submitter->bio_queue_data[vio->bio_zone];
278 	bio_list_init(&vio->bios_merged);
279 	bio_list_add(&vio->bios_merged, bio);
282 	prev_vio = get_mergeable_locked(bio_queue_data->map, vio, true);
283 	next_vio = get_mergeable_locked(bio_queue_data->map, vio, false);
292 					 vio, true, NULL);
295 		result = merge_to_prev_tail(bio_queue_data->map, vio, prev_vio);
298 		result = merge_to_next_head(bio_queue_data->map, vio, next_vio);
316 	if (try_bio_map_merge(&data_vio->vio))
323  * __submit_metadata_vio() - Submit I/O for a metadata vio.
324  * @vio: the vio for which to issue I/O
331  * The vio is enqueued on a vdo bio queue so that bio submission (which may block) does not block
339 void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical,
344 	struct vdo_completion *completion = &vio->completion;
349 	VDO_ASSERT_LOG_ONLY(vio->bio->bi_next == NULL, "metadata bio has no next bio");
353 	result = vio_reset_bio(vio, data, callback, operation | REQ_META, physical);
355 		continue_vio(vio, result);
360 				    get_vio_bio_zone_thread_id(vio));
361 	vdo_launch_completion_with_priority(completion, get_metadata_priority(vio));