drivers/md/dm-io.c

19 #include <linux/dm-io.h>
21 #define DM_MSG_PREFIX "io"
31  * Aligning 'struct io' reduces the number of bits required to store
34 struct io {
86  * We need to keep track of which region a bio is doing io for.
88  * ensure the 'struct io' pointer is aligned so enough low bits are
93 static void store_io_and_region_in_bio(struct bio *bio, struct io *io,
96 	if (unlikely(!IS_ALIGNED((unsigned long)io, DM_IO_MAX_REGIONS))) {
97 		DMCRIT("Unaligned struct io pointer %p", io);
101 	bio->bi_private = (void *)((unsigned long)io | region);
104 static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io,
109 	*io = (void *)(val & -(unsigned long)DM_IO_MAX_REGIONS);
115  * We need an io object to keep track of the number of bios that
116  * have been dispatched for a particular io.
119 static void complete_io(struct io *io)
121 	unsigned long error_bits = io->error_bits;
122 	io_notify_fn fn = io->callback;
123 	void *context = io->context;
125 	if (io->vma_invalidate_size)
126 		invalidate_kernel_vmap_range(io->vma_invalidate_address,
127 					     io->vma_invalidate_size);
129 	mempool_free(io, &io->client->pool);
133 static void dec_count(struct io *io, unsigned int region, blk_status_t error)
136 		set_bit(region, &io->error_bits);
138 	if (atomic_dec_and_test(&io->count))
139 		complete_io(io);
144 	struct io *io;
152 	 * The bio destructor in bio_put() may use the io object.
154 	retrieve_io_and_region_from_bio(bio, &io, &region);
159 	dec_count(io, region, error);
165  * destination page for io.
308 		      struct io *io, unsigned short ioprio)
330 		atomic_inc(&io->count);
331 		dec_count(io, region, BLK_STS_NOTSUPP);
354 				       &io->client->bios);
358 		store_io_and_region_in_bio(bio, io, region);
380 		atomic_inc(&io->count);
387 			struct io *io, int sync, unsigned short ioprio)
404 			do_region(opf, i, where + i, dp, io, ioprio);
409 	 * the io being completed too early.
411 	dec_count(io, 0, 0);
431 	struct io *io;
441 	io = mempool_alloc(&client->pool, GFP_NOIO);
442 	io->error_bits = 0;
443 	atomic_set(&io->count, 1); /* see dispatch_io() */
444 	io->client = client;
445 	io->callback = sync_io_complete;
446 	io->context = &sio;
448 	io->vma_invalidate_address = dp->vma_invalidate_address;
449 	io->vma_invalidate_size = dp->vma_invalidate_size;
451 	dispatch_io(opf, num_regions, where, dp, io, 1, ioprio);
466 	struct io *io;
474 	io = mempool_alloc(&client->pool, GFP_NOIO);
475 	io->error_bits = 0;
476 	atomic_set(&io->count, 1); /* see dispatch_io() */
477 	io->client = client;
478 	io->callback = fn;
479 	io->context = context;
481 	io->vma_invalidate_address = dp->vma_invalidate_address;
482 	io->vma_invalidate_size = dp->vma_invalidate_size;
484 	dispatch_io(opf, num_regions, where, dp, io, 0, ioprio);
548 	_dm_io_cache = KMEM_CACHE(io, 0);