md/bcache/request.c

40 static void bio_csum(struct bio *bio, struct bkey *k)
46 	bio_for_each_segment(bv, bio, iter) {
111 	struct bio *bio = op->bio;
114 		 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector);
116 	while (bio_sectors(bio)) {
117 		unsigned int sectors = min(bio_sectors(bio),
123 		bio->bi_iter.bi_sector	+= sectors;
124 		bio->bi_iter.bi_size	-= sectors << 9;
128 				     bio->bi_iter.bi_sector,
134 	bio_put(bio);
169 static void bch_data_insert_endio(struct bio *bio)
171 	struct closure *cl = bio->bi_private;
174 	if (bio->bi_status) {
177 			op->status = bio->bi_status;
184 	bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache");
190 	struct bio *bio = op->bio, *n;
195 	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0)
202 	bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA);
220 		SET_KEY_OFFSET(k, bio->bi_iter.bi_sector);
222 		if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),
227 		n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split);
249 	} while (n != bio);
279 		bio_put(bio);
302  * It inserts the data in op->bio; bi_sector is used for the key offset,
306  * region of the cache represented by op->bio and op->inode.
312 	trace_bcache_write(op->c, op->inode, op->bio,
316 	bio_get(op->bio);
363 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)
373 	    (bio_op(bio) == REQ_OP_DISCARD))
378 	     op_is_write(bio_op(bio))))
382 	 * If the bio is for read-ahead or background IO, bypass it or
391 	if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) {
392 		if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) &&
397 	if (bio->bi_iter.bi_sector & (c->cache->sb.block_size - 1) ||
398 	    bio_sectors(bio) & (c->cache->sb.block_size - 1)) {
416 	hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)
417 		if (i->last == bio->bi_iter.bi_sector &&
426 	if (i->sequential + bio->bi_iter.bi_size > i->sequential)
427 		i->sequential	+= bio->bi_iter.bi_size;
429 	i->last			 = bio_end_sector(bio);
444 		trace_bcache_bypass_sequential(bio);
449 		trace_bcache_bypass_congested(bio);
454 	bch_rescale_priorities(c, bio_sectors(bio));
457 	bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));
467 	struct bbio		bio;
468 	struct bio		*orig_bio;
469 	struct bio		*cache_miss;
485 static void bch_cache_read_endio(struct bio *bio)
487 	struct bbio *b = container_of(bio, struct bbio, bio);
488 	struct closure *cl = bio->bi_private;
492 	 * If the bucket was reused while our bio was in flight, we might have
498 	if (bio->bi_status)
499 		s->iop.status = bio->bi_status;
506 	bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache");
511  * the middle of the bio
516 	struct bio *n, *bio = &s->bio.bio;
520 	if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0)
524 	    KEY_START(k) > bio->bi_iter.bi_sector) {
525 		unsigned int bio_sectors = bio_sectors(bio);
528 				KEY_START(k) - bio->bi_iter.bi_sector)
530 		int ret = s->d->cache_miss(b, s, bio, sectors);
550 	n = bio_next_split(bio, min_t(uint64_t, INT_MAX,
551 				      KEY_OFFSET(k) - bio->bi_iter.bi_sector),
554 	bio_key = &container_of(n, struct bbio, bio)->key;
564 	 * The bucket we're reading from might be reused while our bio
575 	return n == bio ? MAP_DONE : MAP_CONTINUE;
581 	struct bio *bio = &s->bio.bio;
588 				 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0),
602 	 * before we submit s->bio.bio
621 static void request_endio(struct bio *bio)
623 	struct closure *cl = bio->bi_private;
625 	if (bio->bi_status) {
628 		s->iop.status = bio->bi_status;
633 	bio_put(bio);
637 static void backing_request_endio(struct bio *bio)
639 	struct closure *cl = bio->bi_private;
641 	if (bio->bi_status) {
646 		 * If a bio has REQ_PREFLUSH for writeback mode, it is
653 			     bio->bi_opf & REQ_PREFLUSH)) {
655 				dc->bdev, bio->bi_status);
658 			s->iop.status = bio->bi_status;
662 		bch_count_backing_io_errors(dc, bio);
665 	bio_put(bio);
683 			struct bio *orig_bio,
686 	struct bio *bio = &s->bio.bio;
688 	bio_init_clone(orig_bio->bi_bdev, bio, orig_bio, GFP_NOIO);
695 	bio->bi_end_io		= end_io_fn;
696 	bio->bi_private		= &s->cl;
698 	bio_cnt_set(bio, 3);
707 	if (s->iop.bio)
708 		bio_put(s->iop.bio);
715 static inline struct search *search_alloc(struct bio *bio,
724 	do_bio_hook(s, bio, request_endio);
727 	s->orig_bio		= bio;
732 	s->write		= op_is_write(bio_op(bio));
738 	s->iop.bio		= NULL;
744 	s->iop.flush_journal	= op_is_flush(bio->bi_opf);
770 	if (s->iop.bio)
771 		bio_free_pages(s->iop.bio);
779 	struct bio *bio = &s->bio.bio;
798 		closure_bio_submit(s->iop.c, bio, cl);
812 	if (s->iop.bio)
813 		bio_free_pages(s->iop.bio);
829 	 * to the buffers the original bio pointed to:
832 	if (s->iop.bio) {
833 		bio_reset(s->iop.bio, s->cache_miss->bi_bdev, REQ_OP_READ);
834 		s->iop.bio->bi_iter.bi_sector =
836 		s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;
837 		bio_clone_blkg_association(s->iop.bio, s->cache_miss);
838 		bch_bio_map(s->iop.bio, NULL);
840 		bio_copy_data(s->cache_miss, s->iop.bio);
852 	if (s->iop.bio &&
872 	else if (s->iop.bio || verify(dc))
879 				 struct bio *bio, unsigned int sectors)
883 	struct bio *miss, *cache_bio;
889 		miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);
890 		ret = miss == bio ? MAP_DONE : MAP_CONTINUE;
897 	s->insert_bio_sectors = min3(size_limit, sectors, bio_sectors(bio));
900 				 bio->bi_iter.bi_sector + s->insert_bio_sectors,
909 	miss = bio_next_split(bio, s->insert_bio_sectors, GFP_NOIO,
913 	ret = miss == bio ? MAP_DONE : -EINTR;
932 	s->iop.bio	= cache_bio;
970 	struct bio *bio = &s->bio.bio;
971 	struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0);
972 	struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);
993 	if (bio_op(bio) == REQ_OP_DISCARD)
1004 		s->iop.bio = s->orig_bio;
1005 		bio_get(s->iop.bio);
1007 		if (bio_op(bio) == REQ_OP_DISCARD &&
1012 		bio->bi_end_io = backing_request_endio;
1013 		closure_bio_submit(s->iop.c, bio, cl);
1017 		s->iop.bio = bio;
1019 		if (bio->bi_opf & REQ_PREFLUSH) {
1024 			struct bio *flush;
1026 			flush = bio_alloc_bioset(bio->bi_bdev, 0,
1039 		s->iop.bio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO,
1042 		bio->bi_end_io = backing_request_endio;
1043 		closure_bio_submit(s->iop.c, bio, cl);
1054 	struct bio *bio = &s->bio.bio;
1060 	bio->bi_end_io = backing_request_endio;
1061 	closure_bio_submit(s->iop.c, bio, cl);
1074 static void detached_dev_end_io(struct bio *bio)
1078 	ddip = bio->bi_private;
1079 	bio->bi_end_io = ddip->bi_end_io;
1080 	bio->bi_private = ddip->bi_private;
1083 	bio_end_io_acct_remapped(bio, ddip->start_time, ddip->orig_bdev);
1085 	if (bio->bi_status) {
1089 		bch_count_backing_io_errors(dc, bio);
1093 	bio->bi_end_io(bio);
1096 static void detached_dev_do_request(struct bcache_device *d, struct bio *bio,
1109 		bio->bi_status = BLK_STS_RESOURCE;
1110 		bio->bi_end_io(bio);
1118 	ddip->bi_end_io = bio->bi_end_io;
1119 	ddip->bi_private = bio->bi_private;
1120 	bio->bi_end_io = detached_dev_end_io;
1121 	bio->bi_private = ddip;
1123 	if ((bio_op(bio) == REQ_OP_DISCARD) &&
1125 		bio->bi_end_io(bio);
1127 		submit_bio_noacct(bio);
1170 void cached_dev_submit_bio(struct bio *bio)
1173 	struct block_device *orig_bdev = bio->bi_bdev;
1177 	int rw = bio_data_dir(bio);
1181 		bio->bi_status = BLK_STS_IOERR;
1182 		bio_endio(bio);
1201 	start_time = bio_start_io_acct(bio);
1203 	bio_set_dev(bio, dc->bdev);
1204 	bio->bi_iter.bi_sector += dc->sb.data_offset;
1207 		s = search_alloc(bio, d, orig_bdev, start_time);
1208 		trace_bcache_request_start(s->d, bio);
1210 		if (!bio->bi_iter.bi_size) {
1219 			s->iop.bypass = check_should_bypass(dc, bio);
1228 		detached_dev_do_request(d, bio, orig_bdev, start_time);
1252 				struct bio *bio, unsigned int sectors)
1254 	unsigned int bytes = min(sectors, bio_sectors(bio)) << 9;
1256 	swap(bio->bi_iter.bi_size, bytes);
1257 	zero_fill_bio(bio);
1258 	swap(bio->bi_iter.bi_size, bytes);
1260 	bio_advance(bio, bytes);
1262 	if (!bio->bi_iter.bi_size)
1278 void flash_dev_submit_bio(struct bio *bio)
1282 	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
1285 		bio->bi_status = BLK_STS_IOERR;
1286 		bio_endio(bio);
1290 	s = search_alloc(bio, d, bio->bi_bdev, bio_start_io_acct(bio));
1292 	bio = &s->bio.bio;
1294 	trace_bcache_request_start(s->d, bio);
1296 	if (!bio->bi_iter.bi_size) {
1304 	} else if (bio_data_dir(bio)) {
1306 					&KEY(d->id, bio->bi_iter.bi_sector, 0),
1307 					&KEY(d->id, bio_end_sector(bio), 0));
1309 		s->iop.bypass		= (bio_op(bio) == REQ_OP_DISCARD) != 0;
1311 		s->iop.bio		= bio;