1// SPDX-License-Identifier: GPL-2.0 2/* Maximum size of each resync request */ 3#define RESYNC_BLOCK_SIZE (64*1024) 4#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE) 5 6/* 7 * Number of guaranteed raid bios in case of extreme VM load: 8 */ 9#define NR_RAID_BIOS 256 10 11/* when we get a read error on a read-only array, we redirect to another 12 * device without failing the first device, or trying to over-write to 13 * correct the read error. To keep track of bad blocks on a per-bio 14 * level, we store IO_BLOCKED in the appropriate 'bios' pointer 15 */ 16#define IO_BLOCKED ((struct bio *)1) 17/* When we successfully write to a known bad-block, we need to remove the 18 * bad-block marking which must be done from process context. So we record 19 * the success by setting devs[n].bio to IO_MADE_GOOD 20 */ 21#define IO_MADE_GOOD ((struct bio *)2) 22 23#define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) 24#define MAX_PLUG_BIO 32 25 26/* for managing resync I/O pages */ 27struct resync_pages { 28 void *raid_bio; 29 struct page *pages[RESYNC_PAGES]; 30}; 31 32struct raid1_plug_cb { 33 struct blk_plug_cb cb; 34 struct bio_list pending; 35 unsigned int count; 36}; 37 38static void rbio_pool_free(void *rbio, void *data) 39{ 40 kfree(rbio); 41} 42 43static inline int resync_alloc_pages(struct resync_pages *rp, 44 gfp_t gfp_flags) 45{ 46 int i; 47 48 for (i = 0; i < RESYNC_PAGES; i++) { 49 rp->pages[i] = alloc_page(gfp_flags); 50 if (!rp->pages[i]) 51 goto out_free; 52 } 53 54 return 0; 55 56out_free: 57 while (--i >= 0) 58 put_page(rp->pages[i]); 59 return -ENOMEM; 60} 61 62static inline void resync_free_pages(struct resync_pages *rp) 63{ 64 int i; 65 66 for (i = 0; i < RESYNC_PAGES; i++) 67 put_page(rp->pages[i]); 68} 69 70static inline void resync_get_all_pages(struct resync_pages *rp) 71{ 72 int i; 73 74 for (i = 0; i < RESYNC_PAGES; i++) 75 get_page(rp->pages[i]); 76} 77 78static inline struct page *resync_fetch_page(struct resync_pages *rp, 79 unsigned idx) 80{ 81 if (WARN_ON_ONCE(idx >= RESYNC_PAGES)) 82 return NULL; 83 return rp->pages[idx]; 84} 85 86/* 87 * 'strct resync_pages' stores actual pages used for doing the resync 88 * IO, and it is per-bio, so make .bi_private points to it. 89 */ 90static inline struct resync_pages *get_resync_pages(struct bio *bio) 91{ 92 return bio->bi_private; 93} 94 95/* generally called after bio_reset() for reseting bvec */ 96static void md_bio_reset_resync_pages(struct bio *bio, struct resync_pages *rp, 97 int size) 98{ 99 int idx = 0; 100 101 /* initialize bvec table again */ 102 do { 103 struct page *page = resync_fetch_page(rp, idx); 104 int len = min_t(int, size, PAGE_SIZE); 105 106 if (WARN_ON(!bio_add_page(bio, page, len, 0))) { 107 bio->bi_status = BLK_STS_RESOURCE; 108 bio_endio(bio); 109 return; 110 } 111 112 size -= len; 113 } while (idx++ < RESYNC_PAGES && size > 0); 114} 115 116 117static inline void raid1_submit_write(struct bio *bio) 118{ 119 struct md_rdev *rdev = (void *)bio->bi_bdev; 120 121 bio->bi_next = NULL; 122 bio_set_dev(bio, rdev->bdev); 123 if (test_bit(Faulty, &rdev->flags)) 124 bio_io_error(bio); 125 else if (unlikely(bio_op(bio) == REQ_OP_DISCARD && 126 !bdev_max_discard_sectors(bio->bi_bdev))) 127 /* Just ignore it */ 128 bio_endio(bio); 129 else 130 submit_bio_noacct(bio); 131} 132 133static inline bool raid1_add_bio_to_plug(struct mddev *mddev, struct bio *bio, 134 blk_plug_cb_fn unplug, int copies) 135{ 136 struct raid1_plug_cb *plug = NULL; 137 struct blk_plug_cb *cb; 138 139 /* 140 * If bitmap is not enabled, it's safe to submit the io directly, and 141 * this can get optimal performance. 142 */ 143 if (!md_bitmap_enabled(mddev->bitmap)) { 144 raid1_submit_write(bio); 145 return true; 146 } 147 148 cb = blk_check_plugged(unplug, mddev, sizeof(*plug)); 149 if (!cb) 150 return false; 151 152 plug = container_of(cb, struct raid1_plug_cb, cb); 153 bio_list_add(&plug->pending, bio); 154 if (++plug->count / MAX_PLUG_BIO >= copies) { 155 list_del(&cb->list); 156 cb->callback(cb, false); 157 } 158 159 160 return true; 161} 162 163/* 164 * current->bio_list will be set under submit_bio() context, in this case bitmap 165 * io will be added to the list and wait for current io submission to finish, 166 * while current io submission must wait for bitmap io to be done. In order to 167 * avoid such deadlock, submit bitmap io asynchronously. 168 */ 169static inline void raid1_prepare_flush_writes(struct bitmap *bitmap) 170{ 171 if (current->bio_list) 172 md_bitmap_unplug_async(bitmap); 173 else 174 md_bitmap_unplug(bitmap); 175} 176 177/* 178 * Used by fix_read_error() to decay the per rdev read_errors. 179 * We halve the read error count for every hour that has elapsed 180 * since the last recorded read error. 181 */ 182static inline void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 183{ 184 long cur_time_mon; 185 unsigned long hours_since_last; 186 unsigned int read_errors = atomic_read(&rdev->read_errors); 187 188 cur_time_mon = ktime_get_seconds(); 189 190 if (rdev->last_read_error == 0) { 191 /* first time we've seen a read error */ 192 rdev->last_read_error = cur_time_mon; 193 return; 194 } 195 196 hours_since_last = (long)(cur_time_mon - 197 rdev->last_read_error) / 3600; 198 199 rdev->last_read_error = cur_time_mon; 200 201 /* 202 * if hours_since_last is > the number of bits in read_errors 203 * just set read errors to 0. We do this to avoid 204 * overflowing the shift of read_errors by hours_since_last. 205 */ 206 if (hours_since_last >= 8 * sizeof(read_errors)) 207 atomic_set(&rdev->read_errors, 0); 208 else 209 atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 210} 211 212static inline bool exceed_read_errors(struct mddev *mddev, struct md_rdev *rdev) 213{ 214 int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 215 int read_errors; 216 217 check_decay_read_errors(mddev, rdev); 218 read_errors = atomic_inc_return(&rdev->read_errors); 219 if (read_errors > max_read_errors) { 220 pr_notice("md/"RAID_1_10_NAME":%s: %pg: Raid device exceeded read_error threshold [cur %d:max %d]\n", 221 mdname(mddev), rdev->bdev, read_errors, max_read_errors); 222 pr_notice("md/"RAID_1_10_NAME":%s: %pg: Failing raid device\n", 223 mdname(mddev), rdev->bdev); 224 md_error(mddev, rdev); 225 return true; 226 } 227 228 return false; 229} 230 231/** 232 * raid1_check_read_range() - check a given read range for bad blocks, 233 * available read length is returned; 234 * @rdev: the rdev to read; 235 * @this_sector: read position; 236 * @len: read length; 237 * 238 * helper function for read_balance() 239 * 240 * 1) If there are no bad blocks in the range, @len is returned; 241 * 2) If the range are all bad blocks, 0 is returned; 242 * 3) If there are partial bad blocks: 243 * - If the bad block range starts after @this_sector, the length of first 244 * good region is returned; 245 * - If the bad block range starts before @this_sector, 0 is returned and 246 * the @len is updated to the offset into the region before we get to the 247 * good blocks; 248 */ 249static inline int raid1_check_read_range(struct md_rdev *rdev, 250 sector_t this_sector, int *len) 251{ 252 sector_t first_bad; 253 int bad_sectors; 254 255 /* no bad block overlap */ 256 if (!is_badblock(rdev, this_sector, *len, &first_bad, &bad_sectors)) 257 return *len; 258 259 /* 260 * bad block range starts offset into our range so we can return the 261 * number of sectors before the bad blocks start. 262 */ 263 if (first_bad > this_sector) 264 return first_bad - this_sector; 265 266 /* read range is fully consumed by bad blocks. */ 267 if (this_sector + *len <= first_bad + bad_sectors) 268 return 0; 269 270 /* 271 * final case, bad block range starts before or at the start of our 272 * range but does not cover our entire range so we still return 0 but 273 * update the length with the number of sectors before we get to the 274 * good ones. 275 */ 276 *len = first_bad + bad_sectors - this_sector; 277 return 0; 278} 279 280/* 281 * Check if read should choose the first rdev. 282 * 283 * Balance on the whole device if no resync is going on (recovery is ok) or 284 * below the resync window. Otherwise, take the first readable disk. 285 */ 286static inline bool raid1_should_read_first(struct mddev *mddev, 287 sector_t this_sector, int len) 288{ 289 if ((mddev->recovery_cp < this_sector + len)) 290 return true; 291 292 if (mddev_is_clustered(mddev) && 293 md_cluster_ops->area_resyncing(mddev, READ, this_sector, 294 this_sector + len)) 295 return true; 296 297 return false; 298} 299