1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2012 Red Hat. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8#include "dm.h" 9#include "dm-bio-prison-v2.h" 10#include "dm-bio-record.h" 11#include "dm-cache-metadata.h" 12#include "dm-io-tracker.h" 13 14#include <linux/dm-io.h> 15#include <linux/dm-kcopyd.h> 16#include <linux/jiffies.h> 17#include <linux/init.h> 18#include <linux/mempool.h> 19#include <linux/module.h> 20#include <linux/rwsem.h> 21#include <linux/slab.h> 22#include <linux/vmalloc.h> 23 24#define DM_MSG_PREFIX "cache" 25 26DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle, 27 "A percentage of time allocated for copying to and/or from cache"); 28 29/*----------------------------------------------------------------*/ 30 31/* 32 * Glossary: 33 * 34 * oblock: index of an origin block 35 * cblock: index of a cache block 36 * promotion: movement of a block from origin to cache 37 * demotion: movement of a block from cache to origin 38 * migration: movement of a block between the origin and cache device, 39 * either direction 40 */ 41 42/*----------------------------------------------------------------*/ 43 44/* 45 * Represents a chunk of future work. 'input' allows continuations to pass 46 * values between themselves, typically error values. 47 */ 48struct continuation { 49 struct work_struct ws; 50 blk_status_t input; 51}; 52 53static inline void init_continuation(struct continuation *k, 54 void (*fn)(struct work_struct *)) 55{ 56 INIT_WORK(&k->ws, fn); 57 k->input = 0; 58} 59 60static inline void queue_continuation(struct workqueue_struct *wq, 61 struct continuation *k) 62{ 63 queue_work(wq, &k->ws); 64} 65 66/*----------------------------------------------------------------*/ 67 68/* 69 * The batcher collects together pieces of work that need a particular 70 * operation to occur before they can proceed (typically a commit). 71 */ 72struct batcher { 73 /* 74 * The operation that everyone is waiting for. 75 */ 76 blk_status_t (*commit_op)(void *context); 77 void *commit_context; 78 79 /* 80 * This is how bios should be issued once the commit op is complete 81 * (accounted_request). 82 */ 83 void (*issue_op)(struct bio *bio, void *context); 84 void *issue_context; 85 86 /* 87 * Queued work gets put on here after commit. 88 */ 89 struct workqueue_struct *wq; 90 91 spinlock_t lock; 92 struct list_head work_items; 93 struct bio_list bios; 94 struct work_struct commit_work; 95 96 bool commit_scheduled; 97}; 98 99static void __commit(struct work_struct *_ws) 100{ 101 struct batcher *b = container_of(_ws, struct batcher, commit_work); 102 blk_status_t r; 103 struct list_head work_items; 104 struct work_struct *ws, *tmp; 105 struct continuation *k; 106 struct bio *bio; 107 struct bio_list bios; 108 109 INIT_LIST_HEAD(&work_items); 110 bio_list_init(&bios); 111 112 /* 113 * We have to grab these before the commit_op to avoid a race 114 * condition. 115 */ 116 spin_lock_irq(&b->lock); 117 list_splice_init(&b->work_items, &work_items); 118 bio_list_merge(&bios, &b->bios); 119 bio_list_init(&b->bios); 120 b->commit_scheduled = false; 121 spin_unlock_irq(&b->lock); 122 123 r = b->commit_op(b->commit_context); 124 125 list_for_each_entry_safe(ws, tmp, &work_items, entry) { 126 k = container_of(ws, struct continuation, ws); 127 k->input = r; 128 INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */ 129 queue_work(b->wq, ws); 130 } 131 132 while ((bio = bio_list_pop(&bios))) { 133 if (r) { 134 bio->bi_status = r; 135 bio_endio(bio); 136 } else 137 b->issue_op(bio, b->issue_context); 138 } 139} 140 141static void batcher_init(struct batcher *b, 142 blk_status_t (*commit_op)(void *), 143 void *commit_context, 144 void (*issue_op)(struct bio *bio, void *), 145 void *issue_context, 146 struct workqueue_struct *wq) 147{ 148 b->commit_op = commit_op; 149 b->commit_context = commit_context; 150 b->issue_op = issue_op; 151 b->issue_context = issue_context; 152 b->wq = wq; 153 154 spin_lock_init(&b->lock); 155 INIT_LIST_HEAD(&b->work_items); 156 bio_list_init(&b->bios); 157 INIT_WORK(&b->commit_work, __commit); 158 b->commit_scheduled = false; 159} 160 161static void async_commit(struct batcher *b) 162{ 163 queue_work(b->wq, &b->commit_work); 164} 165 166static void continue_after_commit(struct batcher *b, struct continuation *k) 167{ 168 bool commit_scheduled; 169 170 spin_lock_irq(&b->lock); 171 commit_scheduled = b->commit_scheduled; 172 list_add_tail(&k->ws.entry, &b->work_items); 173 spin_unlock_irq(&b->lock); 174 175 if (commit_scheduled) 176 async_commit(b); 177} 178 179/* 180 * Bios are errored if commit failed. 181 */ 182static void issue_after_commit(struct batcher *b, struct bio *bio) 183{ 184 bool commit_scheduled; 185 186 spin_lock_irq(&b->lock); 187 commit_scheduled = b->commit_scheduled; 188 bio_list_add(&b->bios, bio); 189 spin_unlock_irq(&b->lock); 190 191 if (commit_scheduled) 192 async_commit(b); 193} 194 195/* 196 * Call this if some urgent work is waiting for the commit to complete. 197 */ 198static void schedule_commit(struct batcher *b) 199{ 200 bool immediate; 201 202 spin_lock_irq(&b->lock); 203 immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios); 204 b->commit_scheduled = true; 205 spin_unlock_irq(&b->lock); 206 207 if (immediate) 208 async_commit(b); 209} 210 211/* 212 * There are a couple of places where we let a bio run, but want to do some 213 * work before calling its endio function. We do this by temporarily 214 * changing the endio fn. 215 */ 216struct dm_hook_info { 217 bio_end_io_t *bi_end_io; 218}; 219 220static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio, 221 bio_end_io_t *bi_end_io, void *bi_private) 222{ 223 h->bi_end_io = bio->bi_end_io; 224 225 bio->bi_end_io = bi_end_io; 226 bio->bi_private = bi_private; 227} 228 229static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio) 230{ 231 bio->bi_end_io = h->bi_end_io; 232} 233 234/*----------------------------------------------------------------*/ 235 236#define MIGRATION_POOL_SIZE 128 237#define COMMIT_PERIOD HZ 238#define MIGRATION_COUNT_WINDOW 10 239 240/* 241 * The block size of the device holding cache data must be 242 * between 32KB and 1GB. 243 */ 244#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT) 245#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 246 247enum cache_metadata_mode { 248 CM_WRITE, /* metadata may be changed */ 249 CM_READ_ONLY, /* metadata may not be changed */ 250 CM_FAIL 251}; 252 253enum cache_io_mode { 254 /* 255 * Data is written to cached blocks only. These blocks are marked 256 * dirty. If you lose the cache device you will lose data. 257 * Potential performance increase for both reads and writes. 258 */ 259 CM_IO_WRITEBACK, 260 261 /* 262 * Data is written to both cache and origin. Blocks are never 263 * dirty. Potential performance benfit for reads only. 264 */ 265 CM_IO_WRITETHROUGH, 266 267 /* 268 * A degraded mode useful for various cache coherency situations 269 * (eg, rolling back snapshots). Reads and writes always go to the 270 * origin. If a write goes to a cached oblock, then the cache 271 * block is invalidated. 272 */ 273 CM_IO_PASSTHROUGH 274}; 275 276struct cache_features { 277 enum cache_metadata_mode mode; 278 enum cache_io_mode io_mode; 279 unsigned int metadata_version; 280 bool discard_passdown:1; 281}; 282 283struct cache_stats { 284 atomic_t read_hit; 285 atomic_t read_miss; 286 atomic_t write_hit; 287 atomic_t write_miss; 288 atomic_t demotion; 289 atomic_t promotion; 290 atomic_t writeback; 291 atomic_t copies_avoided; 292 atomic_t cache_cell_clash; 293 atomic_t commit_count; 294 atomic_t discard_count; 295}; 296 297struct cache { 298 struct dm_target *ti; 299 spinlock_t lock; 300 301 /* 302 * Fields for converting from sectors to blocks. 303 */ 304 int sectors_per_block_shift; 305 sector_t sectors_per_block; 306 307 struct dm_cache_metadata *cmd; 308 309 /* 310 * Metadata is written to this device. 311 */ 312 struct dm_dev *metadata_dev; 313 314 /* 315 * The slower of the two data devices. Typically a spindle. 316 */ 317 struct dm_dev *origin_dev; 318 319 /* 320 * The faster of the two data devices. Typically an SSD. 321 */ 322 struct dm_dev *cache_dev; 323 324 /* 325 * Size of the origin device in _complete_ blocks and native sectors. 326 */ 327 dm_oblock_t origin_blocks; 328 sector_t origin_sectors; 329 330 /* 331 * Size of the cache device in blocks. 332 */ 333 dm_cblock_t cache_size; 334 335 /* 336 * Invalidation fields. 337 */ 338 spinlock_t invalidation_lock; 339 struct list_head invalidation_requests; 340 341 sector_t migration_threshold; 342 wait_queue_head_t migration_wait; 343 atomic_t nr_allocated_migrations; 344 345 /* 346 * The number of in flight migrations that are performing 347 * background io. eg, promotion, writeback. 348 */ 349 atomic_t nr_io_migrations; 350 351 struct bio_list deferred_bios; 352 353 struct rw_semaphore quiesce_lock; 354 355 /* 356 * origin_blocks entries, discarded if set. 357 */ 358 dm_dblock_t discard_nr_blocks; 359 unsigned long *discard_bitset; 360 uint32_t discard_block_size; /* a power of 2 times sectors per block */ 361 362 /* 363 * Rather than reconstructing the table line for the status we just 364 * save it and regurgitate. 365 */ 366 unsigned int nr_ctr_args; 367 const char **ctr_args; 368 369 struct dm_kcopyd_client *copier; 370 struct work_struct deferred_bio_worker; 371 struct work_struct migration_worker; 372 struct workqueue_struct *wq; 373 struct delayed_work waker; 374 struct dm_bio_prison_v2 *prison; 375 376 /* 377 * cache_size entries, dirty if set 378 */ 379 unsigned long *dirty_bitset; 380 atomic_t nr_dirty; 381 382 unsigned int policy_nr_args; 383 struct dm_cache_policy *policy; 384 385 /* 386 * Cache features such as write-through. 387 */ 388 struct cache_features features; 389 390 struct cache_stats stats; 391 392 bool need_tick_bio:1; 393 bool sized:1; 394 bool invalidate:1; 395 bool commit_requested:1; 396 bool loaded_mappings:1; 397 bool loaded_discards:1; 398 399 struct rw_semaphore background_work_lock; 400 401 struct batcher committer; 402 struct work_struct commit_ws; 403 404 struct dm_io_tracker tracker; 405 406 mempool_t migration_pool; 407 408 struct bio_set bs; 409}; 410 411struct per_bio_data { 412 bool tick:1; 413 unsigned int req_nr:2; 414 struct dm_bio_prison_cell_v2 *cell; 415 struct dm_hook_info hook_info; 416 sector_t len; 417}; 418 419struct dm_cache_migration { 420 struct continuation k; 421 struct cache *cache; 422 423 struct policy_work *op; 424 struct bio *overwrite_bio; 425 struct dm_bio_prison_cell_v2 *cell; 426 427 dm_cblock_t invalidate_cblock; 428 dm_oblock_t invalidate_oblock; 429}; 430 431/*----------------------------------------------------------------*/ 432 433static bool writethrough_mode(struct cache *cache) 434{ 435 return cache->features.io_mode == CM_IO_WRITETHROUGH; 436} 437 438static bool writeback_mode(struct cache *cache) 439{ 440 return cache->features.io_mode == CM_IO_WRITEBACK; 441} 442 443static inline bool passthrough_mode(struct cache *cache) 444{ 445 return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH); 446} 447 448/*----------------------------------------------------------------*/ 449 450static void wake_deferred_bio_worker(struct cache *cache) 451{ 452 queue_work(cache->wq, &cache->deferred_bio_worker); 453} 454 455static void wake_migration_worker(struct cache *cache) 456{ 457 if (passthrough_mode(cache)) 458 return; 459 460 queue_work(cache->wq, &cache->migration_worker); 461} 462 463/*----------------------------------------------------------------*/ 464 465static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache) 466{ 467 return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO); 468} 469 470static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell) 471{ 472 dm_bio_prison_free_cell_v2(cache->prison, cell); 473} 474 475static struct dm_cache_migration *alloc_migration(struct cache *cache) 476{ 477 struct dm_cache_migration *mg; 478 479 mg = mempool_alloc(&cache->migration_pool, GFP_NOIO); 480 481 memset(mg, 0, sizeof(*mg)); 482 483 mg->cache = cache; 484 atomic_inc(&cache->nr_allocated_migrations); 485 486 return mg; 487} 488 489static void free_migration(struct dm_cache_migration *mg) 490{ 491 struct cache *cache = mg->cache; 492 493 if (atomic_dec_and_test(&cache->nr_allocated_migrations)) 494 wake_up(&cache->migration_wait); 495 496 mempool_free(mg, &cache->migration_pool); 497} 498 499/*----------------------------------------------------------------*/ 500 501static inline dm_oblock_t oblock_succ(dm_oblock_t b) 502{ 503 return to_oblock(from_oblock(b) + 1ull); 504} 505 506static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key) 507{ 508 key->virtual = 0; 509 key->dev = 0; 510 key->block_begin = from_oblock(begin); 511 key->block_end = from_oblock(end); 512} 513 514/* 515 * We have two lock levels. Level 0, which is used to prevent WRITEs, and 516 * level 1 which prevents *both* READs and WRITEs. 517 */ 518#define WRITE_LOCK_LEVEL 0 519#define READ_WRITE_LOCK_LEVEL 1 520 521static unsigned int lock_level(struct bio *bio) 522{ 523 return bio_data_dir(bio) == WRITE ? 524 WRITE_LOCK_LEVEL : 525 READ_WRITE_LOCK_LEVEL; 526} 527 528/* 529 *-------------------------------------------------------------- 530 * Per bio data 531 *-------------------------------------------------------------- 532 */ 533 534static struct per_bio_data *get_per_bio_data(struct bio *bio) 535{ 536 struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data)); 537 538 BUG_ON(!pb); 539 return pb; 540} 541 542static struct per_bio_data *init_per_bio_data(struct bio *bio) 543{ 544 struct per_bio_data *pb = get_per_bio_data(bio); 545 546 pb->tick = false; 547 pb->req_nr = dm_bio_get_target_bio_nr(bio); 548 pb->cell = NULL; 549 pb->len = 0; 550 551 return pb; 552} 553 554/*----------------------------------------------------------------*/ 555 556static void defer_bio(struct cache *cache, struct bio *bio) 557{ 558 spin_lock_irq(&cache->lock); 559 bio_list_add(&cache->deferred_bios, bio); 560 spin_unlock_irq(&cache->lock); 561 562 wake_deferred_bio_worker(cache); 563} 564 565static void defer_bios(struct cache *cache, struct bio_list *bios) 566{ 567 spin_lock_irq(&cache->lock); 568 bio_list_merge(&cache->deferred_bios, bios); 569 bio_list_init(bios); 570 spin_unlock_irq(&cache->lock); 571 572 wake_deferred_bio_worker(cache); 573} 574 575/*----------------------------------------------------------------*/ 576 577static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio) 578{ 579 bool r; 580 struct per_bio_data *pb; 581 struct dm_cell_key_v2 key; 582 dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL); 583 struct dm_bio_prison_cell_v2 *cell_prealloc, *cell; 584 585 cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */ 586 587 build_key(oblock, end, &key); 588 r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell); 589 if (!r) { 590 /* 591 * Failed to get the lock. 592 */ 593 free_prison_cell(cache, cell_prealloc); 594 return r; 595 } 596 597 if (cell != cell_prealloc) 598 free_prison_cell(cache, cell_prealloc); 599 600 pb = get_per_bio_data(bio); 601 pb->cell = cell; 602 603 return r; 604} 605 606/*----------------------------------------------------------------*/ 607 608static bool is_dirty(struct cache *cache, dm_cblock_t b) 609{ 610 return test_bit(from_cblock(b), cache->dirty_bitset); 611} 612 613static void set_dirty(struct cache *cache, dm_cblock_t cblock) 614{ 615 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) { 616 atomic_inc(&cache->nr_dirty); 617 policy_set_dirty(cache->policy, cblock); 618 } 619} 620 621/* 622 * These two are called when setting after migrations to force the policy 623 * and dirty bitset to be in sync. 624 */ 625static void force_set_dirty(struct cache *cache, dm_cblock_t cblock) 626{ 627 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) 628 atomic_inc(&cache->nr_dirty); 629 policy_set_dirty(cache->policy, cblock); 630} 631 632static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock) 633{ 634 if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) { 635 if (atomic_dec_return(&cache->nr_dirty) == 0) 636 dm_table_event(cache->ti->table); 637 } 638 639 policy_clear_dirty(cache->policy, cblock); 640} 641 642/*----------------------------------------------------------------*/ 643 644static bool block_size_is_power_of_two(struct cache *cache) 645{ 646 return cache->sectors_per_block_shift >= 0; 647} 648 649static dm_block_t block_div(dm_block_t b, uint32_t n) 650{ 651 do_div(b, n); 652 653 return b; 654} 655 656static dm_block_t oblocks_per_dblock(struct cache *cache) 657{ 658 dm_block_t oblocks = cache->discard_block_size; 659 660 if (block_size_is_power_of_two(cache)) 661 oblocks >>= cache->sectors_per_block_shift; 662 else 663 oblocks = block_div(oblocks, cache->sectors_per_block); 664 665 return oblocks; 666} 667 668static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock) 669{ 670 return to_dblock(block_div(from_oblock(oblock), 671 oblocks_per_dblock(cache))); 672} 673 674static void set_discard(struct cache *cache, dm_dblock_t b) 675{ 676 BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks)); 677 atomic_inc(&cache->stats.discard_count); 678 679 spin_lock_irq(&cache->lock); 680 set_bit(from_dblock(b), cache->discard_bitset); 681 spin_unlock_irq(&cache->lock); 682} 683 684static void clear_discard(struct cache *cache, dm_dblock_t b) 685{ 686 spin_lock_irq(&cache->lock); 687 clear_bit(from_dblock(b), cache->discard_bitset); 688 spin_unlock_irq(&cache->lock); 689} 690 691static bool is_discarded(struct cache *cache, dm_dblock_t b) 692{ 693 int r; 694 695 spin_lock_irq(&cache->lock); 696 r = test_bit(from_dblock(b), cache->discard_bitset); 697 spin_unlock_irq(&cache->lock); 698 699 return r; 700} 701 702static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) 703{ 704 int r; 705 706 spin_lock_irq(&cache->lock); 707 r = test_bit(from_dblock(oblock_to_dblock(cache, b)), 708 cache->discard_bitset); 709 spin_unlock_irq(&cache->lock); 710 711 return r; 712} 713 714/* 715 * ------------------------------------------------------------- 716 * Remapping 717 *-------------------------------------------------------------- 718 */ 719static void remap_to_origin(struct cache *cache, struct bio *bio) 720{ 721 bio_set_dev(bio, cache->origin_dev->bdev); 722} 723 724static void remap_to_cache(struct cache *cache, struct bio *bio, 725 dm_cblock_t cblock) 726{ 727 sector_t bi_sector = bio->bi_iter.bi_sector; 728 sector_t block = from_cblock(cblock); 729 730 bio_set_dev(bio, cache->cache_dev->bdev); 731 if (!block_size_is_power_of_two(cache)) 732 bio->bi_iter.bi_sector = 733 (block * cache->sectors_per_block) + 734 sector_div(bi_sector, cache->sectors_per_block); 735 else 736 bio->bi_iter.bi_sector = 737 (block << cache->sectors_per_block_shift) | 738 (bi_sector & (cache->sectors_per_block - 1)); 739} 740 741static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio) 742{ 743 struct per_bio_data *pb; 744 745 spin_lock_irq(&cache->lock); 746 if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) && 747 bio_op(bio) != REQ_OP_DISCARD) { 748 pb = get_per_bio_data(bio); 749 pb->tick = true; 750 cache->need_tick_bio = false; 751 } 752 spin_unlock_irq(&cache->lock); 753} 754 755static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio, 756 dm_oblock_t oblock) 757{ 758 // FIXME: check_if_tick_bio_needed() is called way too much through this interface 759 check_if_tick_bio_needed(cache, bio); 760 remap_to_origin(cache, bio); 761 if (bio_data_dir(bio) == WRITE) 762 clear_discard(cache, oblock_to_dblock(cache, oblock)); 763} 764 765static void remap_to_cache_dirty(struct cache *cache, struct bio *bio, 766 dm_oblock_t oblock, dm_cblock_t cblock) 767{ 768 check_if_tick_bio_needed(cache, bio); 769 remap_to_cache(cache, bio, cblock); 770 if (bio_data_dir(bio) == WRITE) { 771 set_dirty(cache, cblock); 772 clear_discard(cache, oblock_to_dblock(cache, oblock)); 773 } 774} 775 776static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio) 777{ 778 sector_t block_nr = bio->bi_iter.bi_sector; 779 780 if (!block_size_is_power_of_two(cache)) 781 (void) sector_div(block_nr, cache->sectors_per_block); 782 else 783 block_nr >>= cache->sectors_per_block_shift; 784 785 return to_oblock(block_nr); 786} 787 788static bool accountable_bio(struct cache *cache, struct bio *bio) 789{ 790 return bio_op(bio) != REQ_OP_DISCARD; 791} 792 793static void accounted_begin(struct cache *cache, struct bio *bio) 794{ 795 struct per_bio_data *pb; 796 797 if (accountable_bio(cache, bio)) { 798 pb = get_per_bio_data(bio); 799 pb->len = bio_sectors(bio); 800 dm_iot_io_begin(&cache->tracker, pb->len); 801 } 802} 803 804static void accounted_complete(struct cache *cache, struct bio *bio) 805{ 806 struct per_bio_data *pb = get_per_bio_data(bio); 807 808 dm_iot_io_end(&cache->tracker, pb->len); 809} 810 811static void accounted_request(struct cache *cache, struct bio *bio) 812{ 813 accounted_begin(cache, bio); 814 dm_submit_bio_remap(bio, NULL); 815} 816 817static void issue_op(struct bio *bio, void *context) 818{ 819 struct cache *cache = context; 820 821 accounted_request(cache, bio); 822} 823 824/* 825 * When running in writethrough mode we need to send writes to clean blocks 826 * to both the cache and origin devices. Clone the bio and send them in parallel. 827 */ 828static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio, 829 dm_oblock_t oblock, dm_cblock_t cblock) 830{ 831 struct bio *origin_bio = bio_alloc_clone(cache->origin_dev->bdev, bio, 832 GFP_NOIO, &cache->bs); 833 834 BUG_ON(!origin_bio); 835 836 bio_chain(origin_bio, bio); 837 838 if (bio_data_dir(origin_bio) == WRITE) 839 clear_discard(cache, oblock_to_dblock(cache, oblock)); 840 submit_bio(origin_bio); 841 842 remap_to_cache(cache, bio, cblock); 843} 844 845/* 846 *-------------------------------------------------------------- 847 * Failure modes 848 *-------------------------------------------------------------- 849 */ 850static enum cache_metadata_mode get_cache_mode(struct cache *cache) 851{ 852 return cache->features.mode; 853} 854 855static const char *cache_device_name(struct cache *cache) 856{ 857 return dm_table_device_name(cache->ti->table); 858} 859 860static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode) 861{ 862 static const char *descs[] = { 863 "write", 864 "read-only", 865 "fail" 866 }; 867 868 dm_table_event(cache->ti->table); 869 DMINFO("%s: switching cache to %s mode", 870 cache_device_name(cache), descs[(int)mode]); 871} 872 873static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode) 874{ 875 bool needs_check; 876 enum cache_metadata_mode old_mode = get_cache_mode(cache); 877 878 if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) { 879 DMERR("%s: unable to read needs_check flag, setting failure mode.", 880 cache_device_name(cache)); 881 new_mode = CM_FAIL; 882 } 883 884 if (new_mode == CM_WRITE && needs_check) { 885 DMERR("%s: unable to switch cache to write mode until repaired.", 886 cache_device_name(cache)); 887 if (old_mode != new_mode) 888 new_mode = old_mode; 889 else 890 new_mode = CM_READ_ONLY; 891 } 892 893 /* Never move out of fail mode */ 894 if (old_mode == CM_FAIL) 895 new_mode = CM_FAIL; 896 897 switch (new_mode) { 898 case CM_FAIL: 899 case CM_READ_ONLY: 900 dm_cache_metadata_set_read_only(cache->cmd); 901 break; 902 903 case CM_WRITE: 904 dm_cache_metadata_set_read_write(cache->cmd); 905 break; 906 } 907 908 cache->features.mode = new_mode; 909 910 if (new_mode != old_mode) 911 notify_mode_switch(cache, new_mode); 912} 913 914static void abort_transaction(struct cache *cache) 915{ 916 const char *dev_name = cache_device_name(cache); 917 918 if (get_cache_mode(cache) >= CM_READ_ONLY) 919 return; 920 921 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name); 922 if (dm_cache_metadata_abort(cache->cmd)) { 923 DMERR("%s: failed to abort metadata transaction", dev_name); 924 set_cache_mode(cache, CM_FAIL); 925 } 926 927 if (dm_cache_metadata_set_needs_check(cache->cmd)) { 928 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name); 929 set_cache_mode(cache, CM_FAIL); 930 } 931} 932 933static void metadata_operation_failed(struct cache *cache, const char *op, int r) 934{ 935 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d", 936 cache_device_name(cache), op, r); 937 abort_transaction(cache); 938 set_cache_mode(cache, CM_READ_ONLY); 939} 940 941/*----------------------------------------------------------------*/ 942 943static void load_stats(struct cache *cache) 944{ 945 struct dm_cache_statistics stats; 946 947 dm_cache_metadata_get_stats(cache->cmd, &stats); 948 atomic_set(&cache->stats.read_hit, stats.read_hits); 949 atomic_set(&cache->stats.read_miss, stats.read_misses); 950 atomic_set(&cache->stats.write_hit, stats.write_hits); 951 atomic_set(&cache->stats.write_miss, stats.write_misses); 952} 953 954static void save_stats(struct cache *cache) 955{ 956 struct dm_cache_statistics stats; 957 958 if (get_cache_mode(cache) >= CM_READ_ONLY) 959 return; 960 961 stats.read_hits = atomic_read(&cache->stats.read_hit); 962 stats.read_misses = atomic_read(&cache->stats.read_miss); 963 stats.write_hits = atomic_read(&cache->stats.write_hit); 964 stats.write_misses = atomic_read(&cache->stats.write_miss); 965 966 dm_cache_metadata_set_stats(cache->cmd, &stats); 967} 968 969static void update_stats(struct cache_stats *stats, enum policy_operation op) 970{ 971 switch (op) { 972 case POLICY_PROMOTE: 973 atomic_inc(&stats->promotion); 974 break; 975 976 case POLICY_DEMOTE: 977 atomic_inc(&stats->demotion); 978 break; 979 980 case POLICY_WRITEBACK: 981 atomic_inc(&stats->writeback); 982 break; 983 } 984} 985 986/* 987 *--------------------------------------------------------------------- 988 * Migration processing 989 * 990 * Migration covers moving data from the origin device to the cache, or 991 * vice versa. 992 *--------------------------------------------------------------------- 993 */ 994static void inc_io_migrations(struct cache *cache) 995{ 996 atomic_inc(&cache->nr_io_migrations); 997} 998 999static void dec_io_migrations(struct cache *cache) 1000{ 1001 atomic_dec(&cache->nr_io_migrations); 1002} 1003 1004static bool discard_or_flush(struct bio *bio) 1005{ 1006 return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf); 1007} 1008 1009static void calc_discard_block_range(struct cache *cache, struct bio *bio, 1010 dm_dblock_t *b, dm_dblock_t *e) 1011{ 1012 sector_t sb = bio->bi_iter.bi_sector; 1013 sector_t se = bio_end_sector(bio); 1014 1015 *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size)); 1016 1017 if (se - sb < cache->discard_block_size) 1018 *e = *b; 1019 else 1020 *e = to_dblock(block_div(se, cache->discard_block_size)); 1021} 1022 1023/*----------------------------------------------------------------*/ 1024 1025static void prevent_background_work(struct cache *cache) 1026{ 1027 lockdep_off(); 1028 down_write(&cache->background_work_lock); 1029 lockdep_on(); 1030} 1031 1032static void allow_background_work(struct cache *cache) 1033{ 1034 lockdep_off(); 1035 up_write(&cache->background_work_lock); 1036 lockdep_on(); 1037} 1038 1039static bool background_work_begin(struct cache *cache) 1040{ 1041 bool r; 1042 1043 lockdep_off(); 1044 r = down_read_trylock(&cache->background_work_lock); 1045 lockdep_on(); 1046 1047 return r; 1048} 1049 1050static void background_work_end(struct cache *cache) 1051{ 1052 lockdep_off(); 1053 up_read(&cache->background_work_lock); 1054 lockdep_on(); 1055} 1056 1057/*----------------------------------------------------------------*/ 1058 1059static bool bio_writes_complete_block(struct cache *cache, struct bio *bio) 1060{ 1061 return (bio_data_dir(bio) == WRITE) && 1062 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT)); 1063} 1064 1065static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block) 1066{ 1067 return writeback_mode(cache) && 1068 (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio)); 1069} 1070 1071static void quiesce(struct dm_cache_migration *mg, 1072 void (*continuation)(struct work_struct *)) 1073{ 1074 init_continuation(&mg->k, continuation); 1075 dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws); 1076} 1077 1078static struct dm_cache_migration *ws_to_mg(struct work_struct *ws) 1079{ 1080 struct continuation *k = container_of(ws, struct continuation, ws); 1081 1082 return container_of(k, struct dm_cache_migration, k); 1083} 1084 1085static void copy_complete(int read_err, unsigned long write_err, void *context) 1086{ 1087 struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k); 1088 1089 if (read_err || write_err) 1090 mg->k.input = BLK_STS_IOERR; 1091 1092 queue_continuation(mg->cache->wq, &mg->k); 1093} 1094 1095static void copy(struct dm_cache_migration *mg, bool promote) 1096{ 1097 struct dm_io_region o_region, c_region; 1098 struct cache *cache = mg->cache; 1099 1100 o_region.bdev = cache->origin_dev->bdev; 1101 o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block; 1102 o_region.count = cache->sectors_per_block; 1103 1104 c_region.bdev = cache->cache_dev->bdev; 1105 c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block; 1106 c_region.count = cache->sectors_per_block; 1107 1108 if (promote) 1109 dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k); 1110 else 1111 dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k); 1112} 1113 1114static void bio_drop_shared_lock(struct cache *cache, struct bio *bio) 1115{ 1116 struct per_bio_data *pb = get_per_bio_data(bio); 1117 1118 if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell)) 1119 free_prison_cell(cache, pb->cell); 1120 pb->cell = NULL; 1121} 1122 1123static void overwrite_endio(struct bio *bio) 1124{ 1125 struct dm_cache_migration *mg = bio->bi_private; 1126 struct cache *cache = mg->cache; 1127 struct per_bio_data *pb = get_per_bio_data(bio); 1128 1129 dm_unhook_bio(&pb->hook_info, bio); 1130 1131 if (bio->bi_status) 1132 mg->k.input = bio->bi_status; 1133 1134 queue_continuation(cache->wq, &mg->k); 1135} 1136 1137static void overwrite(struct dm_cache_migration *mg, 1138 void (*continuation)(struct work_struct *)) 1139{ 1140 struct bio *bio = mg->overwrite_bio; 1141 struct per_bio_data *pb = get_per_bio_data(bio); 1142 1143 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); 1144 1145 /* 1146 * The overwrite bio is part of the copy operation, as such it does 1147 * not set/clear discard or dirty flags. 1148 */ 1149 if (mg->op->op == POLICY_PROMOTE) 1150 remap_to_cache(mg->cache, bio, mg->op->cblock); 1151 else 1152 remap_to_origin(mg->cache, bio); 1153 1154 init_continuation(&mg->k, continuation); 1155 accounted_request(mg->cache, bio); 1156} 1157 1158/* 1159 * Migration steps: 1160 * 1161 * 1) exclusive lock preventing WRITEs 1162 * 2) quiesce 1163 * 3) copy or issue overwrite bio 1164 * 4) upgrade to exclusive lock preventing READs and WRITEs 1165 * 5) quiesce 1166 * 6) update metadata and commit 1167 * 7) unlock 1168 */ 1169static void mg_complete(struct dm_cache_migration *mg, bool success) 1170{ 1171 struct bio_list bios; 1172 struct cache *cache = mg->cache; 1173 struct policy_work *op = mg->op; 1174 dm_cblock_t cblock = op->cblock; 1175 1176 if (success) 1177 update_stats(&cache->stats, op->op); 1178 1179 switch (op->op) { 1180 case POLICY_PROMOTE: 1181 clear_discard(cache, oblock_to_dblock(cache, op->oblock)); 1182 policy_complete_background_work(cache->policy, op, success); 1183 1184 if (mg->overwrite_bio) { 1185 if (success) 1186 force_set_dirty(cache, cblock); 1187 else if (mg->k.input) 1188 mg->overwrite_bio->bi_status = mg->k.input; 1189 else 1190 mg->overwrite_bio->bi_status = BLK_STS_IOERR; 1191 bio_endio(mg->overwrite_bio); 1192 } else { 1193 if (success) 1194 force_clear_dirty(cache, cblock); 1195 dec_io_migrations(cache); 1196 } 1197 break; 1198 1199 case POLICY_DEMOTE: 1200 /* 1201 * We clear dirty here to update the nr_dirty counter. 1202 */ 1203 if (success) 1204 force_clear_dirty(cache, cblock); 1205 policy_complete_background_work(cache->policy, op, success); 1206 dec_io_migrations(cache); 1207 break; 1208 1209 case POLICY_WRITEBACK: 1210 if (success) 1211 force_clear_dirty(cache, cblock); 1212 policy_complete_background_work(cache->policy, op, success); 1213 dec_io_migrations(cache); 1214 break; 1215 } 1216 1217 bio_list_init(&bios); 1218 if (mg->cell) { 1219 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) 1220 free_prison_cell(cache, mg->cell); 1221 } 1222 1223 free_migration(mg); 1224 defer_bios(cache, &bios); 1225 wake_migration_worker(cache); 1226 1227 background_work_end(cache); 1228} 1229 1230static void mg_success(struct work_struct *ws) 1231{ 1232 struct dm_cache_migration *mg = ws_to_mg(ws); 1233 1234 mg_complete(mg, mg->k.input == 0); 1235} 1236 1237static void mg_update_metadata(struct work_struct *ws) 1238{ 1239 int r; 1240 struct dm_cache_migration *mg = ws_to_mg(ws); 1241 struct cache *cache = mg->cache; 1242 struct policy_work *op = mg->op; 1243 1244 switch (op->op) { 1245 case POLICY_PROMOTE: 1246 r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock); 1247 if (r) { 1248 DMERR_LIMIT("%s: migration failed; couldn't insert mapping", 1249 cache_device_name(cache)); 1250 metadata_operation_failed(cache, "dm_cache_insert_mapping", r); 1251 1252 mg_complete(mg, false); 1253 return; 1254 } 1255 mg_complete(mg, true); 1256 break; 1257 1258 case POLICY_DEMOTE: 1259 r = dm_cache_remove_mapping(cache->cmd, op->cblock); 1260 if (r) { 1261 DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata", 1262 cache_device_name(cache)); 1263 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 1264 1265 mg_complete(mg, false); 1266 return; 1267 } 1268 1269 /* 1270 * It would be nice if we only had to commit when a REQ_FLUSH 1271 * comes through. But there's one scenario that we have to 1272 * look out for: 1273 * 1274 * - vblock x in a cache block 1275 * - domotion occurs 1276 * - cache block gets reallocated and over written 1277 * - crash 1278 * 1279 * When we recover, because there was no commit the cache will 1280 * rollback to having the data for vblock x in the cache block. 1281 * But the cache block has since been overwritten, so it'll end 1282 * up pointing to data that was never in 'x' during the history 1283 * of the device. 1284 * 1285 * To avoid this issue we require a commit as part of the 1286 * demotion operation. 1287 */ 1288 init_continuation(&mg->k, mg_success); 1289 continue_after_commit(&cache->committer, &mg->k); 1290 schedule_commit(&cache->committer); 1291 break; 1292 1293 case POLICY_WRITEBACK: 1294 mg_complete(mg, true); 1295 break; 1296 } 1297} 1298 1299static void mg_update_metadata_after_copy(struct work_struct *ws) 1300{ 1301 struct dm_cache_migration *mg = ws_to_mg(ws); 1302 1303 /* 1304 * Did the copy succeed? 1305 */ 1306 if (mg->k.input) 1307 mg_complete(mg, false); 1308 else 1309 mg_update_metadata(ws); 1310} 1311 1312static void mg_upgrade_lock(struct work_struct *ws) 1313{ 1314 int r; 1315 struct dm_cache_migration *mg = ws_to_mg(ws); 1316 1317 /* 1318 * Did the copy succeed? 1319 */ 1320 if (mg->k.input) 1321 mg_complete(mg, false); 1322 1323 else { 1324 /* 1325 * Now we want the lock to prevent both reads and writes. 1326 */ 1327 r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell, 1328 READ_WRITE_LOCK_LEVEL); 1329 if (r < 0) 1330 mg_complete(mg, false); 1331 1332 else if (r) 1333 quiesce(mg, mg_update_metadata); 1334 1335 else 1336 mg_update_metadata(ws); 1337 } 1338} 1339 1340static void mg_full_copy(struct work_struct *ws) 1341{ 1342 struct dm_cache_migration *mg = ws_to_mg(ws); 1343 struct cache *cache = mg->cache; 1344 struct policy_work *op = mg->op; 1345 bool is_policy_promote = (op->op == POLICY_PROMOTE); 1346 1347 if ((!is_policy_promote && !is_dirty(cache, op->cblock)) || 1348 is_discarded_oblock(cache, op->oblock)) { 1349 mg_upgrade_lock(ws); 1350 return; 1351 } 1352 1353 init_continuation(&mg->k, mg_upgrade_lock); 1354 copy(mg, is_policy_promote); 1355} 1356 1357static void mg_copy(struct work_struct *ws) 1358{ 1359 struct dm_cache_migration *mg = ws_to_mg(ws); 1360 1361 if (mg->overwrite_bio) { 1362 /* 1363 * No exclusive lock was held when we last checked if the bio 1364 * was optimisable. So we have to check again in case things 1365 * have changed (eg, the block may no longer be discarded). 1366 */ 1367 if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) { 1368 /* 1369 * Fallback to a real full copy after doing some tidying up. 1370 */ 1371 bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio); 1372 1373 BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */ 1374 mg->overwrite_bio = NULL; 1375 inc_io_migrations(mg->cache); 1376 mg_full_copy(ws); 1377 return; 1378 } 1379 1380 /* 1381 * It's safe to do this here, even though it's new data 1382 * because all IO has been locked out of the block. 1383 * 1384 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL 1385 * so _not_ using mg_upgrade_lock() as continutation. 1386 */ 1387 overwrite(mg, mg_update_metadata_after_copy); 1388 1389 } else 1390 mg_full_copy(ws); 1391} 1392 1393static int mg_lock_writes(struct dm_cache_migration *mg) 1394{ 1395 int r; 1396 struct dm_cell_key_v2 key; 1397 struct cache *cache = mg->cache; 1398 struct dm_bio_prison_cell_v2 *prealloc; 1399 1400 prealloc = alloc_prison_cell(cache); 1401 1402 /* 1403 * Prevent writes to the block, but allow reads to continue. 1404 * Unless we're using an overwrite bio, in which case we lock 1405 * everything. 1406 */ 1407 build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key); 1408 r = dm_cell_lock_v2(cache->prison, &key, 1409 mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL, 1410 prealloc, &mg->cell); 1411 if (r < 0) { 1412 free_prison_cell(cache, prealloc); 1413 mg_complete(mg, false); 1414 return r; 1415 } 1416 1417 if (mg->cell != prealloc) 1418 free_prison_cell(cache, prealloc); 1419 1420 if (r == 0) 1421 mg_copy(&mg->k.ws); 1422 else 1423 quiesce(mg, mg_copy); 1424 1425 return 0; 1426} 1427 1428static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio) 1429{ 1430 struct dm_cache_migration *mg; 1431 1432 if (!background_work_begin(cache)) { 1433 policy_complete_background_work(cache->policy, op, false); 1434 return -EPERM; 1435 } 1436 1437 mg = alloc_migration(cache); 1438 1439 mg->op = op; 1440 mg->overwrite_bio = bio; 1441 1442 if (!bio) 1443 inc_io_migrations(cache); 1444 1445 return mg_lock_writes(mg); 1446} 1447 1448/* 1449 *-------------------------------------------------------------- 1450 * invalidation processing 1451 *-------------------------------------------------------------- 1452 */ 1453 1454static void invalidate_complete(struct dm_cache_migration *mg, bool success) 1455{ 1456 struct bio_list bios; 1457 struct cache *cache = mg->cache; 1458 1459 bio_list_init(&bios); 1460 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) 1461 free_prison_cell(cache, mg->cell); 1462 1463 if (!success && mg->overwrite_bio) 1464 bio_io_error(mg->overwrite_bio); 1465 1466 free_migration(mg); 1467 defer_bios(cache, &bios); 1468 1469 background_work_end(cache); 1470} 1471 1472static void invalidate_completed(struct work_struct *ws) 1473{ 1474 struct dm_cache_migration *mg = ws_to_mg(ws); 1475 1476 invalidate_complete(mg, !mg->k.input); 1477} 1478 1479static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock) 1480{ 1481 int r; 1482 1483 r = policy_invalidate_mapping(cache->policy, cblock); 1484 if (!r) { 1485 r = dm_cache_remove_mapping(cache->cmd, cblock); 1486 if (r) { 1487 DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata", 1488 cache_device_name(cache)); 1489 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 1490 } 1491 1492 } else if (r == -ENODATA) { 1493 /* 1494 * Harmless, already unmapped. 1495 */ 1496 r = 0; 1497 1498 } else 1499 DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache)); 1500 1501 return r; 1502} 1503 1504static void invalidate_remove(struct work_struct *ws) 1505{ 1506 int r; 1507 struct dm_cache_migration *mg = ws_to_mg(ws); 1508 struct cache *cache = mg->cache; 1509 1510 r = invalidate_cblock(cache, mg->invalidate_cblock); 1511 if (r) { 1512 invalidate_complete(mg, false); 1513 return; 1514 } 1515 1516 init_continuation(&mg->k, invalidate_completed); 1517 continue_after_commit(&cache->committer, &mg->k); 1518 remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock); 1519 mg->overwrite_bio = NULL; 1520 schedule_commit(&cache->committer); 1521} 1522 1523static int invalidate_lock(struct dm_cache_migration *mg) 1524{ 1525 int r; 1526 struct dm_cell_key_v2 key; 1527 struct cache *cache = mg->cache; 1528 struct dm_bio_prison_cell_v2 *prealloc; 1529 1530 prealloc = alloc_prison_cell(cache); 1531 1532 build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key); 1533 r = dm_cell_lock_v2(cache->prison, &key, 1534 READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell); 1535 if (r < 0) { 1536 free_prison_cell(cache, prealloc); 1537 invalidate_complete(mg, false); 1538 return r; 1539 } 1540 1541 if (mg->cell != prealloc) 1542 free_prison_cell(cache, prealloc); 1543 1544 if (r) 1545 quiesce(mg, invalidate_remove); 1546 1547 else { 1548 /* 1549 * We can't call invalidate_remove() directly here because we 1550 * might still be in request context. 1551 */ 1552 init_continuation(&mg->k, invalidate_remove); 1553 queue_work(cache->wq, &mg->k.ws); 1554 } 1555 1556 return 0; 1557} 1558 1559static int invalidate_start(struct cache *cache, dm_cblock_t cblock, 1560 dm_oblock_t oblock, struct bio *bio) 1561{ 1562 struct dm_cache_migration *mg; 1563 1564 if (!background_work_begin(cache)) 1565 return -EPERM; 1566 1567 mg = alloc_migration(cache); 1568 1569 mg->overwrite_bio = bio; 1570 mg->invalidate_cblock = cblock; 1571 mg->invalidate_oblock = oblock; 1572 1573 return invalidate_lock(mg); 1574} 1575 1576/* 1577 *-------------------------------------------------------------- 1578 * bio processing 1579 *-------------------------------------------------------------- 1580 */ 1581 1582enum busy { 1583 IDLE, 1584 BUSY 1585}; 1586 1587static enum busy spare_migration_bandwidth(struct cache *cache) 1588{ 1589 bool idle = dm_iot_idle_for(&cache->tracker, HZ); 1590 sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) * 1591 cache->sectors_per_block; 1592 1593 if (idle && current_volume <= cache->migration_threshold) 1594 return IDLE; 1595 else 1596 return BUSY; 1597} 1598 1599static void inc_hit_counter(struct cache *cache, struct bio *bio) 1600{ 1601 atomic_inc(bio_data_dir(bio) == READ ? 1602 &cache->stats.read_hit : &cache->stats.write_hit); 1603} 1604 1605static void inc_miss_counter(struct cache *cache, struct bio *bio) 1606{ 1607 atomic_inc(bio_data_dir(bio) == READ ? 1608 &cache->stats.read_miss : &cache->stats.write_miss); 1609} 1610 1611/*----------------------------------------------------------------*/ 1612 1613static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block, 1614 bool *commit_needed) 1615{ 1616 int r, data_dir; 1617 bool rb, background_queued; 1618 dm_cblock_t cblock; 1619 1620 *commit_needed = false; 1621 1622 rb = bio_detain_shared(cache, block, bio); 1623 if (!rb) { 1624 /* 1625 * An exclusive lock is held for this block, so we have to 1626 * wait. We set the commit_needed flag so the current 1627 * transaction will be committed asap, allowing this lock 1628 * to be dropped. 1629 */ 1630 *commit_needed = true; 1631 return DM_MAPIO_SUBMITTED; 1632 } 1633 1634 data_dir = bio_data_dir(bio); 1635 1636 if (optimisable_bio(cache, bio, block)) { 1637 struct policy_work *op = NULL; 1638 1639 r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op); 1640 if (unlikely(r && r != -ENOENT)) { 1641 DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d", 1642 cache_device_name(cache), r); 1643 bio_io_error(bio); 1644 return DM_MAPIO_SUBMITTED; 1645 } 1646 1647 if (r == -ENOENT && op) { 1648 bio_drop_shared_lock(cache, bio); 1649 BUG_ON(op->op != POLICY_PROMOTE); 1650 mg_start(cache, op, bio); 1651 return DM_MAPIO_SUBMITTED; 1652 } 1653 } else { 1654 r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued); 1655 if (unlikely(r && r != -ENOENT)) { 1656 DMERR_LIMIT("%s: policy_lookup() failed with r = %d", 1657 cache_device_name(cache), r); 1658 bio_io_error(bio); 1659 return DM_MAPIO_SUBMITTED; 1660 } 1661 1662 if (background_queued) 1663 wake_migration_worker(cache); 1664 } 1665 1666 if (r == -ENOENT) { 1667 struct per_bio_data *pb = get_per_bio_data(bio); 1668 1669 /* 1670 * Miss. 1671 */ 1672 inc_miss_counter(cache, bio); 1673 if (pb->req_nr == 0) { 1674 accounted_begin(cache, bio); 1675 remap_to_origin_clear_discard(cache, bio, block); 1676 } else { 1677 /* 1678 * This is a duplicate writethrough io that is no 1679 * longer needed because the block has been demoted. 1680 */ 1681 bio_endio(bio); 1682 return DM_MAPIO_SUBMITTED; 1683 } 1684 } else { 1685 /* 1686 * Hit. 1687 */ 1688 inc_hit_counter(cache, bio); 1689 1690 /* 1691 * Passthrough always maps to the origin, invalidating any 1692 * cache blocks that are written to. 1693 */ 1694 if (passthrough_mode(cache)) { 1695 if (bio_data_dir(bio) == WRITE) { 1696 bio_drop_shared_lock(cache, bio); 1697 atomic_inc(&cache->stats.demotion); 1698 invalidate_start(cache, cblock, block, bio); 1699 } else 1700 remap_to_origin_clear_discard(cache, bio, block); 1701 } else { 1702 if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) && 1703 !is_dirty(cache, cblock)) { 1704 remap_to_origin_and_cache(cache, bio, block, cblock); 1705 accounted_begin(cache, bio); 1706 } else 1707 remap_to_cache_dirty(cache, bio, block, cblock); 1708 } 1709 } 1710 1711 /* 1712 * dm core turns FUA requests into a separate payload and FLUSH req. 1713 */ 1714 if (bio->bi_opf & REQ_FUA) { 1715 /* 1716 * issue_after_commit will call accounted_begin a second time. So 1717 * we call accounted_complete() to avoid double accounting. 1718 */ 1719 accounted_complete(cache, bio); 1720 issue_after_commit(&cache->committer, bio); 1721 *commit_needed = true; 1722 return DM_MAPIO_SUBMITTED; 1723 } 1724 1725 return DM_MAPIO_REMAPPED; 1726} 1727 1728static bool process_bio(struct cache *cache, struct bio *bio) 1729{ 1730 bool commit_needed; 1731 1732 if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED) 1733 dm_submit_bio_remap(bio, NULL); 1734 1735 return commit_needed; 1736} 1737 1738/* 1739 * A non-zero return indicates read_only or fail_io mode. 1740 */ 1741static int commit(struct cache *cache, bool clean_shutdown) 1742{ 1743 int r; 1744 1745 if (get_cache_mode(cache) >= CM_READ_ONLY) 1746 return -EINVAL; 1747 1748 atomic_inc(&cache->stats.commit_count); 1749 r = dm_cache_commit(cache->cmd, clean_shutdown); 1750 if (r) 1751 metadata_operation_failed(cache, "dm_cache_commit", r); 1752 1753 return r; 1754} 1755 1756/* 1757 * Used by the batcher. 1758 */ 1759static blk_status_t commit_op(void *context) 1760{ 1761 struct cache *cache = context; 1762 1763 if (dm_cache_changed_this_transaction(cache->cmd)) 1764 return errno_to_blk_status(commit(cache, false)); 1765 1766 return 0; 1767} 1768 1769/*----------------------------------------------------------------*/ 1770 1771static bool process_flush_bio(struct cache *cache, struct bio *bio) 1772{ 1773 struct per_bio_data *pb = get_per_bio_data(bio); 1774 1775 if (!pb->req_nr) 1776 remap_to_origin(cache, bio); 1777 else 1778 remap_to_cache(cache, bio, 0); 1779 1780 issue_after_commit(&cache->committer, bio); 1781 return true; 1782} 1783 1784static bool process_discard_bio(struct cache *cache, struct bio *bio) 1785{ 1786 dm_dblock_t b, e; 1787 1788 /* 1789 * FIXME: do we need to lock the region? Or can we just assume the 1790 * user wont be so foolish as to issue discard concurrently with 1791 * other IO? 1792 */ 1793 calc_discard_block_range(cache, bio, &b, &e); 1794 while (b != e) { 1795 set_discard(cache, b); 1796 b = to_dblock(from_dblock(b) + 1); 1797 } 1798 1799 if (cache->features.discard_passdown) { 1800 remap_to_origin(cache, bio); 1801 dm_submit_bio_remap(bio, NULL); 1802 } else 1803 bio_endio(bio); 1804 1805 return false; 1806} 1807 1808static void process_deferred_bios(struct work_struct *ws) 1809{ 1810 struct cache *cache = container_of(ws, struct cache, deferred_bio_worker); 1811 1812 bool commit_needed = false; 1813 struct bio_list bios; 1814 struct bio *bio; 1815 1816 bio_list_init(&bios); 1817 1818 spin_lock_irq(&cache->lock); 1819 bio_list_merge(&bios, &cache->deferred_bios); 1820 bio_list_init(&cache->deferred_bios); 1821 spin_unlock_irq(&cache->lock); 1822 1823 while ((bio = bio_list_pop(&bios))) { 1824 if (bio->bi_opf & REQ_PREFLUSH) 1825 commit_needed = process_flush_bio(cache, bio) || commit_needed; 1826 1827 else if (bio_op(bio) == REQ_OP_DISCARD) 1828 commit_needed = process_discard_bio(cache, bio) || commit_needed; 1829 1830 else 1831 commit_needed = process_bio(cache, bio) || commit_needed; 1832 cond_resched(); 1833 } 1834 1835 if (commit_needed) 1836 schedule_commit(&cache->committer); 1837} 1838 1839/* 1840 *-------------------------------------------------------------- 1841 * Main worker loop 1842 *-------------------------------------------------------------- 1843 */ 1844static void requeue_deferred_bios(struct cache *cache) 1845{ 1846 struct bio *bio; 1847 struct bio_list bios; 1848 1849 bio_list_init(&bios); 1850 bio_list_merge(&bios, &cache->deferred_bios); 1851 bio_list_init(&cache->deferred_bios); 1852 1853 while ((bio = bio_list_pop(&bios))) { 1854 bio->bi_status = BLK_STS_DM_REQUEUE; 1855 bio_endio(bio); 1856 cond_resched(); 1857 } 1858} 1859 1860/* 1861 * We want to commit periodically so that not too much 1862 * unwritten metadata builds up. 1863 */ 1864static void do_waker(struct work_struct *ws) 1865{ 1866 struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker); 1867 1868 policy_tick(cache->policy, true); 1869 wake_migration_worker(cache); 1870 schedule_commit(&cache->committer); 1871 queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD); 1872} 1873 1874static void check_migrations(struct work_struct *ws) 1875{ 1876 int r; 1877 struct policy_work *op; 1878 struct cache *cache = container_of(ws, struct cache, migration_worker); 1879 enum busy b; 1880 1881 for (;;) { 1882 b = spare_migration_bandwidth(cache); 1883 1884 r = policy_get_background_work(cache->policy, b == IDLE, &op); 1885 if (r == -ENODATA) 1886 break; 1887 1888 if (r) { 1889 DMERR_LIMIT("%s: policy_background_work failed", 1890 cache_device_name(cache)); 1891 break; 1892 } 1893 1894 r = mg_start(cache, op, NULL); 1895 if (r) 1896 break; 1897 1898 cond_resched(); 1899 } 1900} 1901 1902/* 1903 *-------------------------------------------------------------- 1904 * Target methods 1905 *-------------------------------------------------------------- 1906 */ 1907 1908/* 1909 * This function gets called on the error paths of the constructor, so we 1910 * have to cope with a partially initialised struct. 1911 */ 1912static void destroy(struct cache *cache) 1913{ 1914 unsigned int i; 1915 1916 mempool_exit(&cache->migration_pool); 1917 1918 if (cache->prison) 1919 dm_bio_prison_destroy_v2(cache->prison); 1920 1921 cancel_delayed_work_sync(&cache->waker); 1922 if (cache->wq) 1923 destroy_workqueue(cache->wq); 1924 1925 if (cache->dirty_bitset) 1926 free_bitset(cache->dirty_bitset); 1927 1928 if (cache->discard_bitset) 1929 free_bitset(cache->discard_bitset); 1930 1931 if (cache->copier) 1932 dm_kcopyd_client_destroy(cache->copier); 1933 1934 if (cache->cmd) 1935 dm_cache_metadata_close(cache->cmd); 1936 1937 if (cache->metadata_dev) 1938 dm_put_device(cache->ti, cache->metadata_dev); 1939 1940 if (cache->origin_dev) 1941 dm_put_device(cache->ti, cache->origin_dev); 1942 1943 if (cache->cache_dev) 1944 dm_put_device(cache->ti, cache->cache_dev); 1945 1946 if (cache->policy) 1947 dm_cache_policy_destroy(cache->policy); 1948 1949 for (i = 0; i < cache->nr_ctr_args ; i++) 1950 kfree(cache->ctr_args[i]); 1951 kfree(cache->ctr_args); 1952 1953 bioset_exit(&cache->bs); 1954 1955 kfree(cache); 1956} 1957 1958static void cache_dtr(struct dm_target *ti) 1959{ 1960 struct cache *cache = ti->private; 1961 1962 destroy(cache); 1963} 1964 1965static sector_t get_dev_size(struct dm_dev *dev) 1966{ 1967 return bdev_nr_sectors(dev->bdev); 1968} 1969 1970/*----------------------------------------------------------------*/ 1971 1972/* 1973 * Construct a cache device mapping. 1974 * 1975 * cache <metadata dev> <cache dev> <origin dev> <block size> 1976 * <#feature args> [<feature arg>]* 1977 * <policy> <#policy args> [<policy arg>]* 1978 * 1979 * metadata dev : fast device holding the persistent metadata 1980 * cache dev : fast device holding cached data blocks 1981 * origin dev : slow device holding original data blocks 1982 * block size : cache unit size in sectors 1983 * 1984 * #feature args : number of feature arguments passed 1985 * feature args : writethrough. (The default is writeback.) 1986 * 1987 * policy : the replacement policy to use 1988 * #policy args : an even number of policy arguments corresponding 1989 * to key/value pairs passed to the policy 1990 * policy args : key/value pairs passed to the policy 1991 * E.g. 'sequential_threshold 1024' 1992 * See cache-policies.txt for details. 1993 * 1994 * Optional feature arguments are: 1995 * writethrough : write through caching that prohibits cache block 1996 * content from being different from origin block content. 1997 * Without this argument, the default behaviour is to write 1998 * back cache block contents later for performance reasons, 1999 * so they may differ from the corresponding origin blocks. 2000 */ 2001struct cache_args { 2002 struct dm_target *ti; 2003 2004 struct dm_dev *metadata_dev; 2005 2006 struct dm_dev *cache_dev; 2007 sector_t cache_sectors; 2008 2009 struct dm_dev *origin_dev; 2010 sector_t origin_sectors; 2011 2012 uint32_t block_size; 2013 2014 const char *policy_name; 2015 int policy_argc; 2016 const char **policy_argv; 2017 2018 struct cache_features features; 2019}; 2020 2021static void destroy_cache_args(struct cache_args *ca) 2022{ 2023 if (ca->metadata_dev) 2024 dm_put_device(ca->ti, ca->metadata_dev); 2025 2026 if (ca->cache_dev) 2027 dm_put_device(ca->ti, ca->cache_dev); 2028 2029 if (ca->origin_dev) 2030 dm_put_device(ca->ti, ca->origin_dev); 2031 2032 kfree(ca); 2033} 2034 2035static bool at_least_one_arg(struct dm_arg_set *as, char **error) 2036{ 2037 if (!as->argc) { 2038 *error = "Insufficient args"; 2039 return false; 2040 } 2041 2042 return true; 2043} 2044 2045static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as, 2046 char **error) 2047{ 2048 int r; 2049 sector_t metadata_dev_size; 2050 2051 if (!at_least_one_arg(as, error)) 2052 return -EINVAL; 2053 2054 r = dm_get_device(ca->ti, dm_shift_arg(as), 2055 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->metadata_dev); 2056 if (r) { 2057 *error = "Error opening metadata device"; 2058 return r; 2059 } 2060 2061 metadata_dev_size = get_dev_size(ca->metadata_dev); 2062 if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING) 2063 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.", 2064 ca->metadata_dev->bdev, THIN_METADATA_MAX_SECTORS); 2065 2066 return 0; 2067} 2068 2069static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as, 2070 char **error) 2071{ 2072 int r; 2073 2074 if (!at_least_one_arg(as, error)) 2075 return -EINVAL; 2076 2077 r = dm_get_device(ca->ti, dm_shift_arg(as), 2078 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->cache_dev); 2079 if (r) { 2080 *error = "Error opening cache device"; 2081 return r; 2082 } 2083 ca->cache_sectors = get_dev_size(ca->cache_dev); 2084 2085 return 0; 2086} 2087 2088static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as, 2089 char **error) 2090{ 2091 int r; 2092 2093 if (!at_least_one_arg(as, error)) 2094 return -EINVAL; 2095 2096 r = dm_get_device(ca->ti, dm_shift_arg(as), 2097 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->origin_dev); 2098 if (r) { 2099 *error = "Error opening origin device"; 2100 return r; 2101 } 2102 2103 ca->origin_sectors = get_dev_size(ca->origin_dev); 2104 if (ca->ti->len > ca->origin_sectors) { 2105 *error = "Device size larger than cached device"; 2106 return -EINVAL; 2107 } 2108 2109 return 0; 2110} 2111 2112static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as, 2113 char **error) 2114{ 2115 unsigned long block_size; 2116 2117 if (!at_least_one_arg(as, error)) 2118 return -EINVAL; 2119 2120 if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size || 2121 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 2122 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 2123 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { 2124 *error = "Invalid data block size"; 2125 return -EINVAL; 2126 } 2127 2128 if (block_size > ca->cache_sectors) { 2129 *error = "Data block size is larger than the cache device"; 2130 return -EINVAL; 2131 } 2132 2133 ca->block_size = block_size; 2134 2135 return 0; 2136} 2137 2138static void init_features(struct cache_features *cf) 2139{ 2140 cf->mode = CM_WRITE; 2141 cf->io_mode = CM_IO_WRITEBACK; 2142 cf->metadata_version = 1; 2143 cf->discard_passdown = true; 2144} 2145 2146static int parse_features(struct cache_args *ca, struct dm_arg_set *as, 2147 char **error) 2148{ 2149 static const struct dm_arg _args[] = { 2150 {0, 3, "Invalid number of cache feature arguments"}, 2151 }; 2152 2153 int r, mode_ctr = 0; 2154 unsigned int argc; 2155 const char *arg; 2156 struct cache_features *cf = &ca->features; 2157 2158 init_features(cf); 2159 2160 r = dm_read_arg_group(_args, as, &argc, error); 2161 if (r) 2162 return -EINVAL; 2163 2164 while (argc--) { 2165 arg = dm_shift_arg(as); 2166 2167 if (!strcasecmp(arg, "writeback")) { 2168 cf->io_mode = CM_IO_WRITEBACK; 2169 mode_ctr++; 2170 } 2171 2172 else if (!strcasecmp(arg, "writethrough")) { 2173 cf->io_mode = CM_IO_WRITETHROUGH; 2174 mode_ctr++; 2175 } 2176 2177 else if (!strcasecmp(arg, "passthrough")) { 2178 cf->io_mode = CM_IO_PASSTHROUGH; 2179 mode_ctr++; 2180 } 2181 2182 else if (!strcasecmp(arg, "metadata2")) 2183 cf->metadata_version = 2; 2184 2185 else if (!strcasecmp(arg, "no_discard_passdown")) 2186 cf->discard_passdown = false; 2187 2188 else { 2189 *error = "Unrecognised cache feature requested"; 2190 return -EINVAL; 2191 } 2192 } 2193 2194 if (mode_ctr > 1) { 2195 *error = "Duplicate cache io_mode features requested"; 2196 return -EINVAL; 2197 } 2198 2199 return 0; 2200} 2201 2202static int parse_policy(struct cache_args *ca, struct dm_arg_set *as, 2203 char **error) 2204{ 2205 static const struct dm_arg _args[] = { 2206 {0, 1024, "Invalid number of policy arguments"}, 2207 }; 2208 2209 int r; 2210 2211 if (!at_least_one_arg(as, error)) 2212 return -EINVAL; 2213 2214 ca->policy_name = dm_shift_arg(as); 2215 2216 r = dm_read_arg_group(_args, as, &ca->policy_argc, error); 2217 if (r) 2218 return -EINVAL; 2219 2220 ca->policy_argv = (const char **)as->argv; 2221 dm_consume_args(as, ca->policy_argc); 2222 2223 return 0; 2224} 2225 2226static int parse_cache_args(struct cache_args *ca, int argc, char **argv, 2227 char **error) 2228{ 2229 int r; 2230 struct dm_arg_set as; 2231 2232 as.argc = argc; 2233 as.argv = argv; 2234 2235 r = parse_metadata_dev(ca, &as, error); 2236 if (r) 2237 return r; 2238 2239 r = parse_cache_dev(ca, &as, error); 2240 if (r) 2241 return r; 2242 2243 r = parse_origin_dev(ca, &as, error); 2244 if (r) 2245 return r; 2246 2247 r = parse_block_size(ca, &as, error); 2248 if (r) 2249 return r; 2250 2251 r = parse_features(ca, &as, error); 2252 if (r) 2253 return r; 2254 2255 r = parse_policy(ca, &as, error); 2256 if (r) 2257 return r; 2258 2259 return 0; 2260} 2261 2262/*----------------------------------------------------------------*/ 2263 2264static struct kmem_cache *migration_cache; 2265 2266#define NOT_CORE_OPTION 1 2267 2268static int process_config_option(struct cache *cache, const char *key, const char *value) 2269{ 2270 unsigned long tmp; 2271 2272 if (!strcasecmp(key, "migration_threshold")) { 2273 if (kstrtoul(value, 10, &tmp)) 2274 return -EINVAL; 2275 2276 cache->migration_threshold = tmp; 2277 return 0; 2278 } 2279 2280 return NOT_CORE_OPTION; 2281} 2282 2283static int set_config_value(struct cache *cache, const char *key, const char *value) 2284{ 2285 int r = process_config_option(cache, key, value); 2286 2287 if (r == NOT_CORE_OPTION) 2288 r = policy_set_config_value(cache->policy, key, value); 2289 2290 if (r) 2291 DMWARN("bad config value for %s: %s", key, value); 2292 2293 return r; 2294} 2295 2296static int set_config_values(struct cache *cache, int argc, const char **argv) 2297{ 2298 int r = 0; 2299 2300 if (argc & 1) { 2301 DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs."); 2302 return -EINVAL; 2303 } 2304 2305 while (argc) { 2306 r = set_config_value(cache, argv[0], argv[1]); 2307 if (r) 2308 break; 2309 2310 argc -= 2; 2311 argv += 2; 2312 } 2313 2314 return r; 2315} 2316 2317static int create_cache_policy(struct cache *cache, struct cache_args *ca, 2318 char **error) 2319{ 2320 struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name, 2321 cache->cache_size, 2322 cache->origin_sectors, 2323 cache->sectors_per_block); 2324 if (IS_ERR(p)) { 2325 *error = "Error creating cache's policy"; 2326 return PTR_ERR(p); 2327 } 2328 cache->policy = p; 2329 BUG_ON(!cache->policy); 2330 2331 return 0; 2332} 2333 2334/* 2335 * We want the discard block size to be at least the size of the cache 2336 * block size and have no more than 2^14 discard blocks across the origin. 2337 */ 2338#define MAX_DISCARD_BLOCKS (1 << 14) 2339 2340static bool too_many_discard_blocks(sector_t discard_block_size, 2341 sector_t origin_size) 2342{ 2343 (void) sector_div(origin_size, discard_block_size); 2344 2345 return origin_size > MAX_DISCARD_BLOCKS; 2346} 2347 2348static sector_t calculate_discard_block_size(sector_t cache_block_size, 2349 sector_t origin_size) 2350{ 2351 sector_t discard_block_size = cache_block_size; 2352 2353 if (origin_size) 2354 while (too_many_discard_blocks(discard_block_size, origin_size)) 2355 discard_block_size *= 2; 2356 2357 return discard_block_size; 2358} 2359 2360static void set_cache_size(struct cache *cache, dm_cblock_t size) 2361{ 2362 dm_block_t nr_blocks = from_cblock(size); 2363 2364 if (nr_blocks > (1 << 20) && cache->cache_size != size) 2365 DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n" 2366 "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n" 2367 "Please consider increasing the cache block size to reduce the overall cache block count.", 2368 (unsigned long long) nr_blocks); 2369 2370 cache->cache_size = size; 2371} 2372 2373#define DEFAULT_MIGRATION_THRESHOLD 2048 2374 2375static int cache_create(struct cache_args *ca, struct cache **result) 2376{ 2377 int r = 0; 2378 char **error = &ca->ti->error; 2379 struct cache *cache; 2380 struct dm_target *ti = ca->ti; 2381 dm_block_t origin_blocks; 2382 struct dm_cache_metadata *cmd; 2383 bool may_format = ca->features.mode == CM_WRITE; 2384 2385 cache = kzalloc(sizeof(*cache), GFP_KERNEL); 2386 if (!cache) 2387 return -ENOMEM; 2388 2389 cache->ti = ca->ti; 2390 ti->private = cache; 2391 ti->accounts_remapped_io = true; 2392 ti->num_flush_bios = 2; 2393 ti->flush_supported = true; 2394 2395 ti->num_discard_bios = 1; 2396 ti->discards_supported = true; 2397 2398 ti->per_io_data_size = sizeof(struct per_bio_data); 2399 2400 cache->features = ca->features; 2401 if (writethrough_mode(cache)) { 2402 /* Create bioset for writethrough bios issued to origin */ 2403 r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0); 2404 if (r) 2405 goto bad; 2406 } 2407 2408 cache->metadata_dev = ca->metadata_dev; 2409 cache->origin_dev = ca->origin_dev; 2410 cache->cache_dev = ca->cache_dev; 2411 2412 ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL; 2413 2414 origin_blocks = cache->origin_sectors = ca->origin_sectors; 2415 origin_blocks = block_div(origin_blocks, ca->block_size); 2416 cache->origin_blocks = to_oblock(origin_blocks); 2417 2418 cache->sectors_per_block = ca->block_size; 2419 if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) { 2420 r = -EINVAL; 2421 goto bad; 2422 } 2423 2424 if (ca->block_size & (ca->block_size - 1)) { 2425 dm_block_t cache_size = ca->cache_sectors; 2426 2427 cache->sectors_per_block_shift = -1; 2428 cache_size = block_div(cache_size, ca->block_size); 2429 set_cache_size(cache, to_cblock(cache_size)); 2430 } else { 2431 cache->sectors_per_block_shift = __ffs(ca->block_size); 2432 set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift)); 2433 } 2434 2435 r = create_cache_policy(cache, ca, error); 2436 if (r) 2437 goto bad; 2438 2439 cache->policy_nr_args = ca->policy_argc; 2440 cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD; 2441 2442 r = set_config_values(cache, ca->policy_argc, ca->policy_argv); 2443 if (r) { 2444 *error = "Error setting cache policy's config values"; 2445 goto bad; 2446 } 2447 2448 cmd = dm_cache_metadata_open(cache->metadata_dev->bdev, 2449 ca->block_size, may_format, 2450 dm_cache_policy_get_hint_size(cache->policy), 2451 ca->features.metadata_version); 2452 if (IS_ERR(cmd)) { 2453 *error = "Error creating metadata object"; 2454 r = PTR_ERR(cmd); 2455 goto bad; 2456 } 2457 cache->cmd = cmd; 2458 set_cache_mode(cache, CM_WRITE); 2459 if (get_cache_mode(cache) != CM_WRITE) { 2460 *error = "Unable to get write access to metadata, please check/repair metadata."; 2461 r = -EINVAL; 2462 goto bad; 2463 } 2464 2465 if (passthrough_mode(cache)) { 2466 bool all_clean; 2467 2468 r = dm_cache_metadata_all_clean(cache->cmd, &all_clean); 2469 if (r) { 2470 *error = "dm_cache_metadata_all_clean() failed"; 2471 goto bad; 2472 } 2473 2474 if (!all_clean) { 2475 *error = "Cannot enter passthrough mode unless all blocks are clean"; 2476 r = -EINVAL; 2477 goto bad; 2478 } 2479 2480 policy_allow_migrations(cache->policy, false); 2481 } 2482 2483 spin_lock_init(&cache->lock); 2484 bio_list_init(&cache->deferred_bios); 2485 atomic_set(&cache->nr_allocated_migrations, 0); 2486 atomic_set(&cache->nr_io_migrations, 0); 2487 init_waitqueue_head(&cache->migration_wait); 2488 2489 r = -ENOMEM; 2490 atomic_set(&cache->nr_dirty, 0); 2491 cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size)); 2492 if (!cache->dirty_bitset) { 2493 *error = "could not allocate dirty bitset"; 2494 goto bad; 2495 } 2496 clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size)); 2497 2498 cache->discard_block_size = 2499 calculate_discard_block_size(cache->sectors_per_block, 2500 cache->origin_sectors); 2501 cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors, 2502 cache->discard_block_size)); 2503 cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks)); 2504 if (!cache->discard_bitset) { 2505 *error = "could not allocate discard bitset"; 2506 goto bad; 2507 } 2508 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 2509 2510 cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2511 if (IS_ERR(cache->copier)) { 2512 *error = "could not create kcopyd client"; 2513 r = PTR_ERR(cache->copier); 2514 goto bad; 2515 } 2516 2517 cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0); 2518 if (!cache->wq) { 2519 *error = "could not create workqueue for metadata object"; 2520 goto bad; 2521 } 2522 INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios); 2523 INIT_WORK(&cache->migration_worker, check_migrations); 2524 INIT_DELAYED_WORK(&cache->waker, do_waker); 2525 2526 cache->prison = dm_bio_prison_create_v2(cache->wq); 2527 if (!cache->prison) { 2528 *error = "could not create bio prison"; 2529 goto bad; 2530 } 2531 2532 r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE, 2533 migration_cache); 2534 if (r) { 2535 *error = "Error creating cache's migration mempool"; 2536 goto bad; 2537 } 2538 2539 cache->need_tick_bio = true; 2540 cache->sized = false; 2541 cache->invalidate = false; 2542 cache->commit_requested = false; 2543 cache->loaded_mappings = false; 2544 cache->loaded_discards = false; 2545 2546 load_stats(cache); 2547 2548 atomic_set(&cache->stats.demotion, 0); 2549 atomic_set(&cache->stats.promotion, 0); 2550 atomic_set(&cache->stats.copies_avoided, 0); 2551 atomic_set(&cache->stats.cache_cell_clash, 0); 2552 atomic_set(&cache->stats.commit_count, 0); 2553 atomic_set(&cache->stats.discard_count, 0); 2554 2555 spin_lock_init(&cache->invalidation_lock); 2556 INIT_LIST_HEAD(&cache->invalidation_requests); 2557 2558 batcher_init(&cache->committer, commit_op, cache, 2559 issue_op, cache, cache->wq); 2560 dm_iot_init(&cache->tracker); 2561 2562 init_rwsem(&cache->background_work_lock); 2563 prevent_background_work(cache); 2564 2565 *result = cache; 2566 return 0; 2567bad: 2568 destroy(cache); 2569 return r; 2570} 2571 2572static int copy_ctr_args(struct cache *cache, int argc, const char **argv) 2573{ 2574 unsigned int i; 2575 const char **copy; 2576 2577 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL); 2578 if (!copy) 2579 return -ENOMEM; 2580 for (i = 0; i < argc; i++) { 2581 copy[i] = kstrdup(argv[i], GFP_KERNEL); 2582 if (!copy[i]) { 2583 while (i--) 2584 kfree(copy[i]); 2585 kfree(copy); 2586 return -ENOMEM; 2587 } 2588 } 2589 2590 cache->nr_ctr_args = argc; 2591 cache->ctr_args = copy; 2592 2593 return 0; 2594} 2595 2596static int cache_ctr(struct dm_target *ti, unsigned int argc, char **argv) 2597{ 2598 int r = -EINVAL; 2599 struct cache_args *ca; 2600 struct cache *cache = NULL; 2601 2602 ca = kzalloc(sizeof(*ca), GFP_KERNEL); 2603 if (!ca) { 2604 ti->error = "Error allocating memory for cache"; 2605 return -ENOMEM; 2606 } 2607 ca->ti = ti; 2608 2609 r = parse_cache_args(ca, argc, argv, &ti->error); 2610 if (r) 2611 goto out; 2612 2613 r = cache_create(ca, &cache); 2614 if (r) 2615 goto out; 2616 2617 r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3); 2618 if (r) { 2619 destroy(cache); 2620 goto out; 2621 } 2622 2623 ti->private = cache; 2624out: 2625 destroy_cache_args(ca); 2626 return r; 2627} 2628 2629/*----------------------------------------------------------------*/ 2630 2631static int cache_map(struct dm_target *ti, struct bio *bio) 2632{ 2633 struct cache *cache = ti->private; 2634 2635 int r; 2636 bool commit_needed; 2637 dm_oblock_t block = get_bio_block(cache, bio); 2638 2639 init_per_bio_data(bio); 2640 if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) { 2641 /* 2642 * This can only occur if the io goes to a partial block at 2643 * the end of the origin device. We don't cache these. 2644 * Just remap to the origin and carry on. 2645 */ 2646 remap_to_origin(cache, bio); 2647 accounted_begin(cache, bio); 2648 return DM_MAPIO_REMAPPED; 2649 } 2650 2651 if (discard_or_flush(bio)) { 2652 defer_bio(cache, bio); 2653 return DM_MAPIO_SUBMITTED; 2654 } 2655 2656 r = map_bio(cache, bio, block, &commit_needed); 2657 if (commit_needed) 2658 schedule_commit(&cache->committer); 2659 2660 return r; 2661} 2662 2663static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error) 2664{ 2665 struct cache *cache = ti->private; 2666 unsigned long flags; 2667 struct per_bio_data *pb = get_per_bio_data(bio); 2668 2669 if (pb->tick) { 2670 policy_tick(cache->policy, false); 2671 2672 spin_lock_irqsave(&cache->lock, flags); 2673 cache->need_tick_bio = true; 2674 spin_unlock_irqrestore(&cache->lock, flags); 2675 } 2676 2677 bio_drop_shared_lock(cache, bio); 2678 accounted_complete(cache, bio); 2679 2680 return DM_ENDIO_DONE; 2681} 2682 2683static int write_dirty_bitset(struct cache *cache) 2684{ 2685 int r; 2686 2687 if (get_cache_mode(cache) >= CM_READ_ONLY) 2688 return -EINVAL; 2689 2690 r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset); 2691 if (r) 2692 metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r); 2693 2694 return r; 2695} 2696 2697static int write_discard_bitset(struct cache *cache) 2698{ 2699 unsigned int i, r; 2700 2701 if (get_cache_mode(cache) >= CM_READ_ONLY) 2702 return -EINVAL; 2703 2704 r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size, 2705 cache->discard_nr_blocks); 2706 if (r) { 2707 DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache)); 2708 metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r); 2709 return r; 2710 } 2711 2712 for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) { 2713 r = dm_cache_set_discard(cache->cmd, to_dblock(i), 2714 is_discarded(cache, to_dblock(i))); 2715 if (r) { 2716 metadata_operation_failed(cache, "dm_cache_set_discard", r); 2717 return r; 2718 } 2719 } 2720 2721 return 0; 2722} 2723 2724static int write_hints(struct cache *cache) 2725{ 2726 int r; 2727 2728 if (get_cache_mode(cache) >= CM_READ_ONLY) 2729 return -EINVAL; 2730 2731 r = dm_cache_write_hints(cache->cmd, cache->policy); 2732 if (r) { 2733 metadata_operation_failed(cache, "dm_cache_write_hints", r); 2734 return r; 2735 } 2736 2737 return 0; 2738} 2739 2740/* 2741 * returns true on success 2742 */ 2743static bool sync_metadata(struct cache *cache) 2744{ 2745 int r1, r2, r3, r4; 2746 2747 r1 = write_dirty_bitset(cache); 2748 if (r1) 2749 DMERR("%s: could not write dirty bitset", cache_device_name(cache)); 2750 2751 r2 = write_discard_bitset(cache); 2752 if (r2) 2753 DMERR("%s: could not write discard bitset", cache_device_name(cache)); 2754 2755 save_stats(cache); 2756 2757 r3 = write_hints(cache); 2758 if (r3) 2759 DMERR("%s: could not write hints", cache_device_name(cache)); 2760 2761 /* 2762 * If writing the above metadata failed, we still commit, but don't 2763 * set the clean shutdown flag. This will effectively force every 2764 * dirty bit to be set on reload. 2765 */ 2766 r4 = commit(cache, !r1 && !r2 && !r3); 2767 if (r4) 2768 DMERR("%s: could not write cache metadata", cache_device_name(cache)); 2769 2770 return !r1 && !r2 && !r3 && !r4; 2771} 2772 2773static void cache_postsuspend(struct dm_target *ti) 2774{ 2775 struct cache *cache = ti->private; 2776 2777 prevent_background_work(cache); 2778 BUG_ON(atomic_read(&cache->nr_io_migrations)); 2779 2780 cancel_delayed_work_sync(&cache->waker); 2781 drain_workqueue(cache->wq); 2782 WARN_ON(cache->tracker.in_flight); 2783 2784 /* 2785 * If it's a flush suspend there won't be any deferred bios, so this 2786 * call is harmless. 2787 */ 2788 requeue_deferred_bios(cache); 2789 2790 if (get_cache_mode(cache) == CM_WRITE) 2791 (void) sync_metadata(cache); 2792} 2793 2794static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, 2795 bool dirty, uint32_t hint, bool hint_valid) 2796{ 2797 struct cache *cache = context; 2798 2799 if (dirty) { 2800 set_bit(from_cblock(cblock), cache->dirty_bitset); 2801 atomic_inc(&cache->nr_dirty); 2802 } else 2803 clear_bit(from_cblock(cblock), cache->dirty_bitset); 2804 2805 return policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid); 2806} 2807 2808/* 2809 * The discard block size in the on disk metadata is not 2810 * necessarily the same as we're currently using. So we have to 2811 * be careful to only set the discarded attribute if we know it 2812 * covers a complete block of the new size. 2813 */ 2814struct discard_load_info { 2815 struct cache *cache; 2816 2817 /* 2818 * These blocks are sized using the on disk dblock size, rather 2819 * than the current one. 2820 */ 2821 dm_block_t block_size; 2822 dm_block_t discard_begin, discard_end; 2823}; 2824 2825static void discard_load_info_init(struct cache *cache, 2826 struct discard_load_info *li) 2827{ 2828 li->cache = cache; 2829 li->discard_begin = li->discard_end = 0; 2830} 2831 2832static void set_discard_range(struct discard_load_info *li) 2833{ 2834 sector_t b, e; 2835 2836 if (li->discard_begin == li->discard_end) 2837 return; 2838 2839 /* 2840 * Convert to sectors. 2841 */ 2842 b = li->discard_begin * li->block_size; 2843 e = li->discard_end * li->block_size; 2844 2845 /* 2846 * Then convert back to the current dblock size. 2847 */ 2848 b = dm_sector_div_up(b, li->cache->discard_block_size); 2849 sector_div(e, li->cache->discard_block_size); 2850 2851 /* 2852 * The origin may have shrunk, so we need to check we're still in 2853 * bounds. 2854 */ 2855 if (e > from_dblock(li->cache->discard_nr_blocks)) 2856 e = from_dblock(li->cache->discard_nr_blocks); 2857 2858 for (; b < e; b++) 2859 set_discard(li->cache, to_dblock(b)); 2860} 2861 2862static int load_discard(void *context, sector_t discard_block_size, 2863 dm_dblock_t dblock, bool discard) 2864{ 2865 struct discard_load_info *li = context; 2866 2867 li->block_size = discard_block_size; 2868 2869 if (discard) { 2870 if (from_dblock(dblock) == li->discard_end) 2871 /* 2872 * We're already in a discard range, just extend it. 2873 */ 2874 li->discard_end = li->discard_end + 1ULL; 2875 2876 else { 2877 /* 2878 * Emit the old range and start a new one. 2879 */ 2880 set_discard_range(li); 2881 li->discard_begin = from_dblock(dblock); 2882 li->discard_end = li->discard_begin + 1ULL; 2883 } 2884 } else { 2885 set_discard_range(li); 2886 li->discard_begin = li->discard_end = 0; 2887 } 2888 2889 return 0; 2890} 2891 2892static dm_cblock_t get_cache_dev_size(struct cache *cache) 2893{ 2894 sector_t size = get_dev_size(cache->cache_dev); 2895 (void) sector_div(size, cache->sectors_per_block); 2896 return to_cblock(size); 2897} 2898 2899static bool can_resize(struct cache *cache, dm_cblock_t new_size) 2900{ 2901 if (from_cblock(new_size) > from_cblock(cache->cache_size)) { 2902 if (cache->sized) { 2903 DMERR("%s: unable to extend cache due to missing cache table reload", 2904 cache_device_name(cache)); 2905 return false; 2906 } 2907 } 2908 2909 /* 2910 * We can't drop a dirty block when shrinking the cache. 2911 */ 2912 while (from_cblock(new_size) < from_cblock(cache->cache_size)) { 2913 new_size = to_cblock(from_cblock(new_size) + 1); 2914 if (is_dirty(cache, new_size)) { 2915 DMERR("%s: unable to shrink cache; cache block %llu is dirty", 2916 cache_device_name(cache), 2917 (unsigned long long) from_cblock(new_size)); 2918 return false; 2919 } 2920 } 2921 2922 return true; 2923} 2924 2925static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size) 2926{ 2927 int r; 2928 2929 r = dm_cache_resize(cache->cmd, new_size); 2930 if (r) { 2931 DMERR("%s: could not resize cache metadata", cache_device_name(cache)); 2932 metadata_operation_failed(cache, "dm_cache_resize", r); 2933 return r; 2934 } 2935 2936 set_cache_size(cache, new_size); 2937 2938 return 0; 2939} 2940 2941static int cache_preresume(struct dm_target *ti) 2942{ 2943 int r = 0; 2944 struct cache *cache = ti->private; 2945 dm_cblock_t csize = get_cache_dev_size(cache); 2946 2947 /* 2948 * Check to see if the cache has resized. 2949 */ 2950 if (!cache->sized) { 2951 r = resize_cache_dev(cache, csize); 2952 if (r) 2953 return r; 2954 2955 cache->sized = true; 2956 2957 } else if (csize != cache->cache_size) { 2958 if (!can_resize(cache, csize)) 2959 return -EINVAL; 2960 2961 r = resize_cache_dev(cache, csize); 2962 if (r) 2963 return r; 2964 } 2965 2966 if (!cache->loaded_mappings) { 2967 r = dm_cache_load_mappings(cache->cmd, cache->policy, 2968 load_mapping, cache); 2969 if (r) { 2970 DMERR("%s: could not load cache mappings", cache_device_name(cache)); 2971 metadata_operation_failed(cache, "dm_cache_load_mappings", r); 2972 return r; 2973 } 2974 2975 cache->loaded_mappings = true; 2976 } 2977 2978 if (!cache->loaded_discards) { 2979 struct discard_load_info li; 2980 2981 /* 2982 * The discard bitset could have been resized, or the 2983 * discard block size changed. To be safe we start by 2984 * setting every dblock to not discarded. 2985 */ 2986 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 2987 2988 discard_load_info_init(cache, &li); 2989 r = dm_cache_load_discards(cache->cmd, load_discard, &li); 2990 if (r) { 2991 DMERR("%s: could not load origin discards", cache_device_name(cache)); 2992 metadata_operation_failed(cache, "dm_cache_load_discards", r); 2993 return r; 2994 } 2995 set_discard_range(&li); 2996 2997 cache->loaded_discards = true; 2998 } 2999 3000 return r; 3001} 3002 3003static void cache_resume(struct dm_target *ti) 3004{ 3005 struct cache *cache = ti->private; 3006 3007 cache->need_tick_bio = true; 3008 allow_background_work(cache); 3009 do_waker(&cache->waker.work); 3010} 3011 3012static void emit_flags(struct cache *cache, char *result, 3013 unsigned int maxlen, ssize_t *sz_ptr) 3014{ 3015 ssize_t sz = *sz_ptr; 3016 struct cache_features *cf = &cache->features; 3017 unsigned int count = (cf->metadata_version == 2) + !cf->discard_passdown + 1; 3018 3019 DMEMIT("%u ", count); 3020 3021 if (cf->metadata_version == 2) 3022 DMEMIT("metadata2 "); 3023 3024 if (writethrough_mode(cache)) 3025 DMEMIT("writethrough "); 3026 3027 else if (passthrough_mode(cache)) 3028 DMEMIT("passthrough "); 3029 3030 else if (writeback_mode(cache)) 3031 DMEMIT("writeback "); 3032 3033 else { 3034 DMEMIT("unknown "); 3035 DMERR("%s: internal error: unknown io mode: %d", 3036 cache_device_name(cache), (int) cf->io_mode); 3037 } 3038 3039 if (!cf->discard_passdown) 3040 DMEMIT("no_discard_passdown "); 3041 3042 *sz_ptr = sz; 3043} 3044 3045/* 3046 * Status format: 3047 * 3048 * <metadata block size> <#used metadata blocks>/<#total metadata blocks> 3049 * <cache block size> <#used cache blocks>/<#total cache blocks> 3050 * <#read hits> <#read misses> <#write hits> <#write misses> 3051 * <#demotions> <#promotions> <#dirty> 3052 * <#features> <features>* 3053 * <#core args> <core args> 3054 * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check> 3055 */ 3056static void cache_status(struct dm_target *ti, status_type_t type, 3057 unsigned int status_flags, char *result, unsigned int maxlen) 3058{ 3059 int r = 0; 3060 unsigned int i; 3061 ssize_t sz = 0; 3062 dm_block_t nr_free_blocks_metadata = 0; 3063 dm_block_t nr_blocks_metadata = 0; 3064 char buf[BDEVNAME_SIZE]; 3065 struct cache *cache = ti->private; 3066 dm_cblock_t residency; 3067 bool needs_check; 3068 3069 switch (type) { 3070 case STATUSTYPE_INFO: 3071 if (get_cache_mode(cache) == CM_FAIL) { 3072 DMEMIT("Fail"); 3073 break; 3074 } 3075 3076 /* Commit to ensure statistics aren't out-of-date */ 3077 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 3078 (void) commit(cache, false); 3079 3080 r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata); 3081 if (r) { 3082 DMERR("%s: dm_cache_get_free_metadata_block_count returned %d", 3083 cache_device_name(cache), r); 3084 goto err; 3085 } 3086 3087 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata); 3088 if (r) { 3089 DMERR("%s: dm_cache_get_metadata_dev_size returned %d", 3090 cache_device_name(cache), r); 3091 goto err; 3092 } 3093 3094 residency = policy_residency(cache->policy); 3095 3096 DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ", 3097 (unsigned int)DM_CACHE_METADATA_BLOCK_SIZE, 3098 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 3099 (unsigned long long)nr_blocks_metadata, 3100 (unsigned long long)cache->sectors_per_block, 3101 (unsigned long long) from_cblock(residency), 3102 (unsigned long long) from_cblock(cache->cache_size), 3103 (unsigned int) atomic_read(&cache->stats.read_hit), 3104 (unsigned int) atomic_read(&cache->stats.read_miss), 3105 (unsigned int) atomic_read(&cache->stats.write_hit), 3106 (unsigned int) atomic_read(&cache->stats.write_miss), 3107 (unsigned int) atomic_read(&cache->stats.demotion), 3108 (unsigned int) atomic_read(&cache->stats.promotion), 3109 (unsigned long) atomic_read(&cache->nr_dirty)); 3110 3111 emit_flags(cache, result, maxlen, &sz); 3112 3113 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold); 3114 3115 DMEMIT("%s ", dm_cache_policy_get_name(cache->policy)); 3116 if (sz < maxlen) { 3117 r = policy_emit_config_values(cache->policy, result, maxlen, &sz); 3118 if (r) 3119 DMERR("%s: policy_emit_config_values returned %d", 3120 cache_device_name(cache), r); 3121 } 3122 3123 if (get_cache_mode(cache) == CM_READ_ONLY) 3124 DMEMIT("ro "); 3125 else 3126 DMEMIT("rw "); 3127 3128 r = dm_cache_metadata_needs_check(cache->cmd, &needs_check); 3129 3130 if (r || needs_check) 3131 DMEMIT("needs_check "); 3132 else 3133 DMEMIT("- "); 3134 3135 break; 3136 3137 case STATUSTYPE_TABLE: 3138 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); 3139 DMEMIT("%s ", buf); 3140 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); 3141 DMEMIT("%s ", buf); 3142 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); 3143 DMEMIT("%s", buf); 3144 3145 for (i = 0; i < cache->nr_ctr_args - 1; i++) 3146 DMEMIT(" %s", cache->ctr_args[i]); 3147 if (cache->nr_ctr_args) 3148 DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]); 3149 break; 3150 3151 case STATUSTYPE_IMA: 3152 DMEMIT_TARGET_NAME_VERSION(ti->type); 3153 if (get_cache_mode(cache) == CM_FAIL) 3154 DMEMIT(",metadata_mode=fail"); 3155 else if (get_cache_mode(cache) == CM_READ_ONLY) 3156 DMEMIT(",metadata_mode=ro"); 3157 else 3158 DMEMIT(",metadata_mode=rw"); 3159 3160 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); 3161 DMEMIT(",cache_metadata_device=%s", buf); 3162 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); 3163 DMEMIT(",cache_device=%s", buf); 3164 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); 3165 DMEMIT(",cache_origin_device=%s", buf); 3166 DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n'); 3167 DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n'); 3168 DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n'); 3169 DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n'); 3170 DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y'); 3171 DMEMIT(";"); 3172 break; 3173 } 3174 3175 return; 3176 3177err: 3178 DMEMIT("Error"); 3179} 3180 3181/* 3182 * Defines a range of cblocks, begin to (end - 1) are in the range. end is 3183 * the one-past-the-end value. 3184 */ 3185struct cblock_range { 3186 dm_cblock_t begin; 3187 dm_cblock_t end; 3188}; 3189 3190/* 3191 * A cache block range can take two forms: 3192 * 3193 * i) A single cblock, eg. '3456' 3194 * ii) A begin and end cblock with a dash between, eg. 123-234 3195 */ 3196static int parse_cblock_range(struct cache *cache, const char *str, 3197 struct cblock_range *result) 3198{ 3199 char dummy; 3200 uint64_t b, e; 3201 int r; 3202 3203 /* 3204 * Try and parse form (ii) first. 3205 */ 3206 r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy); 3207 if (r < 0) 3208 return r; 3209 3210 if (r == 2) { 3211 result->begin = to_cblock(b); 3212 result->end = to_cblock(e); 3213 return 0; 3214 } 3215 3216 /* 3217 * That didn't work, try form (i). 3218 */ 3219 r = sscanf(str, "%llu%c", &b, &dummy); 3220 if (r < 0) 3221 return r; 3222 3223 if (r == 1) { 3224 result->begin = to_cblock(b); 3225 result->end = to_cblock(from_cblock(result->begin) + 1u); 3226 return 0; 3227 } 3228 3229 DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str); 3230 return -EINVAL; 3231} 3232 3233static int validate_cblock_range(struct cache *cache, struct cblock_range *range) 3234{ 3235 uint64_t b = from_cblock(range->begin); 3236 uint64_t e = from_cblock(range->end); 3237 uint64_t n = from_cblock(cache->cache_size); 3238 3239 if (b >= n) { 3240 DMERR("%s: begin cblock out of range: %llu >= %llu", 3241 cache_device_name(cache), b, n); 3242 return -EINVAL; 3243 } 3244 3245 if (e > n) { 3246 DMERR("%s: end cblock out of range: %llu > %llu", 3247 cache_device_name(cache), e, n); 3248 return -EINVAL; 3249 } 3250 3251 if (b >= e) { 3252 DMERR("%s: invalid cblock range: %llu >= %llu", 3253 cache_device_name(cache), b, e); 3254 return -EINVAL; 3255 } 3256 3257 return 0; 3258} 3259 3260static inline dm_cblock_t cblock_succ(dm_cblock_t b) 3261{ 3262 return to_cblock(from_cblock(b) + 1); 3263} 3264 3265static int request_invalidation(struct cache *cache, struct cblock_range *range) 3266{ 3267 int r = 0; 3268 3269 /* 3270 * We don't need to do any locking here because we know we're in 3271 * passthrough mode. There's is potential for a race between an 3272 * invalidation triggered by an io and an invalidation message. This 3273 * is harmless, we must not worry if the policy call fails. 3274 */ 3275 while (range->begin != range->end) { 3276 r = invalidate_cblock(cache, range->begin); 3277 if (r) 3278 return r; 3279 3280 range->begin = cblock_succ(range->begin); 3281 } 3282 3283 cache->commit_requested = true; 3284 return r; 3285} 3286 3287static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count, 3288 const char **cblock_ranges) 3289{ 3290 int r = 0; 3291 unsigned int i; 3292 struct cblock_range range; 3293 3294 if (!passthrough_mode(cache)) { 3295 DMERR("%s: cache has to be in passthrough mode for invalidation", 3296 cache_device_name(cache)); 3297 return -EPERM; 3298 } 3299 3300 for (i = 0; i < count; i++) { 3301 r = parse_cblock_range(cache, cblock_ranges[i], &range); 3302 if (r) 3303 break; 3304 3305 r = validate_cblock_range(cache, &range); 3306 if (r) 3307 break; 3308 3309 /* 3310 * Pass begin and end origin blocks to the worker and wake it. 3311 */ 3312 r = request_invalidation(cache, &range); 3313 if (r) 3314 break; 3315 } 3316 3317 return r; 3318} 3319 3320/* 3321 * Supports 3322 * "<key> <value>" 3323 * and 3324 * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]* 3325 * 3326 * The key migration_threshold is supported by the cache target core. 3327 */ 3328static int cache_message(struct dm_target *ti, unsigned int argc, char **argv, 3329 char *result, unsigned int maxlen) 3330{ 3331 struct cache *cache = ti->private; 3332 3333 if (!argc) 3334 return -EINVAL; 3335 3336 if (get_cache_mode(cache) >= CM_READ_ONLY) { 3337 DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode", 3338 cache_device_name(cache)); 3339 return -EOPNOTSUPP; 3340 } 3341 3342 if (!strcasecmp(argv[0], "invalidate_cblocks")) 3343 return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1); 3344 3345 if (argc != 2) 3346 return -EINVAL; 3347 3348 return set_config_value(cache, argv[0], argv[1]); 3349} 3350 3351static int cache_iterate_devices(struct dm_target *ti, 3352 iterate_devices_callout_fn fn, void *data) 3353{ 3354 int r = 0; 3355 struct cache *cache = ti->private; 3356 3357 r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data); 3358 if (!r) 3359 r = fn(ti, cache->origin_dev, 0, ti->len, data); 3360 3361 return r; 3362} 3363 3364/* 3365 * If discard_passdown was enabled verify that the origin device 3366 * supports discards. Disable discard_passdown if not. 3367 */ 3368static void disable_passdown_if_not_supported(struct cache *cache) 3369{ 3370 struct block_device *origin_bdev = cache->origin_dev->bdev; 3371 struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits; 3372 const char *reason = NULL; 3373 3374 if (!cache->features.discard_passdown) 3375 return; 3376 3377 if (!bdev_max_discard_sectors(origin_bdev)) 3378 reason = "discard unsupported"; 3379 3380 else if (origin_limits->max_discard_sectors < cache->sectors_per_block) 3381 reason = "max discard sectors smaller than a block"; 3382 3383 if (reason) { 3384 DMWARN("Origin device (%pg) %s: Disabling discard passdown.", 3385 origin_bdev, reason); 3386 cache->features.discard_passdown = false; 3387 } 3388} 3389 3390static void set_discard_limits(struct cache *cache, struct queue_limits *limits) 3391{ 3392 struct block_device *origin_bdev = cache->origin_dev->bdev; 3393 struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits; 3394 3395 if (!cache->features.discard_passdown) { 3396 /* No passdown is done so setting own virtual limits */ 3397 limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024, 3398 cache->origin_sectors); 3399 limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT; 3400 return; 3401 } 3402 3403 /* 3404 * cache_iterate_devices() is stacking both origin and fast device limits 3405 * but discards aren't passed to fast device, so inherit origin's limits. 3406 */ 3407 limits->max_discard_sectors = origin_limits->max_discard_sectors; 3408 limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors; 3409 limits->discard_granularity = origin_limits->discard_granularity; 3410 limits->discard_alignment = origin_limits->discard_alignment; 3411 limits->discard_misaligned = origin_limits->discard_misaligned; 3412} 3413 3414static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits) 3415{ 3416 struct cache *cache = ti->private; 3417 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 3418 3419 /* 3420 * If the system-determined stacked limits are compatible with the 3421 * cache's blocksize (io_opt is a factor) do not override them. 3422 */ 3423 if (io_opt_sectors < cache->sectors_per_block || 3424 do_div(io_opt_sectors, cache->sectors_per_block)) { 3425 blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT); 3426 blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT); 3427 } 3428 3429 disable_passdown_if_not_supported(cache); 3430 set_discard_limits(cache, limits); 3431} 3432 3433/*----------------------------------------------------------------*/ 3434 3435static struct target_type cache_target = { 3436 .name = "cache", 3437 .version = {2, 2, 0}, 3438 .module = THIS_MODULE, 3439 .ctr = cache_ctr, 3440 .dtr = cache_dtr, 3441 .map = cache_map, 3442 .end_io = cache_end_io, 3443 .postsuspend = cache_postsuspend, 3444 .preresume = cache_preresume, 3445 .resume = cache_resume, 3446 .status = cache_status, 3447 .message = cache_message, 3448 .iterate_devices = cache_iterate_devices, 3449 .io_hints = cache_io_hints, 3450}; 3451 3452static int __init dm_cache_init(void) 3453{ 3454 int r; 3455 3456 migration_cache = KMEM_CACHE(dm_cache_migration, 0); 3457 if (!migration_cache) 3458 return -ENOMEM; 3459 3460 r = dm_register_target(&cache_target); 3461 if (r) { 3462 kmem_cache_destroy(migration_cache); 3463 return r; 3464 } 3465 3466 return 0; 3467} 3468 3469static void __exit dm_cache_exit(void) 3470{ 3471 dm_unregister_target(&cache_target); 3472 kmem_cache_destroy(migration_cache); 3473} 3474 3475module_init(dm_cache_init); 3476module_exit(dm_cache_exit); 3477 3478MODULE_DESCRIPTION(DM_NAME " cache target"); 3479MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>"); 3480MODULE_LICENSE("GPL"); 3481