1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2019 Arrikto, Inc. All Rights Reserved. 4 */ 5 6#include <linux/mm.h> 7#include <linux/bio.h> 8#include <linux/err.h> 9#include <linux/hash.h> 10#include <linux/list.h> 11#include <linux/log2.h> 12#include <linux/init.h> 13#include <linux/slab.h> 14#include <linux/wait.h> 15#include <linux/dm-io.h> 16#include <linux/mutex.h> 17#include <linux/atomic.h> 18#include <linux/bitops.h> 19#include <linux/blkdev.h> 20#include <linux/kdev_t.h> 21#include <linux/kernel.h> 22#include <linux/module.h> 23#include <linux/jiffies.h> 24#include <linux/mempool.h> 25#include <linux/spinlock.h> 26#include <linux/blk_types.h> 27#include <linux/dm-kcopyd.h> 28#include <linux/workqueue.h> 29#include <linux/backing-dev.h> 30#include <linux/device-mapper.h> 31 32#include "dm.h" 33#include "dm-clone-metadata.h" 34 35#define DM_MSG_PREFIX "clone" 36 37/* 38 * Minimum and maximum allowed region sizes 39 */ 40#define MIN_REGION_SIZE (1 << 3) /* 4KB */ 41#define MAX_REGION_SIZE (1 << 21) /* 1GB */ 42 43#define MIN_HYDRATIONS 256 /* Size of hydration mempool */ 44#define DEFAULT_HYDRATION_THRESHOLD 1 /* 1 region */ 45#define DEFAULT_HYDRATION_BATCH_SIZE 1 /* Hydrate in batches of 1 region */ 46 47#define COMMIT_PERIOD HZ /* 1 sec */ 48 49/* 50 * Hydration hash table size: 1 << HASH_TABLE_BITS 51 */ 52#define HASH_TABLE_BITS 15 53 54DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(clone_hydration_throttle, 55 "A percentage of time allocated for hydrating regions"); 56 57/* Slab cache for struct dm_clone_region_hydration */ 58static struct kmem_cache *_hydration_cache; 59 60/* dm-clone metadata modes */ 61enum clone_metadata_mode { 62 CM_WRITE, /* metadata may be changed */ 63 CM_READ_ONLY, /* metadata may not be changed */ 64 CM_FAIL, /* all metadata I/O fails */ 65}; 66 67struct hash_table_bucket; 68 69struct clone { 70 struct dm_target *ti; 71 72 struct dm_dev *metadata_dev; 73 struct dm_dev *dest_dev; 74 struct dm_dev *source_dev; 75 76 unsigned long nr_regions; 77 sector_t region_size; 78 unsigned int region_shift; 79 80 /* 81 * A metadata commit and the actions taken in case it fails should run 82 * as a single atomic step. 83 */ 84 struct mutex commit_lock; 85 86 struct dm_clone_metadata *cmd; 87 88 /* Region hydration hash table */ 89 struct hash_table_bucket *ht; 90 91 atomic_t ios_in_flight; 92 93 wait_queue_head_t hydration_stopped; 94 95 mempool_t hydration_pool; 96 97 unsigned long last_commit_jiffies; 98 99 /* 100 * We defer incoming WRITE bios for regions that are not hydrated, 101 * until after these regions have been hydrated. 102 * 103 * Also, we defer REQ_FUA and REQ_PREFLUSH bios, until after the 104 * metadata have been committed. 105 */ 106 spinlock_t lock; 107 struct bio_list deferred_bios; 108 struct bio_list deferred_discard_bios; 109 struct bio_list deferred_flush_bios; 110 struct bio_list deferred_flush_completions; 111 112 /* Maximum number of regions being copied during background hydration. */ 113 unsigned int hydration_threshold; 114 115 /* Number of regions to batch together during background hydration. */ 116 unsigned int hydration_batch_size; 117 118 /* Which region to hydrate next */ 119 unsigned long hydration_offset; 120 121 atomic_t hydrations_in_flight; 122 123 /* 124 * Save a copy of the table line rather than reconstructing it for the 125 * status. 126 */ 127 unsigned int nr_ctr_args; 128 const char **ctr_args; 129 130 struct workqueue_struct *wq; 131 struct work_struct worker; 132 struct delayed_work waker; 133 134 struct dm_kcopyd_client *kcopyd_client; 135 136 enum clone_metadata_mode mode; 137 unsigned long flags; 138}; 139 140/* 141 * dm-clone flags 142 */ 143#define DM_CLONE_DISCARD_PASSDOWN 0 144#define DM_CLONE_HYDRATION_ENABLED 1 145#define DM_CLONE_HYDRATION_SUSPENDED 2 146 147/*---------------------------------------------------------------------------*/ 148 149/* 150 * Metadata failure handling. 151 */ 152static enum clone_metadata_mode get_clone_mode(struct clone *clone) 153{ 154 return READ_ONCE(clone->mode); 155} 156 157static const char *clone_device_name(struct clone *clone) 158{ 159 return dm_table_device_name(clone->ti->table); 160} 161 162static void __set_clone_mode(struct clone *clone, enum clone_metadata_mode new_mode) 163{ 164 static const char * const descs[] = { 165 "read-write", 166 "read-only", 167 "fail" 168 }; 169 170 enum clone_metadata_mode old_mode = get_clone_mode(clone); 171 172 /* Never move out of fail mode */ 173 if (old_mode == CM_FAIL) 174 new_mode = CM_FAIL; 175 176 switch (new_mode) { 177 case CM_FAIL: 178 case CM_READ_ONLY: 179 dm_clone_metadata_set_read_only(clone->cmd); 180 break; 181 182 case CM_WRITE: 183 dm_clone_metadata_set_read_write(clone->cmd); 184 break; 185 } 186 187 WRITE_ONCE(clone->mode, new_mode); 188 189 if (new_mode != old_mode) { 190 dm_table_event(clone->ti->table); 191 DMINFO("%s: Switching to %s mode", clone_device_name(clone), 192 descs[(int)new_mode]); 193 } 194} 195 196static void __abort_transaction(struct clone *clone) 197{ 198 const char *dev_name = clone_device_name(clone); 199 200 if (get_clone_mode(clone) >= CM_READ_ONLY) 201 return; 202 203 DMERR("%s: Aborting current metadata transaction", dev_name); 204 if (dm_clone_metadata_abort(clone->cmd)) { 205 DMERR("%s: Failed to abort metadata transaction", dev_name); 206 __set_clone_mode(clone, CM_FAIL); 207 } 208} 209 210static void __reload_in_core_bitset(struct clone *clone) 211{ 212 const char *dev_name = clone_device_name(clone); 213 214 if (get_clone_mode(clone) == CM_FAIL) 215 return; 216 217 /* Reload the on-disk bitset */ 218 DMINFO("%s: Reloading on-disk bitmap", dev_name); 219 if (dm_clone_reload_in_core_bitset(clone->cmd)) { 220 DMERR("%s: Failed to reload on-disk bitmap", dev_name); 221 __set_clone_mode(clone, CM_FAIL); 222 } 223} 224 225static void __metadata_operation_failed(struct clone *clone, const char *op, int r) 226{ 227 DMERR("%s: Metadata operation `%s' failed: error = %d", 228 clone_device_name(clone), op, r); 229 230 __abort_transaction(clone); 231 __set_clone_mode(clone, CM_READ_ONLY); 232 233 /* 234 * dm_clone_reload_in_core_bitset() may run concurrently with either 235 * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), but 236 * it's safe as we have already set the metadata to read-only mode. 237 */ 238 __reload_in_core_bitset(clone); 239} 240 241/*---------------------------------------------------------------------------*/ 242 243/* Wake up anyone waiting for region hydrations to stop */ 244static inline void wakeup_hydration_waiters(struct clone *clone) 245{ 246 wake_up_all(&clone->hydration_stopped); 247} 248 249static inline void wake_worker(struct clone *clone) 250{ 251 queue_work(clone->wq, &clone->worker); 252} 253 254/*---------------------------------------------------------------------------*/ 255 256/* 257 * bio helper functions. 258 */ 259static inline void remap_to_source(struct clone *clone, struct bio *bio) 260{ 261 bio_set_dev(bio, clone->source_dev->bdev); 262} 263 264static inline void remap_to_dest(struct clone *clone, struct bio *bio) 265{ 266 bio_set_dev(bio, clone->dest_dev->bdev); 267} 268 269static bool bio_triggers_commit(struct clone *clone, struct bio *bio) 270{ 271 return op_is_flush(bio->bi_opf) && 272 dm_clone_changed_this_transaction(clone->cmd); 273} 274 275/* Get the address of the region in sectors */ 276static inline sector_t region_to_sector(struct clone *clone, unsigned long region_nr) 277{ 278 return ((sector_t)region_nr << clone->region_shift); 279} 280 281/* Get the region number of the bio */ 282static inline unsigned long bio_to_region(struct clone *clone, struct bio *bio) 283{ 284 return (bio->bi_iter.bi_sector >> clone->region_shift); 285} 286 287/* Get the region range covered by the bio */ 288static void bio_region_range(struct clone *clone, struct bio *bio, 289 unsigned long *rs, unsigned long *nr_regions) 290{ 291 unsigned long end; 292 293 *rs = dm_sector_div_up(bio->bi_iter.bi_sector, clone->region_size); 294 end = bio_end_sector(bio) >> clone->region_shift; 295 296 if (*rs >= end) 297 *nr_regions = 0; 298 else 299 *nr_regions = end - *rs; 300} 301 302/* Check whether a bio overwrites a region */ 303static inline bool is_overwrite_bio(struct clone *clone, struct bio *bio) 304{ 305 return (bio_data_dir(bio) == WRITE && bio_sectors(bio) == clone->region_size); 306} 307 308static void fail_bios(struct bio_list *bios, blk_status_t status) 309{ 310 struct bio *bio; 311 312 while ((bio = bio_list_pop(bios))) { 313 bio->bi_status = status; 314 bio_endio(bio); 315 } 316} 317 318static void submit_bios(struct bio_list *bios) 319{ 320 struct bio *bio; 321 struct blk_plug plug; 322 323 blk_start_plug(&plug); 324 325 while ((bio = bio_list_pop(bios))) 326 submit_bio_noacct(bio); 327 328 blk_finish_plug(&plug); 329} 330 331/* 332 * Submit bio to the underlying device. 333 * 334 * If the bio triggers a commit, delay it, until after the metadata have been 335 * committed. 336 * 337 * NOTE: The bio remapping must be performed by the caller. 338 */ 339static void issue_bio(struct clone *clone, struct bio *bio) 340{ 341 if (!bio_triggers_commit(clone, bio)) { 342 submit_bio_noacct(bio); 343 return; 344 } 345 346 /* 347 * If the metadata mode is RO or FAIL we won't be able to commit the 348 * metadata, so we complete the bio with an error. 349 */ 350 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { 351 bio_io_error(bio); 352 return; 353 } 354 355 /* 356 * Batch together any bios that trigger commits and then issue a single 357 * commit for them in process_deferred_flush_bios(). 358 */ 359 spin_lock_irq(&clone->lock); 360 bio_list_add(&clone->deferred_flush_bios, bio); 361 spin_unlock_irq(&clone->lock); 362 363 wake_worker(clone); 364} 365 366/* 367 * Remap bio to the destination device and submit it. 368 * 369 * If the bio triggers a commit, delay it, until after the metadata have been 370 * committed. 371 */ 372static void remap_and_issue(struct clone *clone, struct bio *bio) 373{ 374 remap_to_dest(clone, bio); 375 issue_bio(clone, bio); 376} 377 378/* 379 * Issue bios that have been deferred until after their region has finished 380 * hydrating. 381 * 382 * We delegate the bio submission to the worker thread, so this is safe to call 383 * from interrupt context. 384 */ 385static void issue_deferred_bios(struct clone *clone, struct bio_list *bios) 386{ 387 struct bio *bio; 388 unsigned long flags; 389 struct bio_list flush_bios = BIO_EMPTY_LIST; 390 struct bio_list normal_bios = BIO_EMPTY_LIST; 391 392 if (bio_list_empty(bios)) 393 return; 394 395 while ((bio = bio_list_pop(bios))) { 396 if (bio_triggers_commit(clone, bio)) 397 bio_list_add(&flush_bios, bio); 398 else 399 bio_list_add(&normal_bios, bio); 400 } 401 402 spin_lock_irqsave(&clone->lock, flags); 403 bio_list_merge(&clone->deferred_bios, &normal_bios); 404 bio_list_merge(&clone->deferred_flush_bios, &flush_bios); 405 spin_unlock_irqrestore(&clone->lock, flags); 406 407 wake_worker(clone); 408} 409 410static void complete_overwrite_bio(struct clone *clone, struct bio *bio) 411{ 412 unsigned long flags; 413 414 /* 415 * If the bio has the REQ_FUA flag set we must commit the metadata 416 * before signaling its completion. 417 * 418 * complete_overwrite_bio() is only called by hydration_complete(), 419 * after having successfully updated the metadata. This means we don't 420 * need to call dm_clone_changed_this_transaction() to check if the 421 * metadata has changed and thus we can avoid taking the metadata spin 422 * lock. 423 */ 424 if (!(bio->bi_opf & REQ_FUA)) { 425 bio_endio(bio); 426 return; 427 } 428 429 /* 430 * If the metadata mode is RO or FAIL we won't be able to commit the 431 * metadata, so we complete the bio with an error. 432 */ 433 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { 434 bio_io_error(bio); 435 return; 436 } 437 438 /* 439 * Batch together any bios that trigger commits and then issue a single 440 * commit for them in process_deferred_flush_bios(). 441 */ 442 spin_lock_irqsave(&clone->lock, flags); 443 bio_list_add(&clone->deferred_flush_completions, bio); 444 spin_unlock_irqrestore(&clone->lock, flags); 445 446 wake_worker(clone); 447} 448 449static void trim_bio(struct bio *bio, sector_t sector, unsigned int len) 450{ 451 bio->bi_iter.bi_sector = sector; 452 bio->bi_iter.bi_size = to_bytes(len); 453} 454 455static void complete_discard_bio(struct clone *clone, struct bio *bio, bool success) 456{ 457 unsigned long rs, nr_regions; 458 459 /* 460 * If the destination device supports discards, remap and trim the 461 * discard bio and pass it down. Otherwise complete the bio 462 * immediately. 463 */ 464 if (test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags) && success) { 465 remap_to_dest(clone, bio); 466 bio_region_range(clone, bio, &rs, &nr_regions); 467 trim_bio(bio, region_to_sector(clone, rs), 468 nr_regions << clone->region_shift); 469 submit_bio_noacct(bio); 470 } else 471 bio_endio(bio); 472} 473 474static void process_discard_bio(struct clone *clone, struct bio *bio) 475{ 476 unsigned long rs, nr_regions; 477 478 bio_region_range(clone, bio, &rs, &nr_regions); 479 if (!nr_regions) { 480 bio_endio(bio); 481 return; 482 } 483 484 if (WARN_ON(rs >= clone->nr_regions || (rs + nr_regions) < rs || 485 (rs + nr_regions) > clone->nr_regions)) { 486 DMERR("%s: Invalid range (%lu + %lu, total regions %lu) for discard (%llu + %u)", 487 clone_device_name(clone), rs, nr_regions, 488 clone->nr_regions, 489 (unsigned long long)bio->bi_iter.bi_sector, 490 bio_sectors(bio)); 491 bio_endio(bio); 492 return; 493 } 494 495 /* 496 * The covered regions are already hydrated so we just need to pass 497 * down the discard. 498 */ 499 if (dm_clone_is_range_hydrated(clone->cmd, rs, nr_regions)) { 500 complete_discard_bio(clone, bio, true); 501 return; 502 } 503 504 /* 505 * If the metadata mode is RO or FAIL we won't be able to update the 506 * metadata for the regions covered by the discard so we just ignore 507 * it. 508 */ 509 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { 510 bio_endio(bio); 511 return; 512 } 513 514 /* 515 * Defer discard processing. 516 */ 517 spin_lock_irq(&clone->lock); 518 bio_list_add(&clone->deferred_discard_bios, bio); 519 spin_unlock_irq(&clone->lock); 520 521 wake_worker(clone); 522} 523 524/*---------------------------------------------------------------------------*/ 525 526/* 527 * dm-clone region hydrations. 528 */ 529struct dm_clone_region_hydration { 530 struct clone *clone; 531 unsigned long region_nr; 532 533 struct bio *overwrite_bio; 534 bio_end_io_t *overwrite_bio_end_io; 535 536 struct bio_list deferred_bios; 537 538 blk_status_t status; 539 540 /* Used by hydration batching */ 541 struct list_head list; 542 543 /* Used by hydration hash table */ 544 struct hlist_node h; 545}; 546 547/* 548 * Hydration hash table implementation. 549 * 550 * Ideally we would like to use list_bl, which uses bit spin locks and employs 551 * the least significant bit of the list head to lock the corresponding bucket, 552 * reducing the memory overhead for the locks. But, currently, list_bl and bit 553 * spin locks don't support IRQ safe versions. Since we have to take the lock 554 * in both process and interrupt context, we must fall back to using regular 555 * spin locks; one per hash table bucket. 556 */ 557struct hash_table_bucket { 558 struct hlist_head head; 559 560 /* Spinlock protecting the bucket */ 561 spinlock_t lock; 562}; 563 564#define bucket_lock_irqsave(bucket, flags) \ 565 spin_lock_irqsave(&(bucket)->lock, flags) 566 567#define bucket_unlock_irqrestore(bucket, flags) \ 568 spin_unlock_irqrestore(&(bucket)->lock, flags) 569 570#define bucket_lock_irq(bucket) \ 571 spin_lock_irq(&(bucket)->lock) 572 573#define bucket_unlock_irq(bucket) \ 574 spin_unlock_irq(&(bucket)->lock) 575 576static int hash_table_init(struct clone *clone) 577{ 578 unsigned int i, sz; 579 struct hash_table_bucket *bucket; 580 581 sz = 1 << HASH_TABLE_BITS; 582 583 clone->ht = kvmalloc_array(sz, sizeof(struct hash_table_bucket), GFP_KERNEL); 584 if (!clone->ht) 585 return -ENOMEM; 586 587 for (i = 0; i < sz; i++) { 588 bucket = clone->ht + i; 589 590 INIT_HLIST_HEAD(&bucket->head); 591 spin_lock_init(&bucket->lock); 592 } 593 594 return 0; 595} 596 597static void hash_table_exit(struct clone *clone) 598{ 599 kvfree(clone->ht); 600} 601 602static struct hash_table_bucket *get_hash_table_bucket(struct clone *clone, 603 unsigned long region_nr) 604{ 605 return &clone->ht[hash_long(region_nr, HASH_TABLE_BITS)]; 606} 607 608/* 609 * Search hash table for a hydration with hd->region_nr == region_nr 610 * 611 * NOTE: Must be called with the bucket lock held 612 */ 613static struct dm_clone_region_hydration *__hash_find(struct hash_table_bucket *bucket, 614 unsigned long region_nr) 615{ 616 struct dm_clone_region_hydration *hd; 617 618 hlist_for_each_entry(hd, &bucket->head, h) { 619 if (hd->region_nr == region_nr) 620 return hd; 621 } 622 623 return NULL; 624} 625 626/* 627 * Insert a hydration into the hash table. 628 * 629 * NOTE: Must be called with the bucket lock held. 630 */ 631static inline void __insert_region_hydration(struct hash_table_bucket *bucket, 632 struct dm_clone_region_hydration *hd) 633{ 634 hlist_add_head(&hd->h, &bucket->head); 635} 636 637/* 638 * This function inserts a hydration into the hash table, unless someone else 639 * managed to insert a hydration for the same region first. In the latter case 640 * it returns the existing hydration descriptor for this region. 641 * 642 * NOTE: Must be called with the hydration hash table lock held. 643 */ 644static struct dm_clone_region_hydration * 645__find_or_insert_region_hydration(struct hash_table_bucket *bucket, 646 struct dm_clone_region_hydration *hd) 647{ 648 struct dm_clone_region_hydration *hd2; 649 650 hd2 = __hash_find(bucket, hd->region_nr); 651 if (hd2) 652 return hd2; 653 654 __insert_region_hydration(bucket, hd); 655 656 return hd; 657} 658 659/*---------------------------------------------------------------------------*/ 660 661/* Allocate a hydration */ 662static struct dm_clone_region_hydration *alloc_hydration(struct clone *clone) 663{ 664 struct dm_clone_region_hydration *hd; 665 666 /* 667 * Allocate a hydration from the hydration mempool. 668 * This might block but it can't fail. 669 */ 670 hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO); 671 hd->clone = clone; 672 673 return hd; 674} 675 676static inline void free_hydration(struct dm_clone_region_hydration *hd) 677{ 678 mempool_free(hd, &hd->clone->hydration_pool); 679} 680 681/* Initialize a hydration */ 682static void hydration_init(struct dm_clone_region_hydration *hd, unsigned long region_nr) 683{ 684 hd->region_nr = region_nr; 685 hd->overwrite_bio = NULL; 686 bio_list_init(&hd->deferred_bios); 687 hd->status = 0; 688 689 INIT_LIST_HEAD(&hd->list); 690 INIT_HLIST_NODE(&hd->h); 691} 692 693/*---------------------------------------------------------------------------*/ 694 695/* 696 * Update dm-clone's metadata after a region has finished hydrating and remove 697 * hydration from the hash table. 698 */ 699static int hydration_update_metadata(struct dm_clone_region_hydration *hd) 700{ 701 int r = 0; 702 unsigned long flags; 703 struct hash_table_bucket *bucket; 704 struct clone *clone = hd->clone; 705 706 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) 707 r = -EPERM; 708 709 /* Update the metadata */ 710 if (likely(!r) && hd->status == BLK_STS_OK) 711 r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr); 712 713 bucket = get_hash_table_bucket(clone, hd->region_nr); 714 715 /* Remove hydration from hash table */ 716 bucket_lock_irqsave(bucket, flags); 717 hlist_del(&hd->h); 718 bucket_unlock_irqrestore(bucket, flags); 719 720 return r; 721} 722 723/* 724 * Complete a region's hydration: 725 * 726 * 1. Update dm-clone's metadata. 727 * 2. Remove hydration from hash table. 728 * 3. Complete overwrite bio. 729 * 4. Issue deferred bios. 730 * 5. If this was the last hydration, wake up anyone waiting for 731 * hydrations to finish. 732 */ 733static void hydration_complete(struct dm_clone_region_hydration *hd) 734{ 735 int r; 736 blk_status_t status; 737 struct clone *clone = hd->clone; 738 739 r = hydration_update_metadata(hd); 740 741 if (hd->status == BLK_STS_OK && likely(!r)) { 742 if (hd->overwrite_bio) 743 complete_overwrite_bio(clone, hd->overwrite_bio); 744 745 issue_deferred_bios(clone, &hd->deferred_bios); 746 } else { 747 status = r ? BLK_STS_IOERR : hd->status; 748 749 if (hd->overwrite_bio) 750 bio_list_add(&hd->deferred_bios, hd->overwrite_bio); 751 752 fail_bios(&hd->deferred_bios, status); 753 } 754 755 free_hydration(hd); 756 757 if (atomic_dec_and_test(&clone->hydrations_in_flight)) 758 wakeup_hydration_waiters(clone); 759} 760 761static void hydration_kcopyd_callback(int read_err, unsigned long write_err, void *context) 762{ 763 blk_status_t status; 764 765 struct dm_clone_region_hydration *tmp, *hd = context; 766 struct clone *clone = hd->clone; 767 768 LIST_HEAD(batched_hydrations); 769 770 if (read_err || write_err) { 771 DMERR_LIMIT("%s: hydration failed", clone_device_name(clone)); 772 status = BLK_STS_IOERR; 773 } else { 774 status = BLK_STS_OK; 775 } 776 list_splice_tail(&hd->list, &batched_hydrations); 777 778 hd->status = status; 779 hydration_complete(hd); 780 781 /* Complete batched hydrations */ 782 list_for_each_entry_safe(hd, tmp, &batched_hydrations, list) { 783 hd->status = status; 784 hydration_complete(hd); 785 } 786 787 /* Continue background hydration, if there is no I/O in-flight */ 788 if (test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) && 789 !atomic_read(&clone->ios_in_flight)) 790 wake_worker(clone); 791} 792 793static void hydration_copy(struct dm_clone_region_hydration *hd, unsigned int nr_regions) 794{ 795 unsigned long region_start, region_end; 796 sector_t tail_size, region_size, total_size; 797 struct dm_io_region from, to; 798 struct clone *clone = hd->clone; 799 800 if (WARN_ON(!nr_regions)) 801 return; 802 803 region_size = clone->region_size; 804 region_start = hd->region_nr; 805 region_end = region_start + nr_regions - 1; 806 807 total_size = region_to_sector(clone, nr_regions - 1); 808 809 if (region_end == clone->nr_regions - 1) { 810 /* 811 * The last region of the target might be smaller than 812 * region_size. 813 */ 814 tail_size = clone->ti->len & (region_size - 1); 815 if (!tail_size) 816 tail_size = region_size; 817 } else { 818 tail_size = region_size; 819 } 820 821 total_size += tail_size; 822 823 from.bdev = clone->source_dev->bdev; 824 from.sector = region_to_sector(clone, region_start); 825 from.count = total_size; 826 827 to.bdev = clone->dest_dev->bdev; 828 to.sector = from.sector; 829 to.count = from.count; 830 831 /* Issue copy */ 832 atomic_add(nr_regions, &clone->hydrations_in_flight); 833 dm_kcopyd_copy(clone->kcopyd_client, &from, 1, &to, 0, 834 hydration_kcopyd_callback, hd); 835} 836 837static void overwrite_endio(struct bio *bio) 838{ 839 struct dm_clone_region_hydration *hd = bio->bi_private; 840 841 bio->bi_end_io = hd->overwrite_bio_end_io; 842 hd->status = bio->bi_status; 843 844 hydration_complete(hd); 845} 846 847static void hydration_overwrite(struct dm_clone_region_hydration *hd, struct bio *bio) 848{ 849 /* 850 * We don't need to save and restore bio->bi_private because device 851 * mapper core generates a new bio for us to use, with clean 852 * bi_private. 853 */ 854 hd->overwrite_bio = bio; 855 hd->overwrite_bio_end_io = bio->bi_end_io; 856 857 bio->bi_end_io = overwrite_endio; 858 bio->bi_private = hd; 859 860 atomic_inc(&hd->clone->hydrations_in_flight); 861 submit_bio_noacct(bio); 862} 863 864/* 865 * Hydrate bio's region. 866 * 867 * This function starts the hydration of the bio's region and puts the bio in 868 * the list of deferred bios for this region. In case, by the time this 869 * function is called, the region has finished hydrating it's submitted to the 870 * destination device. 871 * 872 * NOTE: The bio remapping must be performed by the caller. 873 */ 874static void hydrate_bio_region(struct clone *clone, struct bio *bio) 875{ 876 unsigned long region_nr; 877 struct hash_table_bucket *bucket; 878 struct dm_clone_region_hydration *hd, *hd2; 879 880 region_nr = bio_to_region(clone, bio); 881 bucket = get_hash_table_bucket(clone, region_nr); 882 883 bucket_lock_irq(bucket); 884 885 hd = __hash_find(bucket, region_nr); 886 if (hd) { 887 /* Someone else is hydrating the region */ 888 bio_list_add(&hd->deferred_bios, bio); 889 bucket_unlock_irq(bucket); 890 return; 891 } 892 893 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) { 894 /* The region has been hydrated */ 895 bucket_unlock_irq(bucket); 896 issue_bio(clone, bio); 897 return; 898 } 899 900 /* 901 * We must allocate a hydration descriptor and start the hydration of 902 * the corresponding region. 903 */ 904 bucket_unlock_irq(bucket); 905 906 hd = alloc_hydration(clone); 907 hydration_init(hd, region_nr); 908 909 bucket_lock_irq(bucket); 910 911 /* Check if the region has been hydrated in the meantime. */ 912 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) { 913 bucket_unlock_irq(bucket); 914 free_hydration(hd); 915 issue_bio(clone, bio); 916 return; 917 } 918 919 hd2 = __find_or_insert_region_hydration(bucket, hd); 920 if (hd2 != hd) { 921 /* Someone else started the region's hydration. */ 922 bio_list_add(&hd2->deferred_bios, bio); 923 bucket_unlock_irq(bucket); 924 free_hydration(hd); 925 return; 926 } 927 928 /* 929 * If the metadata mode is RO or FAIL then there is no point starting a 930 * hydration, since we will not be able to update the metadata when the 931 * hydration finishes. 932 */ 933 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { 934 hlist_del(&hd->h); 935 bucket_unlock_irq(bucket); 936 free_hydration(hd); 937 bio_io_error(bio); 938 return; 939 } 940 941 /* 942 * Start region hydration. 943 * 944 * If a bio overwrites a region, i.e., its size is equal to the 945 * region's size, then we don't need to copy the region from the source 946 * to the destination device. 947 */ 948 if (is_overwrite_bio(clone, bio)) { 949 bucket_unlock_irq(bucket); 950 hydration_overwrite(hd, bio); 951 } else { 952 bio_list_add(&hd->deferred_bios, bio); 953 bucket_unlock_irq(bucket); 954 hydration_copy(hd, 1); 955 } 956} 957 958/*---------------------------------------------------------------------------*/ 959 960/* 961 * Background hydrations. 962 */ 963 964/* 965 * Batch region hydrations. 966 * 967 * To better utilize device bandwidth we batch together the hydration of 968 * adjacent regions. This allows us to use small region sizes, e.g., 4KB, which 969 * is good for small, random write performance (because of the overwriting of 970 * un-hydrated regions) and at the same time issue big copy requests to kcopyd 971 * to achieve high hydration bandwidth. 972 */ 973struct batch_info { 974 struct dm_clone_region_hydration *head; 975 unsigned int nr_batched_regions; 976}; 977 978static void __batch_hydration(struct batch_info *batch, 979 struct dm_clone_region_hydration *hd) 980{ 981 struct clone *clone = hd->clone; 982 unsigned int max_batch_size = READ_ONCE(clone->hydration_batch_size); 983 984 if (batch->head) { 985 /* Try to extend the current batch */ 986 if (batch->nr_batched_regions < max_batch_size && 987 (batch->head->region_nr + batch->nr_batched_regions) == hd->region_nr) { 988 list_add_tail(&hd->list, &batch->head->list); 989 batch->nr_batched_regions++; 990 hd = NULL; 991 } 992 993 /* Check if we should issue the current batch */ 994 if (batch->nr_batched_regions >= max_batch_size || hd) { 995 hydration_copy(batch->head, batch->nr_batched_regions); 996 batch->head = NULL; 997 batch->nr_batched_regions = 0; 998 } 999 } 1000 1001 if (!hd) 1002 return; 1003 1004 /* We treat max batch sizes of zero and one equivalently */ 1005 if (max_batch_size <= 1) { 1006 hydration_copy(hd, 1); 1007 return; 1008 } 1009 1010 /* Start a new batch */ 1011 BUG_ON(!list_empty(&hd->list)); 1012 batch->head = hd; 1013 batch->nr_batched_regions = 1; 1014} 1015 1016static unsigned long __start_next_hydration(struct clone *clone, 1017 unsigned long offset, 1018 struct batch_info *batch) 1019{ 1020 struct hash_table_bucket *bucket; 1021 struct dm_clone_region_hydration *hd; 1022 unsigned long nr_regions = clone->nr_regions; 1023 1024 hd = alloc_hydration(clone); 1025 1026 /* Try to find a region to hydrate. */ 1027 do { 1028 offset = dm_clone_find_next_unhydrated_region(clone->cmd, offset); 1029 if (offset == nr_regions) 1030 break; 1031 1032 bucket = get_hash_table_bucket(clone, offset); 1033 bucket_lock_irq(bucket); 1034 1035 if (!dm_clone_is_region_hydrated(clone->cmd, offset) && 1036 !__hash_find(bucket, offset)) { 1037 hydration_init(hd, offset); 1038 __insert_region_hydration(bucket, hd); 1039 bucket_unlock_irq(bucket); 1040 1041 /* Batch hydration */ 1042 __batch_hydration(batch, hd); 1043 1044 return (offset + 1); 1045 } 1046 1047 bucket_unlock_irq(bucket); 1048 1049 } while (++offset < nr_regions); 1050 1051 if (hd) 1052 free_hydration(hd); 1053 1054 return offset; 1055} 1056 1057/* 1058 * This function searches for regions that still reside in the source device 1059 * and starts their hydration. 1060 */ 1061static void do_hydration(struct clone *clone) 1062{ 1063 unsigned int current_volume; 1064 unsigned long offset, nr_regions = clone->nr_regions; 1065 1066 struct batch_info batch = { 1067 .head = NULL, 1068 .nr_batched_regions = 0, 1069 }; 1070 1071 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) 1072 return; 1073 1074 if (dm_clone_is_hydration_done(clone->cmd)) 1075 return; 1076 1077 /* 1078 * Avoid race with device suspension. 1079 */ 1080 atomic_inc(&clone->hydrations_in_flight); 1081 1082 /* 1083 * Make sure atomic_inc() is ordered before test_bit(), otherwise we 1084 * might race with clone_postsuspend() and start a region hydration 1085 * after the target has been suspended. 1086 * 1087 * This is paired with the smp_mb__after_atomic() in 1088 * clone_postsuspend(). 1089 */ 1090 smp_mb__after_atomic(); 1091 1092 offset = clone->hydration_offset; 1093 while (likely(!test_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags)) && 1094 !atomic_read(&clone->ios_in_flight) && 1095 test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) && 1096 offset < nr_regions) { 1097 current_volume = atomic_read(&clone->hydrations_in_flight); 1098 current_volume += batch.nr_batched_regions; 1099 1100 if (current_volume > READ_ONCE(clone->hydration_threshold)) 1101 break; 1102 1103 offset = __start_next_hydration(clone, offset, &batch); 1104 } 1105 1106 if (batch.head) 1107 hydration_copy(batch.head, batch.nr_batched_regions); 1108 1109 if (offset >= nr_regions) 1110 offset = 0; 1111 1112 clone->hydration_offset = offset; 1113 1114 if (atomic_dec_and_test(&clone->hydrations_in_flight)) 1115 wakeup_hydration_waiters(clone); 1116} 1117 1118/*---------------------------------------------------------------------------*/ 1119 1120static bool need_commit_due_to_time(struct clone *clone) 1121{ 1122 return !time_in_range(jiffies, clone->last_commit_jiffies, 1123 clone->last_commit_jiffies + COMMIT_PERIOD); 1124} 1125 1126/* 1127 * A non-zero return indicates read-only or fail mode. 1128 */ 1129static int commit_metadata(struct clone *clone, bool *dest_dev_flushed) 1130{ 1131 int r = 0; 1132 1133 if (dest_dev_flushed) 1134 *dest_dev_flushed = false; 1135 1136 mutex_lock(&clone->commit_lock); 1137 1138 if (!dm_clone_changed_this_transaction(clone->cmd)) 1139 goto out; 1140 1141 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { 1142 r = -EPERM; 1143 goto out; 1144 } 1145 1146 r = dm_clone_metadata_pre_commit(clone->cmd); 1147 if (unlikely(r)) { 1148 __metadata_operation_failed(clone, "dm_clone_metadata_pre_commit", r); 1149 goto out; 1150 } 1151 1152 r = blkdev_issue_flush(clone->dest_dev->bdev); 1153 if (unlikely(r)) { 1154 __metadata_operation_failed(clone, "flush destination device", r); 1155 goto out; 1156 } 1157 1158 if (dest_dev_flushed) 1159 *dest_dev_flushed = true; 1160 1161 r = dm_clone_metadata_commit(clone->cmd); 1162 if (unlikely(r)) { 1163 __metadata_operation_failed(clone, "dm_clone_metadata_commit", r); 1164 goto out; 1165 } 1166 1167 if (dm_clone_is_hydration_done(clone->cmd)) 1168 dm_table_event(clone->ti->table); 1169out: 1170 mutex_unlock(&clone->commit_lock); 1171 1172 return r; 1173} 1174 1175static void process_deferred_discards(struct clone *clone) 1176{ 1177 int r = -EPERM; 1178 struct bio *bio; 1179 struct blk_plug plug; 1180 unsigned long rs, nr_regions; 1181 struct bio_list discards = BIO_EMPTY_LIST; 1182 1183 spin_lock_irq(&clone->lock); 1184 bio_list_merge(&discards, &clone->deferred_discard_bios); 1185 bio_list_init(&clone->deferred_discard_bios); 1186 spin_unlock_irq(&clone->lock); 1187 1188 if (bio_list_empty(&discards)) 1189 return; 1190 1191 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) 1192 goto out; 1193 1194 /* Update the metadata */ 1195 bio_list_for_each(bio, &discards) { 1196 bio_region_range(clone, bio, &rs, &nr_regions); 1197 /* 1198 * A discard request might cover regions that have been already 1199 * hydrated. There is no need to update the metadata for these 1200 * regions. 1201 */ 1202 r = dm_clone_cond_set_range(clone->cmd, rs, nr_regions); 1203 if (unlikely(r)) 1204 break; 1205 } 1206out: 1207 blk_start_plug(&plug); 1208 while ((bio = bio_list_pop(&discards))) 1209 complete_discard_bio(clone, bio, r == 0); 1210 blk_finish_plug(&plug); 1211} 1212 1213static void process_deferred_bios(struct clone *clone) 1214{ 1215 struct bio_list bios = BIO_EMPTY_LIST; 1216 1217 spin_lock_irq(&clone->lock); 1218 bio_list_merge(&bios, &clone->deferred_bios); 1219 bio_list_init(&clone->deferred_bios); 1220 spin_unlock_irq(&clone->lock); 1221 1222 if (bio_list_empty(&bios)) 1223 return; 1224 1225 submit_bios(&bios); 1226} 1227 1228static void process_deferred_flush_bios(struct clone *clone) 1229{ 1230 struct bio *bio; 1231 bool dest_dev_flushed; 1232 struct bio_list bios = BIO_EMPTY_LIST; 1233 struct bio_list bio_completions = BIO_EMPTY_LIST; 1234 1235 /* 1236 * If there are any deferred flush bios, we must commit the metadata 1237 * before issuing them or signaling their completion. 1238 */ 1239 spin_lock_irq(&clone->lock); 1240 bio_list_merge(&bios, &clone->deferred_flush_bios); 1241 bio_list_init(&clone->deferred_flush_bios); 1242 1243 bio_list_merge(&bio_completions, &clone->deferred_flush_completions); 1244 bio_list_init(&clone->deferred_flush_completions); 1245 spin_unlock_irq(&clone->lock); 1246 1247 if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) && 1248 !(dm_clone_changed_this_transaction(clone->cmd) && need_commit_due_to_time(clone))) 1249 return; 1250 1251 if (commit_metadata(clone, &dest_dev_flushed)) { 1252 bio_list_merge(&bios, &bio_completions); 1253 1254 while ((bio = bio_list_pop(&bios))) 1255 bio_io_error(bio); 1256 1257 return; 1258 } 1259 1260 clone->last_commit_jiffies = jiffies; 1261 1262 while ((bio = bio_list_pop(&bio_completions))) 1263 bio_endio(bio); 1264 1265 while ((bio = bio_list_pop(&bios))) { 1266 if ((bio->bi_opf & REQ_PREFLUSH) && dest_dev_flushed) { 1267 /* We just flushed the destination device as part of 1268 * the metadata commit, so there is no reason to send 1269 * another flush. 1270 */ 1271 bio_endio(bio); 1272 } else { 1273 submit_bio_noacct(bio); 1274 } 1275 } 1276} 1277 1278static void do_worker(struct work_struct *work) 1279{ 1280 struct clone *clone = container_of(work, typeof(*clone), worker); 1281 1282 process_deferred_bios(clone); 1283 process_deferred_discards(clone); 1284 1285 /* 1286 * process_deferred_flush_bios(): 1287 * 1288 * - Commit metadata 1289 * 1290 * - Process deferred REQ_FUA completions 1291 * 1292 * - Process deferred REQ_PREFLUSH bios 1293 */ 1294 process_deferred_flush_bios(clone); 1295 1296 /* Background hydration */ 1297 do_hydration(clone); 1298} 1299 1300/* 1301 * Commit periodically so that not too much unwritten data builds up. 1302 * 1303 * Also, restart background hydration, if it has been stopped by in-flight I/O. 1304 */ 1305static void do_waker(struct work_struct *work) 1306{ 1307 struct clone *clone = container_of(to_delayed_work(work), struct clone, waker); 1308 1309 wake_worker(clone); 1310 queue_delayed_work(clone->wq, &clone->waker, COMMIT_PERIOD); 1311} 1312 1313/*---------------------------------------------------------------------------*/ 1314 1315/* 1316 * Target methods 1317 */ 1318static int clone_map(struct dm_target *ti, struct bio *bio) 1319{ 1320 struct clone *clone = ti->private; 1321 unsigned long region_nr; 1322 1323 atomic_inc(&clone->ios_in_flight); 1324 1325 if (unlikely(get_clone_mode(clone) == CM_FAIL)) 1326 return DM_MAPIO_KILL; 1327 1328 /* 1329 * REQ_PREFLUSH bios carry no data: 1330 * 1331 * - Commit metadata, if changed 1332 * 1333 * - Pass down to destination device 1334 */ 1335 if (bio->bi_opf & REQ_PREFLUSH) { 1336 remap_and_issue(clone, bio); 1337 return DM_MAPIO_SUBMITTED; 1338 } 1339 1340 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 1341 1342 /* 1343 * dm-clone interprets discards and performs a fast hydration of the 1344 * discarded regions, i.e., we skip the copy from the source device and 1345 * just mark the regions as hydrated. 1346 */ 1347 if (bio_op(bio) == REQ_OP_DISCARD) { 1348 process_discard_bio(clone, bio); 1349 return DM_MAPIO_SUBMITTED; 1350 } 1351 1352 /* 1353 * If the bio's region is hydrated, redirect it to the destination 1354 * device. 1355 * 1356 * If the region is not hydrated and the bio is a READ, redirect it to 1357 * the source device. 1358 * 1359 * Else, defer WRITE bio until after its region has been hydrated and 1360 * start the region's hydration immediately. 1361 */ 1362 region_nr = bio_to_region(clone, bio); 1363 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) { 1364 remap_and_issue(clone, bio); 1365 return DM_MAPIO_SUBMITTED; 1366 } else if (bio_data_dir(bio) == READ) { 1367 remap_to_source(clone, bio); 1368 return DM_MAPIO_REMAPPED; 1369 } 1370 1371 remap_to_dest(clone, bio); 1372 hydrate_bio_region(clone, bio); 1373 1374 return DM_MAPIO_SUBMITTED; 1375} 1376 1377static int clone_endio(struct dm_target *ti, struct bio *bio, blk_status_t *error) 1378{ 1379 struct clone *clone = ti->private; 1380 1381 atomic_dec(&clone->ios_in_flight); 1382 1383 return DM_ENDIO_DONE; 1384} 1385 1386static void emit_flags(struct clone *clone, char *result, unsigned int maxlen, 1387 ssize_t *sz_ptr) 1388{ 1389 ssize_t sz = *sz_ptr; 1390 unsigned int count; 1391 1392 count = !test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); 1393 count += !test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); 1394 1395 DMEMIT("%u ", count); 1396 1397 if (!test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags)) 1398 DMEMIT("no_hydration "); 1399 1400 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) 1401 DMEMIT("no_discard_passdown "); 1402 1403 *sz_ptr = sz; 1404} 1405 1406static void emit_core_args(struct clone *clone, char *result, 1407 unsigned int maxlen, ssize_t *sz_ptr) 1408{ 1409 ssize_t sz = *sz_ptr; 1410 unsigned int count = 4; 1411 1412 DMEMIT("%u hydration_threshold %u hydration_batch_size %u ", count, 1413 READ_ONCE(clone->hydration_threshold), 1414 READ_ONCE(clone->hydration_batch_size)); 1415 1416 *sz_ptr = sz; 1417} 1418 1419/* 1420 * Status format: 1421 * 1422 * <metadata block size> <#used metadata blocks>/<#total metadata blocks> 1423 * <clone region size> <#hydrated regions>/<#total regions> <#hydrating regions> 1424 * <#features> <features>* <#core args> <core args>* <clone metadata mode> 1425 */ 1426static void clone_status(struct dm_target *ti, status_type_t type, 1427 unsigned int status_flags, char *result, 1428 unsigned int maxlen) 1429{ 1430 int r; 1431 unsigned int i; 1432 ssize_t sz = 0; 1433 dm_block_t nr_free_metadata_blocks = 0; 1434 dm_block_t nr_metadata_blocks = 0; 1435 char buf[BDEVNAME_SIZE]; 1436 struct clone *clone = ti->private; 1437 1438 switch (type) { 1439 case STATUSTYPE_INFO: 1440 if (get_clone_mode(clone) == CM_FAIL) { 1441 DMEMIT("Fail"); 1442 break; 1443 } 1444 1445 /* Commit to ensure statistics aren't out-of-date */ 1446 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 1447 (void) commit_metadata(clone, NULL); 1448 1449 r = dm_clone_get_free_metadata_block_count(clone->cmd, &nr_free_metadata_blocks); 1450 1451 if (r) { 1452 DMERR("%s: dm_clone_get_free_metadata_block_count returned %d", 1453 clone_device_name(clone), r); 1454 goto error; 1455 } 1456 1457 r = dm_clone_get_metadata_dev_size(clone->cmd, &nr_metadata_blocks); 1458 1459 if (r) { 1460 DMERR("%s: dm_clone_get_metadata_dev_size returned %d", 1461 clone_device_name(clone), r); 1462 goto error; 1463 } 1464 1465 DMEMIT("%u %llu/%llu %llu %u/%lu %u ", 1466 DM_CLONE_METADATA_BLOCK_SIZE, 1467 (unsigned long long)(nr_metadata_blocks - nr_free_metadata_blocks), 1468 (unsigned long long)nr_metadata_blocks, 1469 (unsigned long long)clone->region_size, 1470 dm_clone_nr_of_hydrated_regions(clone->cmd), 1471 clone->nr_regions, 1472 atomic_read(&clone->hydrations_in_flight)); 1473 1474 emit_flags(clone, result, maxlen, &sz); 1475 emit_core_args(clone, result, maxlen, &sz); 1476 1477 switch (get_clone_mode(clone)) { 1478 case CM_WRITE: 1479 DMEMIT("rw"); 1480 break; 1481 case CM_READ_ONLY: 1482 DMEMIT("ro"); 1483 break; 1484 case CM_FAIL: 1485 DMEMIT("Fail"); 1486 } 1487 1488 break; 1489 1490 case STATUSTYPE_TABLE: 1491 format_dev_t(buf, clone->metadata_dev->bdev->bd_dev); 1492 DMEMIT("%s ", buf); 1493 1494 format_dev_t(buf, clone->dest_dev->bdev->bd_dev); 1495 DMEMIT("%s ", buf); 1496 1497 format_dev_t(buf, clone->source_dev->bdev->bd_dev); 1498 DMEMIT("%s", buf); 1499 1500 for (i = 0; i < clone->nr_ctr_args; i++) 1501 DMEMIT(" %s", clone->ctr_args[i]); 1502 break; 1503 1504 case STATUSTYPE_IMA: 1505 *result = '\0'; 1506 break; 1507 } 1508 1509 return; 1510 1511error: 1512 DMEMIT("Error"); 1513} 1514 1515static sector_t get_dev_size(struct dm_dev *dev) 1516{ 1517 return bdev_nr_sectors(dev->bdev); 1518} 1519 1520/*---------------------------------------------------------------------------*/ 1521 1522/* 1523 * Construct a clone device mapping: 1524 * 1525 * clone <metadata dev> <destination dev> <source dev> <region size> 1526 * [<#feature args> [<feature arg>]* [<#core args> [key value]*]] 1527 * 1528 * metadata dev: Fast device holding the persistent metadata 1529 * destination dev: The destination device, which will become a clone of the 1530 * source device 1531 * source dev: The read-only source device that gets cloned 1532 * region size: dm-clone unit size in sectors 1533 * 1534 * #feature args: Number of feature arguments passed 1535 * feature args: E.g. no_hydration, no_discard_passdown 1536 * 1537 * #core arguments: An even number of core arguments 1538 * core arguments: Key/value pairs for tuning the core 1539 * E.g. 'hydration_threshold 256' 1540 */ 1541static int parse_feature_args(struct dm_arg_set *as, struct clone *clone) 1542{ 1543 int r; 1544 unsigned int argc; 1545 const char *arg_name; 1546 struct dm_target *ti = clone->ti; 1547 1548 const struct dm_arg args = { 1549 .min = 0, 1550 .max = 2, 1551 .error = "Invalid number of feature arguments" 1552 }; 1553 1554 /* No feature arguments supplied */ 1555 if (!as->argc) 1556 return 0; 1557 1558 r = dm_read_arg_group(&args, as, &argc, &ti->error); 1559 if (r) 1560 return r; 1561 1562 while (argc) { 1563 arg_name = dm_shift_arg(as); 1564 argc--; 1565 1566 if (!strcasecmp(arg_name, "no_hydration")) { 1567 __clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); 1568 } else if (!strcasecmp(arg_name, "no_discard_passdown")) { 1569 __clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); 1570 } else { 1571 ti->error = "Invalid feature argument"; 1572 return -EINVAL; 1573 } 1574 } 1575 1576 return 0; 1577} 1578 1579static int parse_core_args(struct dm_arg_set *as, struct clone *clone) 1580{ 1581 int r; 1582 unsigned int argc; 1583 unsigned int value; 1584 const char *arg_name; 1585 struct dm_target *ti = clone->ti; 1586 1587 const struct dm_arg args = { 1588 .min = 0, 1589 .max = 4, 1590 .error = "Invalid number of core arguments" 1591 }; 1592 1593 /* Initialize core arguments */ 1594 clone->hydration_batch_size = DEFAULT_HYDRATION_BATCH_SIZE; 1595 clone->hydration_threshold = DEFAULT_HYDRATION_THRESHOLD; 1596 1597 /* No core arguments supplied */ 1598 if (!as->argc) 1599 return 0; 1600 1601 r = dm_read_arg_group(&args, as, &argc, &ti->error); 1602 if (r) 1603 return r; 1604 1605 if (argc & 1) { 1606 ti->error = "Number of core arguments must be even"; 1607 return -EINVAL; 1608 } 1609 1610 while (argc) { 1611 arg_name = dm_shift_arg(as); 1612 argc -= 2; 1613 1614 if (!strcasecmp(arg_name, "hydration_threshold")) { 1615 if (kstrtouint(dm_shift_arg(as), 10, &value)) { 1616 ti->error = "Invalid value for argument `hydration_threshold'"; 1617 return -EINVAL; 1618 } 1619 clone->hydration_threshold = value; 1620 } else if (!strcasecmp(arg_name, "hydration_batch_size")) { 1621 if (kstrtouint(dm_shift_arg(as), 10, &value)) { 1622 ti->error = "Invalid value for argument `hydration_batch_size'"; 1623 return -EINVAL; 1624 } 1625 clone->hydration_batch_size = value; 1626 } else { 1627 ti->error = "Invalid core argument"; 1628 return -EINVAL; 1629 } 1630 } 1631 1632 return 0; 1633} 1634 1635static int parse_region_size(struct clone *clone, struct dm_arg_set *as, char **error) 1636{ 1637 int r; 1638 unsigned int region_size; 1639 struct dm_arg arg; 1640 1641 arg.min = MIN_REGION_SIZE; 1642 arg.max = MAX_REGION_SIZE; 1643 arg.error = "Invalid region size"; 1644 1645 r = dm_read_arg(&arg, as, ®ion_size, error); 1646 if (r) 1647 return r; 1648 1649 /* Check region size is a power of 2 */ 1650 if (!is_power_of_2(region_size)) { 1651 *error = "Region size is not a power of 2"; 1652 return -EINVAL; 1653 } 1654 1655 /* Validate the region size against the device logical block size */ 1656 if (region_size % (bdev_logical_block_size(clone->source_dev->bdev) >> 9) || 1657 region_size % (bdev_logical_block_size(clone->dest_dev->bdev) >> 9)) { 1658 *error = "Region size is not a multiple of device logical block size"; 1659 return -EINVAL; 1660 } 1661 1662 clone->region_size = region_size; 1663 1664 return 0; 1665} 1666 1667static int validate_nr_regions(unsigned long n, char **error) 1668{ 1669 /* 1670 * dm_bitset restricts us to 2^32 regions. test_bit & co. restrict us 1671 * further to 2^31 regions. 1672 */ 1673 if (n > (1UL << 31)) { 1674 *error = "Too many regions. Consider increasing the region size"; 1675 return -EINVAL; 1676 } 1677 1678 return 0; 1679} 1680 1681static int parse_metadata_dev(struct clone *clone, struct dm_arg_set *as, char **error) 1682{ 1683 int r; 1684 sector_t metadata_dev_size; 1685 1686 r = dm_get_device(clone->ti, dm_shift_arg(as), 1687 BLK_OPEN_READ | BLK_OPEN_WRITE, &clone->metadata_dev); 1688 if (r) { 1689 *error = "Error opening metadata device"; 1690 return r; 1691 } 1692 1693 metadata_dev_size = get_dev_size(clone->metadata_dev); 1694 if (metadata_dev_size > DM_CLONE_METADATA_MAX_SECTORS_WARNING) 1695 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.", 1696 clone->metadata_dev->bdev, DM_CLONE_METADATA_MAX_SECTORS); 1697 1698 return 0; 1699} 1700 1701static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error) 1702{ 1703 int r; 1704 sector_t dest_dev_size; 1705 1706 r = dm_get_device(clone->ti, dm_shift_arg(as), 1707 BLK_OPEN_READ | BLK_OPEN_WRITE, &clone->dest_dev); 1708 if (r) { 1709 *error = "Error opening destination device"; 1710 return r; 1711 } 1712 1713 dest_dev_size = get_dev_size(clone->dest_dev); 1714 if (dest_dev_size < clone->ti->len) { 1715 dm_put_device(clone->ti, clone->dest_dev); 1716 *error = "Device size larger than destination device"; 1717 return -EINVAL; 1718 } 1719 1720 return 0; 1721} 1722 1723static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error) 1724{ 1725 int r; 1726 sector_t source_dev_size; 1727 1728 r = dm_get_device(clone->ti, dm_shift_arg(as), BLK_OPEN_READ, 1729 &clone->source_dev); 1730 if (r) { 1731 *error = "Error opening source device"; 1732 return r; 1733 } 1734 1735 source_dev_size = get_dev_size(clone->source_dev); 1736 if (source_dev_size < clone->ti->len) { 1737 dm_put_device(clone->ti, clone->source_dev); 1738 *error = "Device size larger than source device"; 1739 return -EINVAL; 1740 } 1741 1742 return 0; 1743} 1744 1745static int copy_ctr_args(struct clone *clone, int argc, const char **argv, char **error) 1746{ 1747 unsigned int i; 1748 const char **copy; 1749 1750 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL); 1751 if (!copy) 1752 goto error; 1753 1754 for (i = 0; i < argc; i++) { 1755 copy[i] = kstrdup(argv[i], GFP_KERNEL); 1756 1757 if (!copy[i]) { 1758 while (i--) 1759 kfree(copy[i]); 1760 kfree(copy); 1761 goto error; 1762 } 1763 } 1764 1765 clone->nr_ctr_args = argc; 1766 clone->ctr_args = copy; 1767 return 0; 1768 1769error: 1770 *error = "Failed to allocate memory for table line"; 1771 return -ENOMEM; 1772} 1773 1774static int clone_ctr(struct dm_target *ti, unsigned int argc, char **argv) 1775{ 1776 int r; 1777 sector_t nr_regions; 1778 struct clone *clone; 1779 struct dm_arg_set as; 1780 1781 if (argc < 4) { 1782 ti->error = "Invalid number of arguments"; 1783 return -EINVAL; 1784 } 1785 1786 as.argc = argc; 1787 as.argv = argv; 1788 1789 clone = kzalloc(sizeof(*clone), GFP_KERNEL); 1790 if (!clone) { 1791 ti->error = "Failed to allocate clone structure"; 1792 return -ENOMEM; 1793 } 1794 1795 clone->ti = ti; 1796 1797 /* Initialize dm-clone flags */ 1798 __set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); 1799 __set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags); 1800 __set_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); 1801 1802 r = parse_metadata_dev(clone, &as, &ti->error); 1803 if (r) 1804 goto out_with_clone; 1805 1806 r = parse_dest_dev(clone, &as, &ti->error); 1807 if (r) 1808 goto out_with_meta_dev; 1809 1810 r = parse_source_dev(clone, &as, &ti->error); 1811 if (r) 1812 goto out_with_dest_dev; 1813 1814 r = parse_region_size(clone, &as, &ti->error); 1815 if (r) 1816 goto out_with_source_dev; 1817 1818 clone->region_shift = __ffs(clone->region_size); 1819 nr_regions = dm_sector_div_up(ti->len, clone->region_size); 1820 1821 /* Check for overflow */ 1822 if (nr_regions != (unsigned long)nr_regions) { 1823 ti->error = "Too many regions. Consider increasing the region size"; 1824 r = -EOVERFLOW; 1825 goto out_with_source_dev; 1826 } 1827 1828 clone->nr_regions = nr_regions; 1829 1830 r = validate_nr_regions(clone->nr_regions, &ti->error); 1831 if (r) 1832 goto out_with_source_dev; 1833 1834 r = dm_set_target_max_io_len(ti, clone->region_size); 1835 if (r) { 1836 ti->error = "Failed to set max io len"; 1837 goto out_with_source_dev; 1838 } 1839 1840 r = parse_feature_args(&as, clone); 1841 if (r) 1842 goto out_with_source_dev; 1843 1844 r = parse_core_args(&as, clone); 1845 if (r) 1846 goto out_with_source_dev; 1847 1848 /* Load metadata */ 1849 clone->cmd = dm_clone_metadata_open(clone->metadata_dev->bdev, ti->len, 1850 clone->region_size); 1851 if (IS_ERR(clone->cmd)) { 1852 ti->error = "Failed to load metadata"; 1853 r = PTR_ERR(clone->cmd); 1854 goto out_with_source_dev; 1855 } 1856 1857 __set_clone_mode(clone, CM_WRITE); 1858 1859 if (get_clone_mode(clone) != CM_WRITE) { 1860 ti->error = "Unable to get write access to metadata, please check/repair metadata"; 1861 r = -EPERM; 1862 goto out_with_metadata; 1863 } 1864 1865 clone->last_commit_jiffies = jiffies; 1866 1867 /* Allocate hydration hash table */ 1868 r = hash_table_init(clone); 1869 if (r) { 1870 ti->error = "Failed to allocate hydration hash table"; 1871 goto out_with_metadata; 1872 } 1873 1874 atomic_set(&clone->ios_in_flight, 0); 1875 init_waitqueue_head(&clone->hydration_stopped); 1876 spin_lock_init(&clone->lock); 1877 bio_list_init(&clone->deferred_bios); 1878 bio_list_init(&clone->deferred_discard_bios); 1879 bio_list_init(&clone->deferred_flush_bios); 1880 bio_list_init(&clone->deferred_flush_completions); 1881 clone->hydration_offset = 0; 1882 atomic_set(&clone->hydrations_in_flight, 0); 1883 1884 clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0); 1885 if (!clone->wq) { 1886 ti->error = "Failed to allocate workqueue"; 1887 r = -ENOMEM; 1888 goto out_with_ht; 1889 } 1890 1891 INIT_WORK(&clone->worker, do_worker); 1892 INIT_DELAYED_WORK(&clone->waker, do_waker); 1893 1894 clone->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); 1895 if (IS_ERR(clone->kcopyd_client)) { 1896 r = PTR_ERR(clone->kcopyd_client); 1897 goto out_with_wq; 1898 } 1899 1900 r = mempool_init_slab_pool(&clone->hydration_pool, MIN_HYDRATIONS, 1901 _hydration_cache); 1902 if (r) { 1903 ti->error = "Failed to create dm_clone_region_hydration memory pool"; 1904 goto out_with_kcopyd; 1905 } 1906 1907 /* Save a copy of the table line */ 1908 r = copy_ctr_args(clone, argc - 3, (const char **)argv + 3, &ti->error); 1909 if (r) 1910 goto out_with_mempool; 1911 1912 mutex_init(&clone->commit_lock); 1913 1914 /* Enable flushes */ 1915 ti->num_flush_bios = 1; 1916 ti->flush_supported = true; 1917 1918 /* Enable discards */ 1919 ti->discards_supported = true; 1920 ti->num_discard_bios = 1; 1921 1922 ti->private = clone; 1923 1924 return 0; 1925 1926out_with_mempool: 1927 mempool_exit(&clone->hydration_pool); 1928out_with_kcopyd: 1929 dm_kcopyd_client_destroy(clone->kcopyd_client); 1930out_with_wq: 1931 destroy_workqueue(clone->wq); 1932out_with_ht: 1933 hash_table_exit(clone); 1934out_with_metadata: 1935 dm_clone_metadata_close(clone->cmd); 1936out_with_source_dev: 1937 dm_put_device(ti, clone->source_dev); 1938out_with_dest_dev: 1939 dm_put_device(ti, clone->dest_dev); 1940out_with_meta_dev: 1941 dm_put_device(ti, clone->metadata_dev); 1942out_with_clone: 1943 kfree(clone); 1944 1945 return r; 1946} 1947 1948static void clone_dtr(struct dm_target *ti) 1949{ 1950 unsigned int i; 1951 struct clone *clone = ti->private; 1952 1953 mutex_destroy(&clone->commit_lock); 1954 1955 for (i = 0; i < clone->nr_ctr_args; i++) 1956 kfree(clone->ctr_args[i]); 1957 kfree(clone->ctr_args); 1958 1959 mempool_exit(&clone->hydration_pool); 1960 dm_kcopyd_client_destroy(clone->kcopyd_client); 1961 cancel_delayed_work_sync(&clone->waker); 1962 destroy_workqueue(clone->wq); 1963 hash_table_exit(clone); 1964 dm_clone_metadata_close(clone->cmd); 1965 dm_put_device(ti, clone->source_dev); 1966 dm_put_device(ti, clone->dest_dev); 1967 dm_put_device(ti, clone->metadata_dev); 1968 1969 kfree(clone); 1970} 1971 1972/*---------------------------------------------------------------------------*/ 1973 1974static void clone_postsuspend(struct dm_target *ti) 1975{ 1976 struct clone *clone = ti->private; 1977 1978 /* 1979 * To successfully suspend the device: 1980 * 1981 * - We cancel the delayed work for periodic commits and wait for 1982 * it to finish. 1983 * 1984 * - We stop the background hydration, i.e. we prevent new region 1985 * hydrations from starting. 1986 * 1987 * - We wait for any in-flight hydrations to finish. 1988 * 1989 * - We flush the workqueue. 1990 * 1991 * - We commit the metadata. 1992 */ 1993 cancel_delayed_work_sync(&clone->waker); 1994 1995 set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags); 1996 1997 /* 1998 * Make sure set_bit() is ordered before atomic_read(), otherwise we 1999 * might race with do_hydration() and miss some started region 2000 * hydrations. 2001 * 2002 * This is paired with smp_mb__after_atomic() in do_hydration(). 2003 */ 2004 smp_mb__after_atomic(); 2005 2006 wait_event(clone->hydration_stopped, !atomic_read(&clone->hydrations_in_flight)); 2007 flush_workqueue(clone->wq); 2008 2009 (void) commit_metadata(clone, NULL); 2010} 2011 2012static void clone_resume(struct dm_target *ti) 2013{ 2014 struct clone *clone = ti->private; 2015 2016 clear_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags); 2017 do_waker(&clone->waker.work); 2018} 2019 2020/* 2021 * If discard_passdown was enabled verify that the destination device supports 2022 * discards. Disable discard_passdown if not. 2023 */ 2024static void disable_passdown_if_not_supported(struct clone *clone) 2025{ 2026 struct block_device *dest_dev = clone->dest_dev->bdev; 2027 struct queue_limits *dest_limits = &bdev_get_queue(dest_dev)->limits; 2028 const char *reason = NULL; 2029 2030 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) 2031 return; 2032 2033 if (!bdev_max_discard_sectors(dest_dev)) 2034 reason = "discard unsupported"; 2035 else if (dest_limits->max_discard_sectors < clone->region_size) 2036 reason = "max discard sectors smaller than a region"; 2037 2038 if (reason) { 2039 DMWARN("Destination device (%pg) %s: Disabling discard passdown.", 2040 dest_dev, reason); 2041 clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); 2042 } 2043} 2044 2045static void set_discard_limits(struct clone *clone, struct queue_limits *limits) 2046{ 2047 struct block_device *dest_bdev = clone->dest_dev->bdev; 2048 struct queue_limits *dest_limits = &bdev_get_queue(dest_bdev)->limits; 2049 2050 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) { 2051 /* No passdown is done so we set our own virtual limits */ 2052 limits->discard_granularity = clone->region_size << SECTOR_SHIFT; 2053 limits->max_discard_sectors = round_down(UINT_MAX >> SECTOR_SHIFT, clone->region_size); 2054 return; 2055 } 2056 2057 /* 2058 * clone_iterate_devices() is stacking both the source and destination 2059 * device limits but discards aren't passed to the source device, so 2060 * inherit destination's limits. 2061 */ 2062 limits->max_discard_sectors = dest_limits->max_discard_sectors; 2063 limits->max_hw_discard_sectors = dest_limits->max_hw_discard_sectors; 2064 limits->discard_granularity = dest_limits->discard_granularity; 2065 limits->discard_alignment = dest_limits->discard_alignment; 2066 limits->discard_misaligned = dest_limits->discard_misaligned; 2067 limits->max_discard_segments = dest_limits->max_discard_segments; 2068} 2069 2070static void clone_io_hints(struct dm_target *ti, struct queue_limits *limits) 2071{ 2072 struct clone *clone = ti->private; 2073 u64 io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 2074 2075 /* 2076 * If the system-determined stacked limits are compatible with 2077 * dm-clone's region size (io_opt is a factor) do not override them. 2078 */ 2079 if (io_opt_sectors < clone->region_size || 2080 do_div(io_opt_sectors, clone->region_size)) { 2081 blk_limits_io_min(limits, clone->region_size << SECTOR_SHIFT); 2082 blk_limits_io_opt(limits, clone->region_size << SECTOR_SHIFT); 2083 } 2084 2085 disable_passdown_if_not_supported(clone); 2086 set_discard_limits(clone, limits); 2087} 2088 2089static int clone_iterate_devices(struct dm_target *ti, 2090 iterate_devices_callout_fn fn, void *data) 2091{ 2092 int ret; 2093 struct clone *clone = ti->private; 2094 struct dm_dev *dest_dev = clone->dest_dev; 2095 struct dm_dev *source_dev = clone->source_dev; 2096 2097 ret = fn(ti, source_dev, 0, ti->len, data); 2098 if (!ret) 2099 ret = fn(ti, dest_dev, 0, ti->len, data); 2100 return ret; 2101} 2102 2103/* 2104 * dm-clone message functions. 2105 */ 2106static void set_hydration_threshold(struct clone *clone, unsigned int nr_regions) 2107{ 2108 WRITE_ONCE(clone->hydration_threshold, nr_regions); 2109 2110 /* 2111 * If user space sets hydration_threshold to zero then the hydration 2112 * will stop. If at a later time the hydration_threshold is increased 2113 * we must restart the hydration process by waking up the worker. 2114 */ 2115 wake_worker(clone); 2116} 2117 2118static void set_hydration_batch_size(struct clone *clone, unsigned int nr_regions) 2119{ 2120 WRITE_ONCE(clone->hydration_batch_size, nr_regions); 2121} 2122 2123static void enable_hydration(struct clone *clone) 2124{ 2125 if (!test_and_set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags)) 2126 wake_worker(clone); 2127} 2128 2129static void disable_hydration(struct clone *clone) 2130{ 2131 clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); 2132} 2133 2134static int clone_message(struct dm_target *ti, unsigned int argc, char **argv, 2135 char *result, unsigned int maxlen) 2136{ 2137 struct clone *clone = ti->private; 2138 unsigned int value; 2139 2140 if (!argc) 2141 return -EINVAL; 2142 2143 if (!strcasecmp(argv[0], "enable_hydration")) { 2144 enable_hydration(clone); 2145 return 0; 2146 } 2147 2148 if (!strcasecmp(argv[0], "disable_hydration")) { 2149 disable_hydration(clone); 2150 return 0; 2151 } 2152 2153 if (argc != 2) 2154 return -EINVAL; 2155 2156 if (!strcasecmp(argv[0], "hydration_threshold")) { 2157 if (kstrtouint(argv[1], 10, &value)) 2158 return -EINVAL; 2159 2160 set_hydration_threshold(clone, value); 2161 2162 return 0; 2163 } 2164 2165 if (!strcasecmp(argv[0], "hydration_batch_size")) { 2166 if (kstrtouint(argv[1], 10, &value)) 2167 return -EINVAL; 2168 2169 set_hydration_batch_size(clone, value); 2170 2171 return 0; 2172 } 2173 2174 DMERR("%s: Unsupported message `%s'", clone_device_name(clone), argv[0]); 2175 return -EINVAL; 2176} 2177 2178static struct target_type clone_target = { 2179 .name = "clone", 2180 .version = {1, 0, 0}, 2181 .module = THIS_MODULE, 2182 .ctr = clone_ctr, 2183 .dtr = clone_dtr, 2184 .map = clone_map, 2185 .end_io = clone_endio, 2186 .postsuspend = clone_postsuspend, 2187 .resume = clone_resume, 2188 .status = clone_status, 2189 .message = clone_message, 2190 .io_hints = clone_io_hints, 2191 .iterate_devices = clone_iterate_devices, 2192}; 2193 2194/*---------------------------------------------------------------------------*/ 2195 2196/* Module functions */ 2197static int __init dm_clone_init(void) 2198{ 2199 int r; 2200 2201 _hydration_cache = KMEM_CACHE(dm_clone_region_hydration, 0); 2202 if (!_hydration_cache) 2203 return -ENOMEM; 2204 2205 r = dm_register_target(&clone_target); 2206 if (r < 0) { 2207 kmem_cache_destroy(_hydration_cache); 2208 return r; 2209 } 2210 2211 return 0; 2212} 2213 2214static void __exit dm_clone_exit(void) 2215{ 2216 dm_unregister_target(&clone_target); 2217 2218 kmem_cache_destroy(_hydration_cache); 2219 _hydration_cache = NULL; 2220} 2221 2222/* Module hooks */ 2223module_init(dm_clone_init); 2224module_exit(dm_clone_exit); 2225 2226MODULE_DESCRIPTION(DM_NAME " clone target"); 2227MODULE_AUTHOR("Nikos Tsironis <ntsironis@arrikto.com>"); 2228MODULE_LICENSE("GPL"); 2229