1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2009-2011 Red Hat, Inc. 4 * 5 * Author: Mikulas Patocka <mpatocka@redhat.com> 6 * 7 * This file is released under the GPL. 8 */ 9 10#include <linux/dm-bufio.h> 11 12#include <linux/device-mapper.h> 13#include <linux/dm-io.h> 14#include <linux/slab.h> 15#include <linux/sched/mm.h> 16#include <linux/jiffies.h> 17#include <linux/vmalloc.h> 18#include <linux/shrinker.h> 19#include <linux/module.h> 20#include <linux/rbtree.h> 21#include <linux/stacktrace.h> 22#include <linux/jump_label.h> 23 24#include "dm.h" 25 26#define DM_MSG_PREFIX "bufio" 27 28/* 29 * Memory management policy: 30 * Limit the number of buffers to DM_BUFIO_MEMORY_PERCENT of main memory 31 * or DM_BUFIO_VMALLOC_PERCENT of vmalloc memory (whichever is lower). 32 * Always allocate at least DM_BUFIO_MIN_BUFFERS buffers. 33 * Start background writeback when there are DM_BUFIO_WRITEBACK_PERCENT 34 * dirty buffers. 35 */ 36#define DM_BUFIO_MIN_BUFFERS 8 37 38#define DM_BUFIO_MEMORY_PERCENT 2 39#define DM_BUFIO_VMALLOC_PERCENT 25 40#define DM_BUFIO_WRITEBACK_RATIO 3 41#define DM_BUFIO_LOW_WATERMARK_RATIO 16 42 43/* 44 * Check buffer ages in this interval (seconds) 45 */ 46#define DM_BUFIO_WORK_TIMER_SECS 30 47 48/* 49 * Free buffers when they are older than this (seconds) 50 */ 51#define DM_BUFIO_DEFAULT_AGE_SECS 300 52 53/* 54 * The nr of bytes of cached data to keep around. 55 */ 56#define DM_BUFIO_DEFAULT_RETAIN_BYTES (256 * 1024) 57 58/* 59 * Align buffer writes to this boundary. 60 * Tests show that SSDs have the highest IOPS when using 4k writes. 61 */ 62#define DM_BUFIO_WRITE_ALIGN 4096 63 64/* 65 * dm_buffer->list_mode 66 */ 67#define LIST_CLEAN 0 68#define LIST_DIRTY 1 69#define LIST_SIZE 2 70 71/*--------------------------------------------------------------*/ 72 73/* 74 * Rather than use an LRU list, we use a clock algorithm where entries 75 * are held in a circular list. When an entry is 'hit' a reference bit 76 * is set. The least recently used entry is approximated by running a 77 * cursor around the list selecting unreferenced entries. Referenced 78 * entries have their reference bit cleared as the cursor passes them. 79 */ 80struct lru_entry { 81 struct list_head list; 82 atomic_t referenced; 83}; 84 85struct lru_iter { 86 struct lru *lru; 87 struct list_head list; 88 struct lru_entry *stop; 89 struct lru_entry *e; 90}; 91 92struct lru { 93 struct list_head *cursor; 94 unsigned long count; 95 96 struct list_head iterators; 97}; 98 99/*--------------*/ 100 101static void lru_init(struct lru *lru) 102{ 103 lru->cursor = NULL; 104 lru->count = 0; 105 INIT_LIST_HEAD(&lru->iterators); 106} 107 108static void lru_destroy(struct lru *lru) 109{ 110 WARN_ON_ONCE(lru->cursor); 111 WARN_ON_ONCE(!list_empty(&lru->iterators)); 112} 113 114/* 115 * Insert a new entry into the lru. 116 */ 117static void lru_insert(struct lru *lru, struct lru_entry *le) 118{ 119 /* 120 * Don't be tempted to set to 1, makes the lru aspect 121 * perform poorly. 122 */ 123 atomic_set(&le->referenced, 0); 124 125 if (lru->cursor) { 126 list_add_tail(&le->list, lru->cursor); 127 } else { 128 INIT_LIST_HEAD(&le->list); 129 lru->cursor = &le->list; 130 } 131 lru->count++; 132} 133 134/*--------------*/ 135 136/* 137 * Convert a list_head pointer to an lru_entry pointer. 138 */ 139static inline struct lru_entry *to_le(struct list_head *l) 140{ 141 return container_of(l, struct lru_entry, list); 142} 143 144/* 145 * Initialize an lru_iter and add it to the list of cursors in the lru. 146 */ 147static void lru_iter_begin(struct lru *lru, struct lru_iter *it) 148{ 149 it->lru = lru; 150 it->stop = lru->cursor ? to_le(lru->cursor->prev) : NULL; 151 it->e = lru->cursor ? to_le(lru->cursor) : NULL; 152 list_add(&it->list, &lru->iterators); 153} 154 155/* 156 * Remove an lru_iter from the list of cursors in the lru. 157 */ 158static inline void lru_iter_end(struct lru_iter *it) 159{ 160 list_del(&it->list); 161} 162 163/* Predicate function type to be used with lru_iter_next */ 164typedef bool (*iter_predicate)(struct lru_entry *le, void *context); 165 166/* 167 * Advance the cursor to the next entry that passes the 168 * predicate, and return that entry. Returns NULL if the 169 * iteration is complete. 170 */ 171static struct lru_entry *lru_iter_next(struct lru_iter *it, 172 iter_predicate pred, void *context) 173{ 174 struct lru_entry *e; 175 176 while (it->e) { 177 e = it->e; 178 179 /* advance the cursor */ 180 if (it->e == it->stop) 181 it->e = NULL; 182 else 183 it->e = to_le(it->e->list.next); 184 185 if (pred(e, context)) 186 return e; 187 } 188 189 return NULL; 190} 191 192/* 193 * Invalidate a specific lru_entry and update all cursors in 194 * the lru accordingly. 195 */ 196static void lru_iter_invalidate(struct lru *lru, struct lru_entry *e) 197{ 198 struct lru_iter *it; 199 200 list_for_each_entry(it, &lru->iterators, list) { 201 /* Move c->e forwards if necc. */ 202 if (it->e == e) { 203 it->e = to_le(it->e->list.next); 204 if (it->e == e) 205 it->e = NULL; 206 } 207 208 /* Move it->stop backwards if necc. */ 209 if (it->stop == e) { 210 it->stop = to_le(it->stop->list.prev); 211 if (it->stop == e) 212 it->stop = NULL; 213 } 214 } 215} 216 217/*--------------*/ 218 219/* 220 * Remove a specific entry from the lru. 221 */ 222static void lru_remove(struct lru *lru, struct lru_entry *le) 223{ 224 lru_iter_invalidate(lru, le); 225 if (lru->count == 1) { 226 lru->cursor = NULL; 227 } else { 228 if (lru->cursor == &le->list) 229 lru->cursor = lru->cursor->next; 230 list_del(&le->list); 231 } 232 lru->count--; 233} 234 235/* 236 * Mark as referenced. 237 */ 238static inline void lru_reference(struct lru_entry *le) 239{ 240 atomic_set(&le->referenced, 1); 241} 242 243/*--------------*/ 244 245/* 246 * Remove the least recently used entry (approx), that passes the predicate. 247 * Returns NULL on failure. 248 */ 249enum evict_result { 250 ER_EVICT, 251 ER_DONT_EVICT, 252 ER_STOP, /* stop looking for something to evict */ 253}; 254 255typedef enum evict_result (*le_predicate)(struct lru_entry *le, void *context); 256 257static struct lru_entry *lru_evict(struct lru *lru, le_predicate pred, void *context, bool no_sleep) 258{ 259 unsigned long tested = 0; 260 struct list_head *h = lru->cursor; 261 struct lru_entry *le; 262 263 if (!h) 264 return NULL; 265 /* 266 * In the worst case we have to loop around twice. Once to clear 267 * the reference flags, and then again to discover the predicate 268 * fails for all entries. 269 */ 270 while (tested < lru->count) { 271 le = container_of(h, struct lru_entry, list); 272 273 if (atomic_read(&le->referenced)) { 274 atomic_set(&le->referenced, 0); 275 } else { 276 tested++; 277 switch (pred(le, context)) { 278 case ER_EVICT: 279 /* 280 * Adjust the cursor, so we start the next 281 * search from here. 282 */ 283 lru->cursor = le->list.next; 284 lru_remove(lru, le); 285 return le; 286 287 case ER_DONT_EVICT: 288 break; 289 290 case ER_STOP: 291 lru->cursor = le->list.next; 292 return NULL; 293 } 294 } 295 296 h = h->next; 297 298 if (!no_sleep) 299 cond_resched(); 300 } 301 302 return NULL; 303} 304 305/*--------------------------------------------------------------*/ 306 307/* 308 * Buffer state bits. 309 */ 310#define B_READING 0 311#define B_WRITING 1 312#define B_DIRTY 2 313 314/* 315 * Describes how the block was allocated: 316 * kmem_cache_alloc(), __get_free_pages() or vmalloc(). 317 * See the comment at alloc_buffer_data. 318 */ 319enum data_mode { 320 DATA_MODE_SLAB = 0, 321 DATA_MODE_GET_FREE_PAGES = 1, 322 DATA_MODE_VMALLOC = 2, 323 DATA_MODE_LIMIT = 3 324}; 325 326struct dm_buffer { 327 /* protected by the locks in dm_buffer_cache */ 328 struct rb_node node; 329 330 /* immutable, so don't need protecting */ 331 sector_t block; 332 void *data; 333 unsigned char data_mode; /* DATA_MODE_* */ 334 335 /* 336 * These two fields are used in isolation, so do not need 337 * a surrounding lock. 338 */ 339 atomic_t hold_count; 340 unsigned long last_accessed; 341 342 /* 343 * Everything else is protected by the mutex in 344 * dm_bufio_client 345 */ 346 unsigned long state; 347 struct lru_entry lru; 348 unsigned char list_mode; /* LIST_* */ 349 blk_status_t read_error; 350 blk_status_t write_error; 351 unsigned int dirty_start; 352 unsigned int dirty_end; 353 unsigned int write_start; 354 unsigned int write_end; 355 struct list_head write_list; 356 struct dm_bufio_client *c; 357 void (*end_io)(struct dm_buffer *b, blk_status_t bs); 358#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 359#define MAX_STACK 10 360 unsigned int stack_len; 361 unsigned long stack_entries[MAX_STACK]; 362#endif 363}; 364 365/*--------------------------------------------------------------*/ 366 367/* 368 * The buffer cache manages buffers, particularly: 369 * - inc/dec of holder count 370 * - setting the last_accessed field 371 * - maintains clean/dirty state along with lru 372 * - selecting buffers that match predicates 373 * 374 * It does *not* handle: 375 * - allocation/freeing of buffers. 376 * - IO 377 * - Eviction or cache sizing. 378 * 379 * cache_get() and cache_put() are threadsafe, you do not need to 380 * protect these calls with a surrounding mutex. All the other 381 * methods are not threadsafe; they do use locking primitives, but 382 * only enough to ensure get/put are threadsafe. 383 */ 384 385struct buffer_tree { 386 union { 387 struct rw_semaphore lock; 388 rwlock_t spinlock; 389 } u; 390 struct rb_root root; 391} ____cacheline_aligned_in_smp; 392 393struct dm_buffer_cache { 394 struct lru lru[LIST_SIZE]; 395 /* 396 * We spread entries across multiple trees to reduce contention 397 * on the locks. 398 */ 399 unsigned int num_locks; 400 bool no_sleep; 401 struct buffer_tree trees[]; 402}; 403 404static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled); 405 406static inline unsigned int cache_index(sector_t block, unsigned int num_locks) 407{ 408 return dm_hash_locks_index(block, num_locks); 409} 410 411static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) 412{ 413 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 414 read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 415 else 416 down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 417} 418 419static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block) 420{ 421 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 422 read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 423 else 424 up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 425} 426 427static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block) 428{ 429 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 430 write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 431 else 432 down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 433} 434 435static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block) 436{ 437 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) 438 write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); 439 else 440 up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); 441} 442 443/* 444 * Sometimes we want to repeatedly get and drop locks as part of an iteration. 445 * This struct helps avoid redundant drop and gets of the same lock. 446 */ 447struct lock_history { 448 struct dm_buffer_cache *cache; 449 bool write; 450 unsigned int previous; 451 unsigned int no_previous; 452}; 453 454static void lh_init(struct lock_history *lh, struct dm_buffer_cache *cache, bool write) 455{ 456 lh->cache = cache; 457 lh->write = write; 458 lh->no_previous = cache->num_locks; 459 lh->previous = lh->no_previous; 460} 461 462static void __lh_lock(struct lock_history *lh, unsigned int index) 463{ 464 if (lh->write) { 465 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 466 write_lock_bh(&lh->cache->trees[index].u.spinlock); 467 else 468 down_write(&lh->cache->trees[index].u.lock); 469 } else { 470 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 471 read_lock_bh(&lh->cache->trees[index].u.spinlock); 472 else 473 down_read(&lh->cache->trees[index].u.lock); 474 } 475} 476 477static void __lh_unlock(struct lock_history *lh, unsigned int index) 478{ 479 if (lh->write) { 480 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 481 write_unlock_bh(&lh->cache->trees[index].u.spinlock); 482 else 483 up_write(&lh->cache->trees[index].u.lock); 484 } else { 485 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) 486 read_unlock_bh(&lh->cache->trees[index].u.spinlock); 487 else 488 up_read(&lh->cache->trees[index].u.lock); 489 } 490} 491 492/* 493 * Make sure you call this since it will unlock the final lock. 494 */ 495static void lh_exit(struct lock_history *lh) 496{ 497 if (lh->previous != lh->no_previous) { 498 __lh_unlock(lh, lh->previous); 499 lh->previous = lh->no_previous; 500 } 501} 502 503/* 504 * Named 'next' because there is no corresponding 505 * 'up/unlock' call since it's done automatically. 506 */ 507static void lh_next(struct lock_history *lh, sector_t b) 508{ 509 unsigned int index = cache_index(b, lh->no_previous); /* no_previous is num_locks */ 510 511 if (lh->previous != lh->no_previous) { 512 if (lh->previous != index) { 513 __lh_unlock(lh, lh->previous); 514 __lh_lock(lh, index); 515 lh->previous = index; 516 } 517 } else { 518 __lh_lock(lh, index); 519 lh->previous = index; 520 } 521} 522 523static inline struct dm_buffer *le_to_buffer(struct lru_entry *le) 524{ 525 return container_of(le, struct dm_buffer, lru); 526} 527 528static struct dm_buffer *list_to_buffer(struct list_head *l) 529{ 530 struct lru_entry *le = list_entry(l, struct lru_entry, list); 531 532 if (!le) 533 return NULL; 534 535 return le_to_buffer(le); 536} 537 538static void cache_init(struct dm_buffer_cache *bc, unsigned int num_locks, bool no_sleep) 539{ 540 unsigned int i; 541 542 bc->num_locks = num_locks; 543 bc->no_sleep = no_sleep; 544 545 for (i = 0; i < bc->num_locks; i++) { 546 if (no_sleep) 547 rwlock_init(&bc->trees[i].u.spinlock); 548 else 549 init_rwsem(&bc->trees[i].u.lock); 550 bc->trees[i].root = RB_ROOT; 551 } 552 553 lru_init(&bc->lru[LIST_CLEAN]); 554 lru_init(&bc->lru[LIST_DIRTY]); 555} 556 557static void cache_destroy(struct dm_buffer_cache *bc) 558{ 559 unsigned int i; 560 561 for (i = 0; i < bc->num_locks; i++) 562 WARN_ON_ONCE(!RB_EMPTY_ROOT(&bc->trees[i].root)); 563 564 lru_destroy(&bc->lru[LIST_CLEAN]); 565 lru_destroy(&bc->lru[LIST_DIRTY]); 566} 567 568/*--------------*/ 569 570/* 571 * not threadsafe, or racey depending how you look at it 572 */ 573static inline unsigned long cache_count(struct dm_buffer_cache *bc, int list_mode) 574{ 575 return bc->lru[list_mode].count; 576} 577 578static inline unsigned long cache_total(struct dm_buffer_cache *bc) 579{ 580 return cache_count(bc, LIST_CLEAN) + cache_count(bc, LIST_DIRTY); 581} 582 583/*--------------*/ 584 585/* 586 * Gets a specific buffer, indexed by block. 587 * If the buffer is found then its holder count will be incremented and 588 * lru_reference will be called. 589 * 590 * threadsafe 591 */ 592static struct dm_buffer *__cache_get(const struct rb_root *root, sector_t block) 593{ 594 struct rb_node *n = root->rb_node; 595 struct dm_buffer *b; 596 597 while (n) { 598 b = container_of(n, struct dm_buffer, node); 599 600 if (b->block == block) 601 return b; 602 603 n = block < b->block ? n->rb_left : n->rb_right; 604 } 605 606 return NULL; 607} 608 609static void __cache_inc_buffer(struct dm_buffer *b) 610{ 611 atomic_inc(&b->hold_count); 612 WRITE_ONCE(b->last_accessed, jiffies); 613} 614 615static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block) 616{ 617 struct dm_buffer *b; 618 619 cache_read_lock(bc, block); 620 b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block); 621 if (b) { 622 lru_reference(&b->lru); 623 __cache_inc_buffer(b); 624 } 625 cache_read_unlock(bc, block); 626 627 return b; 628} 629 630/*--------------*/ 631 632/* 633 * Returns true if the hold count hits zero. 634 * threadsafe 635 */ 636static bool cache_put(struct dm_buffer_cache *bc, struct dm_buffer *b) 637{ 638 bool r; 639 640 cache_read_lock(bc, b->block); 641 BUG_ON(!atomic_read(&b->hold_count)); 642 r = atomic_dec_and_test(&b->hold_count); 643 cache_read_unlock(bc, b->block); 644 645 return r; 646} 647 648/*--------------*/ 649 650typedef enum evict_result (*b_predicate)(struct dm_buffer *, void *); 651 652/* 653 * Evicts a buffer based on a predicate. The oldest buffer that 654 * matches the predicate will be selected. In addition to the 655 * predicate the hold_count of the selected buffer will be zero. 656 */ 657struct evict_wrapper { 658 struct lock_history *lh; 659 b_predicate pred; 660 void *context; 661}; 662 663/* 664 * Wraps the buffer predicate turning it into an lru predicate. Adds 665 * extra test for hold_count. 666 */ 667static enum evict_result __evict_pred(struct lru_entry *le, void *context) 668{ 669 struct evict_wrapper *w = context; 670 struct dm_buffer *b = le_to_buffer(le); 671 672 lh_next(w->lh, b->block); 673 674 if (atomic_read(&b->hold_count)) 675 return ER_DONT_EVICT; 676 677 return w->pred(b, w->context); 678} 679 680static struct dm_buffer *__cache_evict(struct dm_buffer_cache *bc, int list_mode, 681 b_predicate pred, void *context, 682 struct lock_history *lh) 683{ 684 struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context}; 685 struct lru_entry *le; 686 struct dm_buffer *b; 687 688 le = lru_evict(&bc->lru[list_mode], __evict_pred, &w, bc->no_sleep); 689 if (!le) 690 return NULL; 691 692 b = le_to_buffer(le); 693 /* __evict_pred will have locked the appropriate tree. */ 694 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); 695 696 return b; 697} 698 699static struct dm_buffer *cache_evict(struct dm_buffer_cache *bc, int list_mode, 700 b_predicate pred, void *context) 701{ 702 struct dm_buffer *b; 703 struct lock_history lh; 704 705 lh_init(&lh, bc, true); 706 b = __cache_evict(bc, list_mode, pred, context, &lh); 707 lh_exit(&lh); 708 709 return b; 710} 711 712/*--------------*/ 713 714/* 715 * Mark a buffer as clean or dirty. Not threadsafe. 716 */ 717static void cache_mark(struct dm_buffer_cache *bc, struct dm_buffer *b, int list_mode) 718{ 719 cache_write_lock(bc, b->block); 720 if (list_mode != b->list_mode) { 721 lru_remove(&bc->lru[b->list_mode], &b->lru); 722 b->list_mode = list_mode; 723 lru_insert(&bc->lru[b->list_mode], &b->lru); 724 } 725 cache_write_unlock(bc, b->block); 726} 727 728/*--------------*/ 729 730/* 731 * Runs through the lru associated with 'old_mode', if the predicate matches then 732 * it moves them to 'new_mode'. Not threadsafe. 733 */ 734static void __cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode, 735 b_predicate pred, void *context, struct lock_history *lh) 736{ 737 struct lru_entry *le; 738 struct dm_buffer *b; 739 struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context}; 740 741 while (true) { 742 le = lru_evict(&bc->lru[old_mode], __evict_pred, &w, bc->no_sleep); 743 if (!le) 744 break; 745 746 b = le_to_buffer(le); 747 b->list_mode = new_mode; 748 lru_insert(&bc->lru[b->list_mode], &b->lru); 749 } 750} 751 752static void cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode, 753 b_predicate pred, void *context) 754{ 755 struct lock_history lh; 756 757 lh_init(&lh, bc, true); 758 __cache_mark_many(bc, old_mode, new_mode, pred, context, &lh); 759 lh_exit(&lh); 760} 761 762/*--------------*/ 763 764/* 765 * Iterates through all clean or dirty entries calling a function for each 766 * entry. The callback may terminate the iteration early. Not threadsafe. 767 */ 768 769/* 770 * Iterator functions should return one of these actions to indicate 771 * how the iteration should proceed. 772 */ 773enum it_action { 774 IT_NEXT, 775 IT_COMPLETE, 776}; 777 778typedef enum it_action (*iter_fn)(struct dm_buffer *b, void *context); 779 780static void __cache_iterate(struct dm_buffer_cache *bc, int list_mode, 781 iter_fn fn, void *context, struct lock_history *lh) 782{ 783 struct lru *lru = &bc->lru[list_mode]; 784 struct lru_entry *le, *first; 785 786 if (!lru->cursor) 787 return; 788 789 first = le = to_le(lru->cursor); 790 do { 791 struct dm_buffer *b = le_to_buffer(le); 792 793 lh_next(lh, b->block); 794 795 switch (fn(b, context)) { 796 case IT_NEXT: 797 break; 798 799 case IT_COMPLETE: 800 return; 801 } 802 cond_resched(); 803 804 le = to_le(le->list.next); 805 } while (le != first); 806} 807 808static void cache_iterate(struct dm_buffer_cache *bc, int list_mode, 809 iter_fn fn, void *context) 810{ 811 struct lock_history lh; 812 813 lh_init(&lh, bc, false); 814 __cache_iterate(bc, list_mode, fn, context, &lh); 815 lh_exit(&lh); 816} 817 818/*--------------*/ 819 820/* 821 * Passes ownership of the buffer to the cache. Returns false if the 822 * buffer was already present (in which case ownership does not pass). 823 * eg, a race with another thread. 824 * 825 * Holder count should be 1 on insertion. 826 * 827 * Not threadsafe. 828 */ 829static bool __cache_insert(struct rb_root *root, struct dm_buffer *b) 830{ 831 struct rb_node **new = &root->rb_node, *parent = NULL; 832 struct dm_buffer *found; 833 834 while (*new) { 835 found = container_of(*new, struct dm_buffer, node); 836 837 if (found->block == b->block) 838 return false; 839 840 parent = *new; 841 new = b->block < found->block ? 842 &found->node.rb_left : &found->node.rb_right; 843 } 844 845 rb_link_node(&b->node, parent, new); 846 rb_insert_color(&b->node, root); 847 848 return true; 849} 850 851static bool cache_insert(struct dm_buffer_cache *bc, struct dm_buffer *b) 852{ 853 bool r; 854 855 if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE)) 856 return false; 857 858 cache_write_lock(bc, b->block); 859 BUG_ON(atomic_read(&b->hold_count) != 1); 860 r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b); 861 if (r) 862 lru_insert(&bc->lru[b->list_mode], &b->lru); 863 cache_write_unlock(bc, b->block); 864 865 return r; 866} 867 868/*--------------*/ 869 870/* 871 * Removes buffer from cache, ownership of the buffer passes back to the caller. 872 * Fails if the hold_count is not one (ie. the caller holds the only reference). 873 * 874 * Not threadsafe. 875 */ 876static bool cache_remove(struct dm_buffer_cache *bc, struct dm_buffer *b) 877{ 878 bool r; 879 880 cache_write_lock(bc, b->block); 881 882 if (atomic_read(&b->hold_count) != 1) { 883 r = false; 884 } else { 885 r = true; 886 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); 887 lru_remove(&bc->lru[b->list_mode], &b->lru); 888 } 889 890 cache_write_unlock(bc, b->block); 891 892 return r; 893} 894 895/*--------------*/ 896 897typedef void (*b_release)(struct dm_buffer *); 898 899static struct dm_buffer *__find_next(struct rb_root *root, sector_t block) 900{ 901 struct rb_node *n = root->rb_node; 902 struct dm_buffer *b; 903 struct dm_buffer *best = NULL; 904 905 while (n) { 906 b = container_of(n, struct dm_buffer, node); 907 908 if (b->block == block) 909 return b; 910 911 if (block <= b->block) { 912 n = n->rb_left; 913 best = b; 914 } else { 915 n = n->rb_right; 916 } 917 } 918 919 return best; 920} 921 922static void __remove_range(struct dm_buffer_cache *bc, 923 struct rb_root *root, 924 sector_t begin, sector_t end, 925 b_predicate pred, b_release release) 926{ 927 struct dm_buffer *b; 928 929 while (true) { 930 cond_resched(); 931 932 b = __find_next(root, begin); 933 if (!b || (b->block >= end)) 934 break; 935 936 begin = b->block + 1; 937 938 if (atomic_read(&b->hold_count)) 939 continue; 940 941 if (pred(b, NULL) == ER_EVICT) { 942 rb_erase(&b->node, root); 943 lru_remove(&bc->lru[b->list_mode], &b->lru); 944 release(b); 945 } 946 } 947} 948 949static void cache_remove_range(struct dm_buffer_cache *bc, 950 sector_t begin, sector_t end, 951 b_predicate pred, b_release release) 952{ 953 unsigned int i; 954 955 BUG_ON(bc->no_sleep); 956 for (i = 0; i < bc->num_locks; i++) { 957 down_write(&bc->trees[i].u.lock); 958 __remove_range(bc, &bc->trees[i].root, begin, end, pred, release); 959 up_write(&bc->trees[i].u.lock); 960 } 961} 962 963/*----------------------------------------------------------------*/ 964 965/* 966 * Linking of buffers: 967 * All buffers are linked to buffer_cache with their node field. 968 * 969 * Clean buffers that are not being written (B_WRITING not set) 970 * are linked to lru[LIST_CLEAN] with their lru_list field. 971 * 972 * Dirty and clean buffers that are being written are linked to 973 * lru[LIST_DIRTY] with their lru_list field. When the write 974 * finishes, the buffer cannot be relinked immediately (because we 975 * are in an interrupt context and relinking requires process 976 * context), so some clean-not-writing buffers can be held on 977 * dirty_lru too. They are later added to lru in the process 978 * context. 979 */ 980struct dm_bufio_client { 981 struct block_device *bdev; 982 unsigned int block_size; 983 s8 sectors_per_block_bits; 984 985 bool no_sleep; 986 struct mutex lock; 987 spinlock_t spinlock; 988 989 int async_write_error; 990 991 void (*alloc_callback)(struct dm_buffer *buf); 992 void (*write_callback)(struct dm_buffer *buf); 993 struct kmem_cache *slab_buffer; 994 struct kmem_cache *slab_cache; 995 struct dm_io_client *dm_io; 996 997 struct list_head reserved_buffers; 998 unsigned int need_reserved_buffers; 999 1000 unsigned int minimum_buffers; 1001 1002 sector_t start; 1003 1004 struct shrinker *shrinker; 1005 struct work_struct shrink_work; 1006 atomic_long_t need_shrink; 1007 1008 wait_queue_head_t free_buffer_wait; 1009 1010 struct list_head client_list; 1011 1012 /* 1013 * Used by global_cleanup to sort the clients list. 1014 */ 1015 unsigned long oldest_buffer; 1016 1017 struct dm_buffer_cache cache; /* must be last member */ 1018}; 1019 1020/*----------------------------------------------------------------*/ 1021 1022#define dm_bufio_in_request() (!!current->bio_list) 1023 1024static void dm_bufio_lock(struct dm_bufio_client *c) 1025{ 1026 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) 1027 spin_lock_bh(&c->spinlock); 1028 else 1029 mutex_lock_nested(&c->lock, dm_bufio_in_request()); 1030} 1031 1032static void dm_bufio_unlock(struct dm_bufio_client *c) 1033{ 1034 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) 1035 spin_unlock_bh(&c->spinlock); 1036 else 1037 mutex_unlock(&c->lock); 1038} 1039 1040/*----------------------------------------------------------------*/ 1041 1042/* 1043 * Default cache size: available memory divided by the ratio. 1044 */ 1045static unsigned long dm_bufio_default_cache_size; 1046 1047/* 1048 * Total cache size set by the user. 1049 */ 1050static unsigned long dm_bufio_cache_size; 1051 1052/* 1053 * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change 1054 * at any time. If it disagrees, the user has changed cache size. 1055 */ 1056static unsigned long dm_bufio_cache_size_latch; 1057 1058static DEFINE_SPINLOCK(global_spinlock); 1059 1060/* 1061 * Buffers are freed after this timeout 1062 */ 1063static unsigned int dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS; 1064static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES; 1065 1066static unsigned long dm_bufio_peak_allocated; 1067static unsigned long dm_bufio_allocated_kmem_cache; 1068static unsigned long dm_bufio_allocated_get_free_pages; 1069static unsigned long dm_bufio_allocated_vmalloc; 1070static unsigned long dm_bufio_current_allocated; 1071 1072/*----------------------------------------------------------------*/ 1073 1074/* 1075 * The current number of clients. 1076 */ 1077static int dm_bufio_client_count; 1078 1079/* 1080 * The list of all clients. 1081 */ 1082static LIST_HEAD(dm_bufio_all_clients); 1083 1084/* 1085 * This mutex protects dm_bufio_cache_size_latch and dm_bufio_client_count 1086 */ 1087static DEFINE_MUTEX(dm_bufio_clients_lock); 1088 1089static struct workqueue_struct *dm_bufio_wq; 1090static struct delayed_work dm_bufio_cleanup_old_work; 1091static struct work_struct dm_bufio_replacement_work; 1092 1093 1094#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 1095static void buffer_record_stack(struct dm_buffer *b) 1096{ 1097 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2); 1098} 1099#endif 1100 1101/*----------------------------------------------------------------*/ 1102 1103static void adjust_total_allocated(struct dm_buffer *b, bool unlink) 1104{ 1105 unsigned char data_mode; 1106 long diff; 1107 1108 static unsigned long * const class_ptr[DATA_MODE_LIMIT] = { 1109 &dm_bufio_allocated_kmem_cache, 1110 &dm_bufio_allocated_get_free_pages, 1111 &dm_bufio_allocated_vmalloc, 1112 }; 1113 1114 data_mode = b->data_mode; 1115 diff = (long)b->c->block_size; 1116 if (unlink) 1117 diff = -diff; 1118 1119 spin_lock(&global_spinlock); 1120 1121 *class_ptr[data_mode] += diff; 1122 1123 dm_bufio_current_allocated += diff; 1124 1125 if (dm_bufio_current_allocated > dm_bufio_peak_allocated) 1126 dm_bufio_peak_allocated = dm_bufio_current_allocated; 1127 1128 if (!unlink) { 1129 if (dm_bufio_current_allocated > dm_bufio_cache_size) 1130 queue_work(dm_bufio_wq, &dm_bufio_replacement_work); 1131 } 1132 1133 spin_unlock(&global_spinlock); 1134} 1135 1136/* 1137 * Change the number of clients and recalculate per-client limit. 1138 */ 1139static void __cache_size_refresh(void) 1140{ 1141 if (WARN_ON(!mutex_is_locked(&dm_bufio_clients_lock))) 1142 return; 1143 if (WARN_ON(dm_bufio_client_count < 0)) 1144 return; 1145 1146 dm_bufio_cache_size_latch = READ_ONCE(dm_bufio_cache_size); 1147 1148 /* 1149 * Use default if set to 0 and report the actual cache size used. 1150 */ 1151 if (!dm_bufio_cache_size_latch) { 1152 (void)cmpxchg(&dm_bufio_cache_size, 0, 1153 dm_bufio_default_cache_size); 1154 dm_bufio_cache_size_latch = dm_bufio_default_cache_size; 1155 } 1156} 1157 1158/* 1159 * Allocating buffer data. 1160 * 1161 * Small buffers are allocated with kmem_cache, to use space optimally. 1162 * 1163 * For large buffers, we choose between get_free_pages and vmalloc. 1164 * Each has advantages and disadvantages. 1165 * 1166 * __get_free_pages can randomly fail if the memory is fragmented. 1167 * __vmalloc won't randomly fail, but vmalloc space is limited (it may be 1168 * as low as 128M) so using it for caching is not appropriate. 1169 * 1170 * If the allocation may fail we use __get_free_pages. Memory fragmentation 1171 * won't have a fatal effect here, but it just causes flushes of some other 1172 * buffers and more I/O will be performed. Don't use __get_free_pages if it 1173 * always fails (i.e. order > MAX_PAGE_ORDER). 1174 * 1175 * If the allocation shouldn't fail we use __vmalloc. This is only for the 1176 * initial reserve allocation, so there's no risk of wasting all vmalloc 1177 * space. 1178 */ 1179static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask, 1180 unsigned char *data_mode) 1181{ 1182 if (unlikely(c->slab_cache != NULL)) { 1183 *data_mode = DATA_MODE_SLAB; 1184 return kmem_cache_alloc(c->slab_cache, gfp_mask); 1185 } 1186 1187 if (c->block_size <= KMALLOC_MAX_SIZE && 1188 gfp_mask & __GFP_NORETRY) { 1189 *data_mode = DATA_MODE_GET_FREE_PAGES; 1190 return (void *)__get_free_pages(gfp_mask, 1191 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); 1192 } 1193 1194 *data_mode = DATA_MODE_VMALLOC; 1195 1196 return __vmalloc(c->block_size, gfp_mask); 1197} 1198 1199/* 1200 * Free buffer's data. 1201 */ 1202static void free_buffer_data(struct dm_bufio_client *c, 1203 void *data, unsigned char data_mode) 1204{ 1205 switch (data_mode) { 1206 case DATA_MODE_SLAB: 1207 kmem_cache_free(c->slab_cache, data); 1208 break; 1209 1210 case DATA_MODE_GET_FREE_PAGES: 1211 free_pages((unsigned long)data, 1212 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); 1213 break; 1214 1215 case DATA_MODE_VMALLOC: 1216 vfree(data); 1217 break; 1218 1219 default: 1220 DMCRIT("dm_bufio_free_buffer_data: bad data mode: %d", 1221 data_mode); 1222 BUG(); 1223 } 1224} 1225 1226/* 1227 * Allocate buffer and its data. 1228 */ 1229static struct dm_buffer *alloc_buffer(struct dm_bufio_client *c, gfp_t gfp_mask) 1230{ 1231 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask); 1232 1233 if (!b) 1234 return NULL; 1235 1236 b->c = c; 1237 1238 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode); 1239 if (!b->data) { 1240 kmem_cache_free(c->slab_buffer, b); 1241 return NULL; 1242 } 1243 adjust_total_allocated(b, false); 1244 1245#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 1246 b->stack_len = 0; 1247#endif 1248 return b; 1249} 1250 1251/* 1252 * Free buffer and its data. 1253 */ 1254static void free_buffer(struct dm_buffer *b) 1255{ 1256 struct dm_bufio_client *c = b->c; 1257 1258 adjust_total_allocated(b, true); 1259 free_buffer_data(c, b->data, b->data_mode); 1260 kmem_cache_free(c->slab_buffer, b); 1261} 1262 1263/* 1264 *-------------------------------------------------------------------------- 1265 * Submit I/O on the buffer. 1266 * 1267 * Bio interface is faster but it has some problems: 1268 * the vector list is limited (increasing this limit increases 1269 * memory-consumption per buffer, so it is not viable); 1270 * 1271 * the memory must be direct-mapped, not vmalloced; 1272 * 1273 * If the buffer is small enough (up to DM_BUFIO_INLINE_VECS pages) and 1274 * it is not vmalloced, try using the bio interface. 1275 * 1276 * If the buffer is big, if it is vmalloced or if the underlying device 1277 * rejects the bio because it is too large, use dm-io layer to do the I/O. 1278 * The dm-io layer splits the I/O into multiple requests, avoiding the above 1279 * shortcomings. 1280 *-------------------------------------------------------------------------- 1281 */ 1282 1283/* 1284 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending 1285 * that the request was handled directly with bio interface. 1286 */ 1287static void dmio_complete(unsigned long error, void *context) 1288{ 1289 struct dm_buffer *b = context; 1290 1291 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); 1292} 1293 1294static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, 1295 unsigned int n_sectors, unsigned int offset, 1296 unsigned short ioprio) 1297{ 1298 int r; 1299 struct dm_io_request io_req = { 1300 .bi_opf = op, 1301 .notify.fn = dmio_complete, 1302 .notify.context = b, 1303 .client = b->c->dm_io, 1304 }; 1305 struct dm_io_region region = { 1306 .bdev = b->c->bdev, 1307 .sector = sector, 1308 .count = n_sectors, 1309 }; 1310 1311 if (b->data_mode != DATA_MODE_VMALLOC) { 1312 io_req.mem.type = DM_IO_KMEM; 1313 io_req.mem.ptr.addr = (char *)b->data + offset; 1314 } else { 1315 io_req.mem.type = DM_IO_VMA; 1316 io_req.mem.ptr.vma = (char *)b->data + offset; 1317 } 1318 1319 r = dm_io(&io_req, 1, ®ion, NULL, ioprio); 1320 if (unlikely(r)) 1321 b->end_io(b, errno_to_blk_status(r)); 1322} 1323 1324static void bio_complete(struct bio *bio) 1325{ 1326 struct dm_buffer *b = bio->bi_private; 1327 blk_status_t status = bio->bi_status; 1328 1329 bio_uninit(bio); 1330 kfree(bio); 1331 b->end_io(b, status); 1332} 1333 1334static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, 1335 unsigned int n_sectors, unsigned int offset, 1336 unsigned short ioprio) 1337{ 1338 struct bio *bio; 1339 char *ptr; 1340 unsigned int len; 1341 1342 bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN); 1343 if (!bio) { 1344 use_dmio(b, op, sector, n_sectors, offset, ioprio); 1345 return; 1346 } 1347 bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op); 1348 bio->bi_iter.bi_sector = sector; 1349 bio->bi_end_io = bio_complete; 1350 bio->bi_private = b; 1351 bio->bi_ioprio = ioprio; 1352 1353 ptr = (char *)b->data + offset; 1354 len = n_sectors << SECTOR_SHIFT; 1355 1356 __bio_add_page(bio, virt_to_page(ptr), len, offset_in_page(ptr)); 1357 1358 submit_bio(bio); 1359} 1360 1361static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block) 1362{ 1363 sector_t sector; 1364 1365 if (likely(c->sectors_per_block_bits >= 0)) 1366 sector = block << c->sectors_per_block_bits; 1367 else 1368 sector = block * (c->block_size >> SECTOR_SHIFT); 1369 sector += c->start; 1370 1371 return sector; 1372} 1373 1374static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio, 1375 void (*end_io)(struct dm_buffer *, blk_status_t)) 1376{ 1377 unsigned int n_sectors; 1378 sector_t sector; 1379 unsigned int offset, end; 1380 1381 b->end_io = end_io; 1382 1383 sector = block_to_sector(b->c, b->block); 1384 1385 if (op != REQ_OP_WRITE) { 1386 n_sectors = b->c->block_size >> SECTOR_SHIFT; 1387 offset = 0; 1388 } else { 1389 if (b->c->write_callback) 1390 b->c->write_callback(b); 1391 offset = b->write_start; 1392 end = b->write_end; 1393 offset &= -DM_BUFIO_WRITE_ALIGN; 1394 end += DM_BUFIO_WRITE_ALIGN - 1; 1395 end &= -DM_BUFIO_WRITE_ALIGN; 1396 if (unlikely(end > b->c->block_size)) 1397 end = b->c->block_size; 1398 1399 sector += offset >> SECTOR_SHIFT; 1400 n_sectors = (end - offset) >> SECTOR_SHIFT; 1401 } 1402 1403 if (b->data_mode != DATA_MODE_VMALLOC) 1404 use_bio(b, op, sector, n_sectors, offset, ioprio); 1405 else 1406 use_dmio(b, op, sector, n_sectors, offset, ioprio); 1407} 1408 1409/* 1410 *-------------------------------------------------------------- 1411 * Writing dirty buffers 1412 *-------------------------------------------------------------- 1413 */ 1414 1415/* 1416 * The endio routine for write. 1417 * 1418 * Set the error, clear B_WRITING bit and wake anyone who was waiting on 1419 * it. 1420 */ 1421static void write_endio(struct dm_buffer *b, blk_status_t status) 1422{ 1423 b->write_error = status; 1424 if (unlikely(status)) { 1425 struct dm_bufio_client *c = b->c; 1426 1427 (void)cmpxchg(&c->async_write_error, 0, 1428 blk_status_to_errno(status)); 1429 } 1430 1431 BUG_ON(!test_bit(B_WRITING, &b->state)); 1432 1433 smp_mb__before_atomic(); 1434 clear_bit(B_WRITING, &b->state); 1435 smp_mb__after_atomic(); 1436 1437 wake_up_bit(&b->state, B_WRITING); 1438} 1439 1440/* 1441 * Initiate a write on a dirty buffer, but don't wait for it. 1442 * 1443 * - If the buffer is not dirty, exit. 1444 * - If there some previous write going on, wait for it to finish (we can't 1445 * have two writes on the same buffer simultaneously). 1446 * - Submit our write and don't wait on it. We set B_WRITING indicating 1447 * that there is a write in progress. 1448 */ 1449static void __write_dirty_buffer(struct dm_buffer *b, 1450 struct list_head *write_list) 1451{ 1452 if (!test_bit(B_DIRTY, &b->state)) 1453 return; 1454 1455 clear_bit(B_DIRTY, &b->state); 1456 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 1457 1458 b->write_start = b->dirty_start; 1459 b->write_end = b->dirty_end; 1460 1461 if (!write_list) 1462 submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); 1463 else 1464 list_add_tail(&b->write_list, write_list); 1465} 1466 1467static void __flush_write_list(struct list_head *write_list) 1468{ 1469 struct blk_plug plug; 1470 1471 blk_start_plug(&plug); 1472 while (!list_empty(write_list)) { 1473 struct dm_buffer *b = 1474 list_entry(write_list->next, struct dm_buffer, write_list); 1475 list_del(&b->write_list); 1476 submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); 1477 cond_resched(); 1478 } 1479 blk_finish_plug(&plug); 1480} 1481 1482/* 1483 * Wait until any activity on the buffer finishes. Possibly write the 1484 * buffer if it is dirty. When this function finishes, there is no I/O 1485 * running on the buffer and the buffer is not dirty. 1486 */ 1487static void __make_buffer_clean(struct dm_buffer *b) 1488{ 1489 BUG_ON(atomic_read(&b->hold_count)); 1490 1491 /* smp_load_acquire() pairs with read_endio()'s smp_mb__before_atomic() */ 1492 if (!smp_load_acquire(&b->state)) /* fast case */ 1493 return; 1494 1495 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); 1496 __write_dirty_buffer(b, NULL); 1497 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 1498} 1499 1500static enum evict_result is_clean(struct dm_buffer *b, void *context) 1501{ 1502 struct dm_bufio_client *c = context; 1503 1504 /* These should never happen */ 1505 if (WARN_ON_ONCE(test_bit(B_WRITING, &b->state))) 1506 return ER_DONT_EVICT; 1507 if (WARN_ON_ONCE(test_bit(B_DIRTY, &b->state))) 1508 return ER_DONT_EVICT; 1509 if (WARN_ON_ONCE(b->list_mode != LIST_CLEAN)) 1510 return ER_DONT_EVICT; 1511 1512 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep && 1513 unlikely(test_bit(B_READING, &b->state))) 1514 return ER_DONT_EVICT; 1515 1516 return ER_EVICT; 1517} 1518 1519static enum evict_result is_dirty(struct dm_buffer *b, void *context) 1520{ 1521 /* These should never happen */ 1522 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) 1523 return ER_DONT_EVICT; 1524 if (WARN_ON_ONCE(b->list_mode != LIST_DIRTY)) 1525 return ER_DONT_EVICT; 1526 1527 return ER_EVICT; 1528} 1529 1530/* 1531 * Find some buffer that is not held by anybody, clean it, unlink it and 1532 * return it. 1533 */ 1534static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c) 1535{ 1536 struct dm_buffer *b; 1537 1538 b = cache_evict(&c->cache, LIST_CLEAN, is_clean, c); 1539 if (b) { 1540 /* this also waits for pending reads */ 1541 __make_buffer_clean(b); 1542 return b; 1543 } 1544 1545 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) 1546 return NULL; 1547 1548 b = cache_evict(&c->cache, LIST_DIRTY, is_dirty, NULL); 1549 if (b) { 1550 __make_buffer_clean(b); 1551 return b; 1552 } 1553 1554 return NULL; 1555} 1556 1557/* 1558 * Wait until some other threads free some buffer or release hold count on 1559 * some buffer. 1560 * 1561 * This function is entered with c->lock held, drops it and regains it 1562 * before exiting. 1563 */ 1564static void __wait_for_free_buffer(struct dm_bufio_client *c) 1565{ 1566 DECLARE_WAITQUEUE(wait, current); 1567 1568 add_wait_queue(&c->free_buffer_wait, &wait); 1569 set_current_state(TASK_UNINTERRUPTIBLE); 1570 dm_bufio_unlock(c); 1571 1572 /* 1573 * It's possible to miss a wake up event since we don't always 1574 * hold c->lock when wake_up is called. So we have a timeout here, 1575 * just in case. 1576 */ 1577 io_schedule_timeout(5 * HZ); 1578 1579 remove_wait_queue(&c->free_buffer_wait, &wait); 1580 1581 dm_bufio_lock(c); 1582} 1583 1584enum new_flag { 1585 NF_FRESH = 0, 1586 NF_READ = 1, 1587 NF_GET = 2, 1588 NF_PREFETCH = 3 1589}; 1590 1591/* 1592 * Allocate a new buffer. If the allocation is not possible, wait until 1593 * some other thread frees a buffer. 1594 * 1595 * May drop the lock and regain it. 1596 */ 1597static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf) 1598{ 1599 struct dm_buffer *b; 1600 bool tried_noio_alloc = false; 1601 1602 /* 1603 * dm-bufio is resistant to allocation failures (it just keeps 1604 * one buffer reserved in cases all the allocations fail). 1605 * So set flags to not try too hard: 1606 * GFP_NOWAIT: don't wait; if we need to sleep we'll release our 1607 * mutex and wait ourselves. 1608 * __GFP_NORETRY: don't retry and rather return failure 1609 * __GFP_NOMEMALLOC: don't use emergency reserves 1610 * __GFP_NOWARN: don't print a warning in case of failure 1611 * 1612 * For debugging, if we set the cache size to 1, no new buffers will 1613 * be allocated. 1614 */ 1615 while (1) { 1616 if (dm_bufio_cache_size_latch != 1) { 1617 b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 1618 if (b) 1619 return b; 1620 } 1621 1622 if (nf == NF_PREFETCH) 1623 return NULL; 1624 1625 if (dm_bufio_cache_size_latch != 1 && !tried_noio_alloc) { 1626 dm_bufio_unlock(c); 1627 b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 1628 dm_bufio_lock(c); 1629 if (b) 1630 return b; 1631 tried_noio_alloc = true; 1632 } 1633 1634 if (!list_empty(&c->reserved_buffers)) { 1635 b = list_to_buffer(c->reserved_buffers.next); 1636 list_del(&b->lru.list); 1637 c->need_reserved_buffers++; 1638 1639 return b; 1640 } 1641 1642 b = __get_unclaimed_buffer(c); 1643 if (b) 1644 return b; 1645 1646 __wait_for_free_buffer(c); 1647 } 1648} 1649 1650static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf) 1651{ 1652 struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf); 1653 1654 if (!b) 1655 return NULL; 1656 1657 if (c->alloc_callback) 1658 c->alloc_callback(b); 1659 1660 return b; 1661} 1662 1663/* 1664 * Free a buffer and wake other threads waiting for free buffers. 1665 */ 1666static void __free_buffer_wake(struct dm_buffer *b) 1667{ 1668 struct dm_bufio_client *c = b->c; 1669 1670 b->block = -1; 1671 if (!c->need_reserved_buffers) 1672 free_buffer(b); 1673 else { 1674 list_add(&b->lru.list, &c->reserved_buffers); 1675 c->need_reserved_buffers--; 1676 } 1677 1678 /* 1679 * We hold the bufio lock here, so no one can add entries to the 1680 * wait queue anyway. 1681 */ 1682 if (unlikely(waitqueue_active(&c->free_buffer_wait))) 1683 wake_up(&c->free_buffer_wait); 1684} 1685 1686static enum evict_result cleaned(struct dm_buffer *b, void *context) 1687{ 1688 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) 1689 return ER_DONT_EVICT; /* should never happen */ 1690 1691 if (test_bit(B_DIRTY, &b->state) || test_bit(B_WRITING, &b->state)) 1692 return ER_DONT_EVICT; 1693 else 1694 return ER_EVICT; 1695} 1696 1697static void __move_clean_buffers(struct dm_bufio_client *c) 1698{ 1699 cache_mark_many(&c->cache, LIST_DIRTY, LIST_CLEAN, cleaned, NULL); 1700} 1701 1702struct write_context { 1703 int no_wait; 1704 struct list_head *write_list; 1705}; 1706 1707static enum it_action write_one(struct dm_buffer *b, void *context) 1708{ 1709 struct write_context *wc = context; 1710 1711 if (wc->no_wait && test_bit(B_WRITING, &b->state)) 1712 return IT_COMPLETE; 1713 1714 __write_dirty_buffer(b, wc->write_list); 1715 return IT_NEXT; 1716} 1717 1718static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait, 1719 struct list_head *write_list) 1720{ 1721 struct write_context wc = {.no_wait = no_wait, .write_list = write_list}; 1722 1723 __move_clean_buffers(c); 1724 cache_iterate(&c->cache, LIST_DIRTY, write_one, &wc); 1725} 1726 1727/* 1728 * Check if we're over watermark. 1729 * If we are over threshold_buffers, start freeing buffers. 1730 * If we're over "limit_buffers", block until we get under the limit. 1731 */ 1732static void __check_watermark(struct dm_bufio_client *c, 1733 struct list_head *write_list) 1734{ 1735 if (cache_count(&c->cache, LIST_DIRTY) > 1736 cache_count(&c->cache, LIST_CLEAN) * DM_BUFIO_WRITEBACK_RATIO) 1737 __write_dirty_buffers_async(c, 1, write_list); 1738} 1739 1740/* 1741 *-------------------------------------------------------------- 1742 * Getting a buffer 1743 *-------------------------------------------------------------- 1744 */ 1745 1746static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer *b) 1747{ 1748 /* 1749 * Relying on waitqueue_active() is racey, but we sleep 1750 * with schedule_timeout anyway. 1751 */ 1752 if (cache_put(&c->cache, b) && 1753 unlikely(waitqueue_active(&c->free_buffer_wait))) 1754 wake_up(&c->free_buffer_wait); 1755} 1756 1757/* 1758 * This assumes you have already checked the cache to see if the buffer 1759 * is already present (it will recheck after dropping the lock for allocation). 1760 */ 1761static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block, 1762 enum new_flag nf, int *need_submit, 1763 struct list_head *write_list) 1764{ 1765 struct dm_buffer *b, *new_b = NULL; 1766 1767 *need_submit = 0; 1768 1769 /* This can't be called with NF_GET */ 1770 if (WARN_ON_ONCE(nf == NF_GET)) 1771 return NULL; 1772 1773 new_b = __alloc_buffer_wait(c, nf); 1774 if (!new_b) 1775 return NULL; 1776 1777 /* 1778 * We've had a period where the mutex was unlocked, so need to 1779 * recheck the buffer tree. 1780 */ 1781 b = cache_get(&c->cache, block); 1782 if (b) { 1783 __free_buffer_wake(new_b); 1784 goto found_buffer; 1785 } 1786 1787 __check_watermark(c, write_list); 1788 1789 b = new_b; 1790 atomic_set(&b->hold_count, 1); 1791 WRITE_ONCE(b->last_accessed, jiffies); 1792 b->block = block; 1793 b->read_error = 0; 1794 b->write_error = 0; 1795 b->list_mode = LIST_CLEAN; 1796 1797 if (nf == NF_FRESH) 1798 b->state = 0; 1799 else { 1800 b->state = 1 << B_READING; 1801 *need_submit = 1; 1802 } 1803 1804 /* 1805 * We mustn't insert into the cache until the B_READING state 1806 * is set. Otherwise another thread could get it and use 1807 * it before it had been read. 1808 */ 1809 cache_insert(&c->cache, b); 1810 1811 return b; 1812 1813found_buffer: 1814 if (nf == NF_PREFETCH) { 1815 cache_put_and_wake(c, b); 1816 return NULL; 1817 } 1818 1819 /* 1820 * Note: it is essential that we don't wait for the buffer to be 1821 * read if dm_bufio_get function is used. Both dm_bufio_get and 1822 * dm_bufio_prefetch can be used in the driver request routine. 1823 * If the user called both dm_bufio_prefetch and dm_bufio_get on 1824 * the same buffer, it would deadlock if we waited. 1825 */ 1826 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { 1827 cache_put_and_wake(c, b); 1828 return NULL; 1829 } 1830 1831 return b; 1832} 1833 1834/* 1835 * The endio routine for reading: set the error, clear the bit and wake up 1836 * anyone waiting on the buffer. 1837 */ 1838static void read_endio(struct dm_buffer *b, blk_status_t status) 1839{ 1840 b->read_error = status; 1841 1842 BUG_ON(!test_bit(B_READING, &b->state)); 1843 1844 smp_mb__before_atomic(); 1845 clear_bit(B_READING, &b->state); 1846 smp_mb__after_atomic(); 1847 1848 wake_up_bit(&b->state, B_READING); 1849} 1850 1851/* 1852 * A common routine for dm_bufio_new and dm_bufio_read. Operation of these 1853 * functions is similar except that dm_bufio_new doesn't read the 1854 * buffer from the disk (assuming that the caller overwrites all the data 1855 * and uses dm_bufio_mark_buffer_dirty to write new data back). 1856 */ 1857static void *new_read(struct dm_bufio_client *c, sector_t block, 1858 enum new_flag nf, struct dm_buffer **bp, 1859 unsigned short ioprio) 1860{ 1861 int need_submit = 0; 1862 struct dm_buffer *b; 1863 1864 LIST_HEAD(write_list); 1865 1866 *bp = NULL; 1867 1868 /* 1869 * Fast path, hopefully the block is already in the cache. No need 1870 * to get the client lock for this. 1871 */ 1872 b = cache_get(&c->cache, block); 1873 if (b) { 1874 if (nf == NF_PREFETCH) { 1875 cache_put_and_wake(c, b); 1876 return NULL; 1877 } 1878 1879 /* 1880 * Note: it is essential that we don't wait for the buffer to be 1881 * read if dm_bufio_get function is used. Both dm_bufio_get and 1882 * dm_bufio_prefetch can be used in the driver request routine. 1883 * If the user called both dm_bufio_prefetch and dm_bufio_get on 1884 * the same buffer, it would deadlock if we waited. 1885 */ 1886 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { 1887 cache_put_and_wake(c, b); 1888 return NULL; 1889 } 1890 } 1891 1892 if (!b) { 1893 if (nf == NF_GET) 1894 return NULL; 1895 1896 dm_bufio_lock(c); 1897 b = __bufio_new(c, block, nf, &need_submit, &write_list); 1898 dm_bufio_unlock(c); 1899 } 1900 1901#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 1902 if (b && (atomic_read(&b->hold_count) == 1)) 1903 buffer_record_stack(b); 1904#endif 1905 1906 __flush_write_list(&write_list); 1907 1908 if (!b) 1909 return NULL; 1910 1911 if (need_submit) 1912 submit_io(b, REQ_OP_READ, ioprio, read_endio); 1913 1914 if (nf != NF_GET) /* we already tested this condition above */ 1915 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); 1916 1917 if (b->read_error) { 1918 int error = blk_status_to_errno(b->read_error); 1919 1920 dm_bufio_release(b); 1921 1922 return ERR_PTR(error); 1923 } 1924 1925 *bp = b; 1926 1927 return b->data; 1928} 1929 1930void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, 1931 struct dm_buffer **bp) 1932{ 1933 return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT); 1934} 1935EXPORT_SYMBOL_GPL(dm_bufio_get); 1936 1937static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block, 1938 struct dm_buffer **bp, unsigned short ioprio) 1939{ 1940 if (WARN_ON_ONCE(dm_bufio_in_request())) 1941 return ERR_PTR(-EINVAL); 1942 1943 return new_read(c, block, NF_READ, bp, ioprio); 1944} 1945 1946void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, 1947 struct dm_buffer **bp) 1948{ 1949 return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT); 1950} 1951EXPORT_SYMBOL_GPL(dm_bufio_read); 1952 1953void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, 1954 struct dm_buffer **bp, unsigned short ioprio) 1955{ 1956 return __dm_bufio_read(c, block, bp, ioprio); 1957} 1958EXPORT_SYMBOL_GPL(dm_bufio_read_with_ioprio); 1959 1960void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, 1961 struct dm_buffer **bp) 1962{ 1963 if (WARN_ON_ONCE(dm_bufio_in_request())) 1964 return ERR_PTR(-EINVAL); 1965 1966 return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT); 1967} 1968EXPORT_SYMBOL_GPL(dm_bufio_new); 1969 1970static void __dm_bufio_prefetch(struct dm_bufio_client *c, 1971 sector_t block, unsigned int n_blocks, 1972 unsigned short ioprio) 1973{ 1974 struct blk_plug plug; 1975 1976 LIST_HEAD(write_list); 1977 1978 if (WARN_ON_ONCE(dm_bufio_in_request())) 1979 return; /* should never happen */ 1980 1981 blk_start_plug(&plug); 1982 1983 for (; n_blocks--; block++) { 1984 int need_submit; 1985 struct dm_buffer *b; 1986 1987 b = cache_get(&c->cache, block); 1988 if (b) { 1989 /* already in cache */ 1990 cache_put_and_wake(c, b); 1991 continue; 1992 } 1993 1994 dm_bufio_lock(c); 1995 b = __bufio_new(c, block, NF_PREFETCH, &need_submit, 1996 &write_list); 1997 if (unlikely(!list_empty(&write_list))) { 1998 dm_bufio_unlock(c); 1999 blk_finish_plug(&plug); 2000 __flush_write_list(&write_list); 2001 blk_start_plug(&plug); 2002 dm_bufio_lock(c); 2003 } 2004 if (unlikely(b != NULL)) { 2005 dm_bufio_unlock(c); 2006 2007 if (need_submit) 2008 submit_io(b, REQ_OP_READ, ioprio, read_endio); 2009 dm_bufio_release(b); 2010 2011 cond_resched(); 2012 2013 if (!n_blocks) 2014 goto flush_plug; 2015 dm_bufio_lock(c); 2016 } 2017 dm_bufio_unlock(c); 2018 } 2019 2020flush_plug: 2021 blk_finish_plug(&plug); 2022} 2023 2024void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks) 2025{ 2026 return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT); 2027} 2028EXPORT_SYMBOL_GPL(dm_bufio_prefetch); 2029 2030void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block, 2031 unsigned int n_blocks, unsigned short ioprio) 2032{ 2033 return __dm_bufio_prefetch(c, block, n_blocks, ioprio); 2034} 2035EXPORT_SYMBOL_GPL(dm_bufio_prefetch_with_ioprio); 2036 2037void dm_bufio_release(struct dm_buffer *b) 2038{ 2039 struct dm_bufio_client *c = b->c; 2040 2041 /* 2042 * If there were errors on the buffer, and the buffer is not 2043 * to be written, free the buffer. There is no point in caching 2044 * invalid buffer. 2045 */ 2046 if ((b->read_error || b->write_error) && 2047 !test_bit_acquire(B_READING, &b->state) && 2048 !test_bit(B_WRITING, &b->state) && 2049 !test_bit(B_DIRTY, &b->state)) { 2050 dm_bufio_lock(c); 2051 2052 /* cache remove can fail if there are other holders */ 2053 if (cache_remove(&c->cache, b)) { 2054 __free_buffer_wake(b); 2055 dm_bufio_unlock(c); 2056 return; 2057 } 2058 2059 dm_bufio_unlock(c); 2060 } 2061 2062 cache_put_and_wake(c, b); 2063} 2064EXPORT_SYMBOL_GPL(dm_bufio_release); 2065 2066void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b, 2067 unsigned int start, unsigned int end) 2068{ 2069 struct dm_bufio_client *c = b->c; 2070 2071 BUG_ON(start >= end); 2072 BUG_ON(end > b->c->block_size); 2073 2074 dm_bufio_lock(c); 2075 2076 BUG_ON(test_bit(B_READING, &b->state)); 2077 2078 if (!test_and_set_bit(B_DIRTY, &b->state)) { 2079 b->dirty_start = start; 2080 b->dirty_end = end; 2081 cache_mark(&c->cache, b, LIST_DIRTY); 2082 } else { 2083 if (start < b->dirty_start) 2084 b->dirty_start = start; 2085 if (end > b->dirty_end) 2086 b->dirty_end = end; 2087 } 2088 2089 dm_bufio_unlock(c); 2090} 2091EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty); 2092 2093void dm_bufio_mark_buffer_dirty(struct dm_buffer *b) 2094{ 2095 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size); 2096} 2097EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty); 2098 2099void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c) 2100{ 2101 LIST_HEAD(write_list); 2102 2103 if (WARN_ON_ONCE(dm_bufio_in_request())) 2104 return; /* should never happen */ 2105 2106 dm_bufio_lock(c); 2107 __write_dirty_buffers_async(c, 0, &write_list); 2108 dm_bufio_unlock(c); 2109 __flush_write_list(&write_list); 2110} 2111EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async); 2112 2113/* 2114 * For performance, it is essential that the buffers are written asynchronously 2115 * and simultaneously (so that the block layer can merge the writes) and then 2116 * waited upon. 2117 * 2118 * Finally, we flush hardware disk cache. 2119 */ 2120static bool is_writing(struct lru_entry *e, void *context) 2121{ 2122 struct dm_buffer *b = le_to_buffer(e); 2123 2124 return test_bit(B_WRITING, &b->state); 2125} 2126 2127int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c) 2128{ 2129 int a, f; 2130 unsigned long nr_buffers; 2131 struct lru_entry *e; 2132 struct lru_iter it; 2133 2134 LIST_HEAD(write_list); 2135 2136 dm_bufio_lock(c); 2137 __write_dirty_buffers_async(c, 0, &write_list); 2138 dm_bufio_unlock(c); 2139 __flush_write_list(&write_list); 2140 dm_bufio_lock(c); 2141 2142 nr_buffers = cache_count(&c->cache, LIST_DIRTY); 2143 lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it); 2144 while ((e = lru_iter_next(&it, is_writing, c))) { 2145 struct dm_buffer *b = le_to_buffer(e); 2146 __cache_inc_buffer(b); 2147 2148 BUG_ON(test_bit(B_READING, &b->state)); 2149 2150 if (nr_buffers) { 2151 nr_buffers--; 2152 dm_bufio_unlock(c); 2153 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 2154 dm_bufio_lock(c); 2155 } else { 2156 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 2157 } 2158 2159 if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) 2160 cache_mark(&c->cache, b, LIST_CLEAN); 2161 2162 cache_put_and_wake(c, b); 2163 2164 cond_resched(); 2165 } 2166 lru_iter_end(&it); 2167 2168 wake_up(&c->free_buffer_wait); 2169 dm_bufio_unlock(c); 2170 2171 a = xchg(&c->async_write_error, 0); 2172 f = dm_bufio_issue_flush(c); 2173 if (a) 2174 return a; 2175 2176 return f; 2177} 2178EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers); 2179 2180/* 2181 * Use dm-io to send an empty barrier to flush the device. 2182 */ 2183int dm_bufio_issue_flush(struct dm_bufio_client *c) 2184{ 2185 struct dm_io_request io_req = { 2186 .bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC, 2187 .mem.type = DM_IO_KMEM, 2188 .mem.ptr.addr = NULL, 2189 .client = c->dm_io, 2190 }; 2191 struct dm_io_region io_reg = { 2192 .bdev = c->bdev, 2193 .sector = 0, 2194 .count = 0, 2195 }; 2196 2197 if (WARN_ON_ONCE(dm_bufio_in_request())) 2198 return -EINVAL; 2199 2200 return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); 2201} 2202EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); 2203 2204/* 2205 * Use dm-io to send a discard request to flush the device. 2206 */ 2207int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count) 2208{ 2209 struct dm_io_request io_req = { 2210 .bi_opf = REQ_OP_DISCARD | REQ_SYNC, 2211 .mem.type = DM_IO_KMEM, 2212 .mem.ptr.addr = NULL, 2213 .client = c->dm_io, 2214 }; 2215 struct dm_io_region io_reg = { 2216 .bdev = c->bdev, 2217 .sector = block_to_sector(c, block), 2218 .count = block_to_sector(c, count), 2219 }; 2220 2221 if (WARN_ON_ONCE(dm_bufio_in_request())) 2222 return -EINVAL; /* discards are optional */ 2223 2224 return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); 2225} 2226EXPORT_SYMBOL_GPL(dm_bufio_issue_discard); 2227 2228static bool forget_buffer(struct dm_bufio_client *c, sector_t block) 2229{ 2230 struct dm_buffer *b; 2231 2232 b = cache_get(&c->cache, block); 2233 if (b) { 2234 if (likely(!smp_load_acquire(&b->state))) { 2235 if (cache_remove(&c->cache, b)) 2236 __free_buffer_wake(b); 2237 else 2238 cache_put_and_wake(c, b); 2239 } else { 2240 cache_put_and_wake(c, b); 2241 } 2242 } 2243 2244 return b ? true : false; 2245} 2246 2247/* 2248 * Free the given buffer. 2249 * 2250 * This is just a hint, if the buffer is in use or dirty, this function 2251 * does nothing. 2252 */ 2253void dm_bufio_forget(struct dm_bufio_client *c, sector_t block) 2254{ 2255 dm_bufio_lock(c); 2256 forget_buffer(c, block); 2257 dm_bufio_unlock(c); 2258} 2259EXPORT_SYMBOL_GPL(dm_bufio_forget); 2260 2261static enum evict_result idle(struct dm_buffer *b, void *context) 2262{ 2263 return b->state ? ER_DONT_EVICT : ER_EVICT; 2264} 2265 2266void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks) 2267{ 2268 dm_bufio_lock(c); 2269 cache_remove_range(&c->cache, block, block + n_blocks, idle, __free_buffer_wake); 2270 dm_bufio_unlock(c); 2271} 2272EXPORT_SYMBOL_GPL(dm_bufio_forget_buffers); 2273 2274void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned int n) 2275{ 2276 c->minimum_buffers = n; 2277} 2278EXPORT_SYMBOL_GPL(dm_bufio_set_minimum_buffers); 2279 2280unsigned int dm_bufio_get_block_size(struct dm_bufio_client *c) 2281{ 2282 return c->block_size; 2283} 2284EXPORT_SYMBOL_GPL(dm_bufio_get_block_size); 2285 2286sector_t dm_bufio_get_device_size(struct dm_bufio_client *c) 2287{ 2288 sector_t s = bdev_nr_sectors(c->bdev); 2289 2290 if (s >= c->start) 2291 s -= c->start; 2292 else 2293 s = 0; 2294 if (likely(c->sectors_per_block_bits >= 0)) 2295 s >>= c->sectors_per_block_bits; 2296 else 2297 sector_div(s, c->block_size >> SECTOR_SHIFT); 2298 return s; 2299} 2300EXPORT_SYMBOL_GPL(dm_bufio_get_device_size); 2301 2302struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c) 2303{ 2304 return c->dm_io; 2305} 2306EXPORT_SYMBOL_GPL(dm_bufio_get_dm_io_client); 2307 2308sector_t dm_bufio_get_block_number(struct dm_buffer *b) 2309{ 2310 return b->block; 2311} 2312EXPORT_SYMBOL_GPL(dm_bufio_get_block_number); 2313 2314void *dm_bufio_get_block_data(struct dm_buffer *b) 2315{ 2316 return b->data; 2317} 2318EXPORT_SYMBOL_GPL(dm_bufio_get_block_data); 2319 2320void *dm_bufio_get_aux_data(struct dm_buffer *b) 2321{ 2322 return b + 1; 2323} 2324EXPORT_SYMBOL_GPL(dm_bufio_get_aux_data); 2325 2326struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b) 2327{ 2328 return b->c; 2329} 2330EXPORT_SYMBOL_GPL(dm_bufio_get_client); 2331 2332static enum it_action warn_leak(struct dm_buffer *b, void *context) 2333{ 2334 bool *warned = context; 2335 2336 WARN_ON(!(*warned)); 2337 *warned = true; 2338 DMERR("leaked buffer %llx, hold count %u, list %d", 2339 (unsigned long long)b->block, atomic_read(&b->hold_count), b->list_mode); 2340#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 2341 stack_trace_print(b->stack_entries, b->stack_len, 1); 2342 /* mark unclaimed to avoid WARN_ON at end of drop_buffers() */ 2343 atomic_set(&b->hold_count, 0); 2344#endif 2345 return IT_NEXT; 2346} 2347 2348static void drop_buffers(struct dm_bufio_client *c) 2349{ 2350 int i; 2351 struct dm_buffer *b; 2352 2353 if (WARN_ON(dm_bufio_in_request())) 2354 return; /* should never happen */ 2355 2356 /* 2357 * An optimization so that the buffers are not written one-by-one. 2358 */ 2359 dm_bufio_write_dirty_buffers_async(c); 2360 2361 dm_bufio_lock(c); 2362 2363 while ((b = __get_unclaimed_buffer(c))) 2364 __free_buffer_wake(b); 2365 2366 for (i = 0; i < LIST_SIZE; i++) { 2367 bool warned = false; 2368 2369 cache_iterate(&c->cache, i, warn_leak, &warned); 2370 } 2371 2372#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 2373 while ((b = __get_unclaimed_buffer(c))) 2374 __free_buffer_wake(b); 2375#endif 2376 2377 for (i = 0; i < LIST_SIZE; i++) 2378 WARN_ON(cache_count(&c->cache, i)); 2379 2380 dm_bufio_unlock(c); 2381} 2382 2383static unsigned long get_retain_buffers(struct dm_bufio_client *c) 2384{ 2385 unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes); 2386 2387 if (likely(c->sectors_per_block_bits >= 0)) 2388 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; 2389 else 2390 retain_bytes /= c->block_size; 2391 2392 return retain_bytes; 2393} 2394 2395static void __scan(struct dm_bufio_client *c) 2396{ 2397 int l; 2398 struct dm_buffer *b; 2399 unsigned long freed = 0; 2400 unsigned long retain_target = get_retain_buffers(c); 2401 unsigned long count = cache_total(&c->cache); 2402 2403 for (l = 0; l < LIST_SIZE; l++) { 2404 while (true) { 2405 if (count - freed <= retain_target) 2406 atomic_long_set(&c->need_shrink, 0); 2407 if (!atomic_long_read(&c->need_shrink)) 2408 break; 2409 2410 b = cache_evict(&c->cache, l, 2411 l == LIST_CLEAN ? is_clean : is_dirty, c); 2412 if (!b) 2413 break; 2414 2415 __make_buffer_clean(b); 2416 __free_buffer_wake(b); 2417 2418 atomic_long_dec(&c->need_shrink); 2419 freed++; 2420 cond_resched(); 2421 } 2422 } 2423} 2424 2425static void shrink_work(struct work_struct *w) 2426{ 2427 struct dm_bufio_client *c = container_of(w, struct dm_bufio_client, shrink_work); 2428 2429 dm_bufio_lock(c); 2430 __scan(c); 2431 dm_bufio_unlock(c); 2432} 2433 2434static unsigned long dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 2435{ 2436 struct dm_bufio_client *c; 2437 2438 c = shrink->private_data; 2439 atomic_long_add(sc->nr_to_scan, &c->need_shrink); 2440 queue_work(dm_bufio_wq, &c->shrink_work); 2441 2442 return sc->nr_to_scan; 2443} 2444 2445static unsigned long dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 2446{ 2447 struct dm_bufio_client *c = shrink->private_data; 2448 unsigned long count = cache_total(&c->cache); 2449 unsigned long retain_target = get_retain_buffers(c); 2450 unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink); 2451 2452 if (unlikely(count < retain_target)) 2453 count = 0; 2454 else 2455 count -= retain_target; 2456 2457 if (unlikely(count < queued_for_cleanup)) 2458 count = 0; 2459 else 2460 count -= queued_for_cleanup; 2461 2462 return count; 2463} 2464 2465/* 2466 * Create the buffering interface 2467 */ 2468struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsigned int block_size, 2469 unsigned int reserved_buffers, unsigned int aux_size, 2470 void (*alloc_callback)(struct dm_buffer *), 2471 void (*write_callback)(struct dm_buffer *), 2472 unsigned int flags) 2473{ 2474 int r; 2475 unsigned int num_locks; 2476 struct dm_bufio_client *c; 2477 char slab_name[27]; 2478 2479 if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) { 2480 DMERR("%s: block size not specified or is not multiple of 512b", __func__); 2481 r = -EINVAL; 2482 goto bad_client; 2483 } 2484 2485 num_locks = dm_num_hash_locks(); 2486 c = kzalloc(sizeof(*c) + (num_locks * sizeof(struct buffer_tree)), GFP_KERNEL); 2487 if (!c) { 2488 r = -ENOMEM; 2489 goto bad_client; 2490 } 2491 cache_init(&c->cache, num_locks, (flags & DM_BUFIO_CLIENT_NO_SLEEP) != 0); 2492 2493 c->bdev = bdev; 2494 c->block_size = block_size; 2495 if (is_power_of_2(block_size)) 2496 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT; 2497 else 2498 c->sectors_per_block_bits = -1; 2499 2500 c->alloc_callback = alloc_callback; 2501 c->write_callback = write_callback; 2502 2503 if (flags & DM_BUFIO_CLIENT_NO_SLEEP) { 2504 c->no_sleep = true; 2505 static_branch_inc(&no_sleep_enabled); 2506 } 2507 2508 mutex_init(&c->lock); 2509 spin_lock_init(&c->spinlock); 2510 INIT_LIST_HEAD(&c->reserved_buffers); 2511 c->need_reserved_buffers = reserved_buffers; 2512 2513 dm_bufio_set_minimum_buffers(c, DM_BUFIO_MIN_BUFFERS); 2514 2515 init_waitqueue_head(&c->free_buffer_wait); 2516 c->async_write_error = 0; 2517 2518 c->dm_io = dm_io_client_create(); 2519 if (IS_ERR(c->dm_io)) { 2520 r = PTR_ERR(c->dm_io); 2521 goto bad_dm_io; 2522 } 2523 2524 if (block_size <= KMALLOC_MAX_SIZE && 2525 (block_size < PAGE_SIZE || !is_power_of_2(block_size))) { 2526 unsigned int align = min(1U << __ffs(block_size), (unsigned int)PAGE_SIZE); 2527 2528 snprintf(slab_name, sizeof(slab_name), "dm_bufio_cache-%u", block_size); 2529 c->slab_cache = kmem_cache_create(slab_name, block_size, align, 2530 SLAB_RECLAIM_ACCOUNT, NULL); 2531 if (!c->slab_cache) { 2532 r = -ENOMEM; 2533 goto bad; 2534 } 2535 } 2536 if (aux_size) 2537 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u", aux_size); 2538 else 2539 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer"); 2540 c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size, 2541 0, SLAB_RECLAIM_ACCOUNT, NULL); 2542 if (!c->slab_buffer) { 2543 r = -ENOMEM; 2544 goto bad; 2545 } 2546 2547 while (c->need_reserved_buffers) { 2548 struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL); 2549 2550 if (!b) { 2551 r = -ENOMEM; 2552 goto bad; 2553 } 2554 __free_buffer_wake(b); 2555 } 2556 2557 INIT_WORK(&c->shrink_work, shrink_work); 2558 atomic_long_set(&c->need_shrink, 0); 2559 2560 c->shrinker = shrinker_alloc(0, "dm-bufio:(%u:%u)", 2561 MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev)); 2562 if (!c->shrinker) { 2563 r = -ENOMEM; 2564 goto bad; 2565 } 2566 2567 c->shrinker->count_objects = dm_bufio_shrink_count; 2568 c->shrinker->scan_objects = dm_bufio_shrink_scan; 2569 c->shrinker->seeks = 1; 2570 c->shrinker->batch = 0; 2571 c->shrinker->private_data = c; 2572 2573 shrinker_register(c->shrinker); 2574 2575 mutex_lock(&dm_bufio_clients_lock); 2576 dm_bufio_client_count++; 2577 list_add(&c->client_list, &dm_bufio_all_clients); 2578 __cache_size_refresh(); 2579 mutex_unlock(&dm_bufio_clients_lock); 2580 2581 return c; 2582 2583bad: 2584 while (!list_empty(&c->reserved_buffers)) { 2585 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); 2586 2587 list_del(&b->lru.list); 2588 free_buffer(b); 2589 } 2590 kmem_cache_destroy(c->slab_cache); 2591 kmem_cache_destroy(c->slab_buffer); 2592 dm_io_client_destroy(c->dm_io); 2593bad_dm_io: 2594 mutex_destroy(&c->lock); 2595 if (c->no_sleep) 2596 static_branch_dec(&no_sleep_enabled); 2597 kfree(c); 2598bad_client: 2599 return ERR_PTR(r); 2600} 2601EXPORT_SYMBOL_GPL(dm_bufio_client_create); 2602 2603/* 2604 * Free the buffering interface. 2605 * It is required that there are no references on any buffers. 2606 */ 2607void dm_bufio_client_destroy(struct dm_bufio_client *c) 2608{ 2609 unsigned int i; 2610 2611 drop_buffers(c); 2612 2613 shrinker_free(c->shrinker); 2614 flush_work(&c->shrink_work); 2615 2616 mutex_lock(&dm_bufio_clients_lock); 2617 2618 list_del(&c->client_list); 2619 dm_bufio_client_count--; 2620 __cache_size_refresh(); 2621 2622 mutex_unlock(&dm_bufio_clients_lock); 2623 2624 WARN_ON(c->need_reserved_buffers); 2625 2626 while (!list_empty(&c->reserved_buffers)) { 2627 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); 2628 2629 list_del(&b->lru.list); 2630 free_buffer(b); 2631 } 2632 2633 for (i = 0; i < LIST_SIZE; i++) 2634 if (cache_count(&c->cache, i)) 2635 DMERR("leaked buffer count %d: %lu", i, cache_count(&c->cache, i)); 2636 2637 for (i = 0; i < LIST_SIZE; i++) 2638 WARN_ON(cache_count(&c->cache, i)); 2639 2640 cache_destroy(&c->cache); 2641 kmem_cache_destroy(c->slab_cache); 2642 kmem_cache_destroy(c->slab_buffer); 2643 dm_io_client_destroy(c->dm_io); 2644 mutex_destroy(&c->lock); 2645 if (c->no_sleep) 2646 static_branch_dec(&no_sleep_enabled); 2647 kfree(c); 2648} 2649EXPORT_SYMBOL_GPL(dm_bufio_client_destroy); 2650 2651void dm_bufio_client_reset(struct dm_bufio_client *c) 2652{ 2653 drop_buffers(c); 2654 flush_work(&c->shrink_work); 2655} 2656EXPORT_SYMBOL_GPL(dm_bufio_client_reset); 2657 2658void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start) 2659{ 2660 c->start = start; 2661} 2662EXPORT_SYMBOL_GPL(dm_bufio_set_sector_offset); 2663 2664/*--------------------------------------------------------------*/ 2665 2666static unsigned int get_max_age_hz(void) 2667{ 2668 unsigned int max_age = READ_ONCE(dm_bufio_max_age); 2669 2670 if (max_age > UINT_MAX / HZ) 2671 max_age = UINT_MAX / HZ; 2672 2673 return max_age * HZ; 2674} 2675 2676static bool older_than(struct dm_buffer *b, unsigned long age_hz) 2677{ 2678 return time_after_eq(jiffies, READ_ONCE(b->last_accessed) + age_hz); 2679} 2680 2681struct evict_params { 2682 gfp_t gfp; 2683 unsigned long age_hz; 2684 2685 /* 2686 * This gets updated with the largest last_accessed (ie. most 2687 * recently used) of the evicted buffers. It will not be reinitialised 2688 * by __evict_many(), so you can use it across multiple invocations. 2689 */ 2690 unsigned long last_accessed; 2691}; 2692 2693/* 2694 * We may not be able to evict this buffer if IO pending or the client 2695 * is still using it. 2696 * 2697 * And if GFP_NOFS is used, we must not do any I/O because we hold 2698 * dm_bufio_clients_lock and we would risk deadlock if the I/O gets 2699 * rerouted to different bufio client. 2700 */ 2701static enum evict_result select_for_evict(struct dm_buffer *b, void *context) 2702{ 2703 struct evict_params *params = context; 2704 2705 if (!(params->gfp & __GFP_FS) || 2706 (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) { 2707 if (test_bit_acquire(B_READING, &b->state) || 2708 test_bit(B_WRITING, &b->state) || 2709 test_bit(B_DIRTY, &b->state)) 2710 return ER_DONT_EVICT; 2711 } 2712 2713 return older_than(b, params->age_hz) ? ER_EVICT : ER_STOP; 2714} 2715 2716static unsigned long __evict_many(struct dm_bufio_client *c, 2717 struct evict_params *params, 2718 int list_mode, unsigned long max_count) 2719{ 2720 unsigned long count; 2721 unsigned long last_accessed; 2722 struct dm_buffer *b; 2723 2724 for (count = 0; count < max_count; count++) { 2725 b = cache_evict(&c->cache, list_mode, select_for_evict, params); 2726 if (!b) 2727 break; 2728 2729 last_accessed = READ_ONCE(b->last_accessed); 2730 if (time_after_eq(params->last_accessed, last_accessed)) 2731 params->last_accessed = last_accessed; 2732 2733 __make_buffer_clean(b); 2734 __free_buffer_wake(b); 2735 2736 cond_resched(); 2737 } 2738 2739 return count; 2740} 2741 2742static void evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz) 2743{ 2744 struct evict_params params = {.gfp = 0, .age_hz = age_hz, .last_accessed = 0}; 2745 unsigned long retain = get_retain_buffers(c); 2746 unsigned long count; 2747 LIST_HEAD(write_list); 2748 2749 dm_bufio_lock(c); 2750 2751 __check_watermark(c, &write_list); 2752 if (unlikely(!list_empty(&write_list))) { 2753 dm_bufio_unlock(c); 2754 __flush_write_list(&write_list); 2755 dm_bufio_lock(c); 2756 } 2757 2758 count = cache_total(&c->cache); 2759 if (count > retain) 2760 __evict_many(c, ¶ms, LIST_CLEAN, count - retain); 2761 2762 dm_bufio_unlock(c); 2763} 2764 2765static void cleanup_old_buffers(void) 2766{ 2767 unsigned long max_age_hz = get_max_age_hz(); 2768 struct dm_bufio_client *c; 2769 2770 mutex_lock(&dm_bufio_clients_lock); 2771 2772 __cache_size_refresh(); 2773 2774 list_for_each_entry(c, &dm_bufio_all_clients, client_list) 2775 evict_old_buffers(c, max_age_hz); 2776 2777 mutex_unlock(&dm_bufio_clients_lock); 2778} 2779 2780static void work_fn(struct work_struct *w) 2781{ 2782 cleanup_old_buffers(); 2783 2784 queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work, 2785 DM_BUFIO_WORK_TIMER_SECS * HZ); 2786} 2787 2788/*--------------------------------------------------------------*/ 2789 2790/* 2791 * Global cleanup tries to evict the oldest buffers from across _all_ 2792 * the clients. It does this by repeatedly evicting a few buffers from 2793 * the client that holds the oldest buffer. It's approximate, but hopefully 2794 * good enough. 2795 */ 2796static struct dm_bufio_client *__pop_client(void) 2797{ 2798 struct list_head *h; 2799 2800 if (list_empty(&dm_bufio_all_clients)) 2801 return NULL; 2802 2803 h = dm_bufio_all_clients.next; 2804 list_del(h); 2805 return container_of(h, struct dm_bufio_client, client_list); 2806} 2807 2808/* 2809 * Inserts the client in the global client list based on its 2810 * 'oldest_buffer' field. 2811 */ 2812static void __insert_client(struct dm_bufio_client *new_client) 2813{ 2814 struct dm_bufio_client *c; 2815 struct list_head *h = dm_bufio_all_clients.next; 2816 2817 while (h != &dm_bufio_all_clients) { 2818 c = container_of(h, struct dm_bufio_client, client_list); 2819 if (time_after_eq(c->oldest_buffer, new_client->oldest_buffer)) 2820 break; 2821 h = h->next; 2822 } 2823 2824 list_add_tail(&new_client->client_list, h); 2825} 2826 2827static unsigned long __evict_a_few(unsigned long nr_buffers) 2828{ 2829 unsigned long count; 2830 struct dm_bufio_client *c; 2831 struct evict_params params = { 2832 .gfp = GFP_KERNEL, 2833 .age_hz = 0, 2834 /* set to jiffies in case there are no buffers in this client */ 2835 .last_accessed = jiffies 2836 }; 2837 2838 c = __pop_client(); 2839 if (!c) 2840 return 0; 2841 2842 dm_bufio_lock(c); 2843 count = __evict_many(c, ¶ms, LIST_CLEAN, nr_buffers); 2844 dm_bufio_unlock(c); 2845 2846 if (count) 2847 c->oldest_buffer = params.last_accessed; 2848 __insert_client(c); 2849 2850 return count; 2851} 2852 2853static void check_watermarks(void) 2854{ 2855 LIST_HEAD(write_list); 2856 struct dm_bufio_client *c; 2857 2858 mutex_lock(&dm_bufio_clients_lock); 2859 list_for_each_entry(c, &dm_bufio_all_clients, client_list) { 2860 dm_bufio_lock(c); 2861 __check_watermark(c, &write_list); 2862 dm_bufio_unlock(c); 2863 } 2864 mutex_unlock(&dm_bufio_clients_lock); 2865 2866 __flush_write_list(&write_list); 2867} 2868 2869static void evict_old(void) 2870{ 2871 unsigned long threshold = dm_bufio_cache_size - 2872 dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO; 2873 2874 mutex_lock(&dm_bufio_clients_lock); 2875 while (dm_bufio_current_allocated > threshold) { 2876 if (!__evict_a_few(64)) 2877 break; 2878 cond_resched(); 2879 } 2880 mutex_unlock(&dm_bufio_clients_lock); 2881} 2882 2883static void do_global_cleanup(struct work_struct *w) 2884{ 2885 check_watermarks(); 2886 evict_old(); 2887} 2888 2889/* 2890 *-------------------------------------------------------------- 2891 * Module setup 2892 *-------------------------------------------------------------- 2893 */ 2894 2895/* 2896 * This is called only once for the whole dm_bufio module. 2897 * It initializes memory limit. 2898 */ 2899static int __init dm_bufio_init(void) 2900{ 2901 __u64 mem; 2902 2903 dm_bufio_allocated_kmem_cache = 0; 2904 dm_bufio_allocated_get_free_pages = 0; 2905 dm_bufio_allocated_vmalloc = 0; 2906 dm_bufio_current_allocated = 0; 2907 2908 mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(), 2909 DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT; 2910 2911 if (mem > ULONG_MAX) 2912 mem = ULONG_MAX; 2913 2914#ifdef CONFIG_MMU 2915 if (mem > mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100)) 2916 mem = mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100); 2917#endif 2918 2919 dm_bufio_default_cache_size = mem; 2920 2921 mutex_lock(&dm_bufio_clients_lock); 2922 __cache_size_refresh(); 2923 mutex_unlock(&dm_bufio_clients_lock); 2924 2925 dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0); 2926 if (!dm_bufio_wq) 2927 return -ENOMEM; 2928 2929 INIT_DELAYED_WORK(&dm_bufio_cleanup_old_work, work_fn); 2930 INIT_WORK(&dm_bufio_replacement_work, do_global_cleanup); 2931 queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work, 2932 DM_BUFIO_WORK_TIMER_SECS * HZ); 2933 2934 return 0; 2935} 2936 2937/* 2938 * This is called once when unloading the dm_bufio module. 2939 */ 2940static void __exit dm_bufio_exit(void) 2941{ 2942 int bug = 0; 2943 2944 cancel_delayed_work_sync(&dm_bufio_cleanup_old_work); 2945 destroy_workqueue(dm_bufio_wq); 2946 2947 if (dm_bufio_client_count) { 2948 DMCRIT("%s: dm_bufio_client_count leaked: %d", 2949 __func__, dm_bufio_client_count); 2950 bug = 1; 2951 } 2952 2953 if (dm_bufio_current_allocated) { 2954 DMCRIT("%s: dm_bufio_current_allocated leaked: %lu", 2955 __func__, dm_bufio_current_allocated); 2956 bug = 1; 2957 } 2958 2959 if (dm_bufio_allocated_get_free_pages) { 2960 DMCRIT("%s: dm_bufio_allocated_get_free_pages leaked: %lu", 2961 __func__, dm_bufio_allocated_get_free_pages); 2962 bug = 1; 2963 } 2964 2965 if (dm_bufio_allocated_vmalloc) { 2966 DMCRIT("%s: dm_bufio_vmalloc leaked: %lu", 2967 __func__, dm_bufio_allocated_vmalloc); 2968 bug = 1; 2969 } 2970 2971 WARN_ON(bug); /* leaks are not worth crashing the system */ 2972} 2973 2974module_init(dm_bufio_init) 2975module_exit(dm_bufio_exit) 2976 2977module_param_named(max_cache_size_bytes, dm_bufio_cache_size, ulong, 0644); 2978MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache"); 2979 2980module_param_named(max_age_seconds, dm_bufio_max_age, uint, 0644); 2981MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds"); 2982 2983module_param_named(retain_bytes, dm_bufio_retain_bytes, ulong, 0644); 2984MODULE_PARM_DESC(retain_bytes, "Try to keep at least this many bytes cached in memory"); 2985 2986module_param_named(peak_allocated_bytes, dm_bufio_peak_allocated, ulong, 0644); 2987MODULE_PARM_DESC(peak_allocated_bytes, "Tracks the maximum allocated memory"); 2988 2989module_param_named(allocated_kmem_cache_bytes, dm_bufio_allocated_kmem_cache, ulong, 0444); 2990MODULE_PARM_DESC(allocated_kmem_cache_bytes, "Memory allocated with kmem_cache_alloc"); 2991 2992module_param_named(allocated_get_free_pages_bytes, dm_bufio_allocated_get_free_pages, ulong, 0444); 2993MODULE_PARM_DESC(allocated_get_free_pages_bytes, "Memory allocated with get_free_pages"); 2994 2995module_param_named(allocated_vmalloc_bytes, dm_bufio_allocated_vmalloc, ulong, 0444); 2996MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc"); 2997 2998module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, 0444); 2999MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache"); 3000 3001MODULE_AUTHOR("Mikulas Patocka <dm-devel@lists.linux.dev>"); 3002MODULE_DESCRIPTION(DM_NAME " buffered I/O library"); 3003MODULE_LICENSE("GPL"); 3004