1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright 2023 Red Hat 4 */ 5 6#ifndef DATA_VIO_H 7#define DATA_VIO_H 8 9#include <linux/atomic.h> 10#include <linux/bio.h> 11#include <linux/list.h> 12 13#include "permassert.h" 14 15#include "indexer.h" 16 17#include "block-map.h" 18#include "completion.h" 19#include "constants.h" 20#include "dedupe.h" 21#include "encodings.h" 22#include "logical-zone.h" 23#include "physical-zone.h" 24#include "types.h" 25#include "vdo.h" 26#include "vio.h" 27#include "wait-queue.h" 28 29/* Codes for describing the last asynchronous operation performed on a vio. */ 30enum async_operation_number { 31 MIN_VIO_ASYNC_OPERATION_NUMBER, 32 VIO_ASYNC_OP_LAUNCH = MIN_VIO_ASYNC_OPERATION_NUMBER, 33 VIO_ASYNC_OP_ACKNOWLEDGE_WRITE, 34 VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK, 35 VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK, 36 VIO_ASYNC_OP_LOCK_DUPLICATE_PBN, 37 VIO_ASYNC_OP_CHECK_FOR_DUPLICATION, 38 VIO_ASYNC_OP_CLEANUP, 39 VIO_ASYNC_OP_COMPRESS_DATA_VIO, 40 VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT, 41 VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ, 42 VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE, 43 VIO_ASYNC_OP_HASH_DATA_VIO, 44 VIO_ASYNC_OP_JOURNAL_REMAPPING, 45 VIO_ASYNC_OP_ATTEMPT_PACKING, 46 VIO_ASYNC_OP_PUT_MAPPED_BLOCK, 47 VIO_ASYNC_OP_READ_DATA_VIO, 48 VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX, 49 VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS, 50 VIO_ASYNC_OP_VERIFY_DUPLICATION, 51 VIO_ASYNC_OP_WRITE_DATA_VIO, 52 MAX_VIO_ASYNC_OPERATION_NUMBER, 53} __packed; 54 55struct lbn_lock { 56 logical_block_number_t lbn; 57 bool locked; 58 struct vdo_wait_queue waiters; 59 struct logical_zone *zone; 60}; 61 62/* A position in the arboreal block map at a specific level. */ 63struct block_map_tree_slot { 64 page_number_t page_index; 65 struct block_map_slot block_map_slot; 66}; 67 68/* Fields for using the arboreal block map. */ 69struct tree_lock { 70 /* The current height at which this data_vio is operating */ 71 height_t height; 72 /* The block map tree for this LBN */ 73 root_count_t root_index; 74 /* Whether we hold a page lock */ 75 bool locked; 76 /* The key for the lock map */ 77 u64 key; 78 /* The queue of waiters for the page this vio is allocating or loading */ 79 struct vdo_wait_queue waiters; 80 /* The block map tree slots for this LBN */ 81 struct block_map_tree_slot tree_slots[VDO_BLOCK_MAP_TREE_HEIGHT + 1]; 82}; 83 84struct zoned_pbn { 85 physical_block_number_t pbn; 86 enum block_mapping_state state; 87 struct physical_zone *zone; 88}; 89 90/* 91 * Where a data_vio is on the compression path; advance_compression_stage() depends on the order of 92 * this enum. 93 */ 94enum data_vio_compression_stage { 95 /* A data_vio which has not yet entered the compression path */ 96 DATA_VIO_PRE_COMPRESSOR, 97 /* A data_vio which is in the compressor */ 98 DATA_VIO_COMPRESSING, 99 /* A data_vio which is blocked in the packer */ 100 DATA_VIO_PACKING, 101 /* A data_vio which is no longer on the compression path (and never will be) */ 102 DATA_VIO_POST_PACKER, 103}; 104 105struct data_vio_compression_status { 106 enum data_vio_compression_stage stage; 107 bool may_not_compress; 108}; 109 110struct compression_state { 111 /* 112 * The current compression status of this data_vio. This field contains a value which 113 * consists of a data_vio_compression_stage and a flag indicating whether a request has 114 * been made to cancel (or prevent) compression for this data_vio. 115 * 116 * This field should be accessed through the get_data_vio_compression_status() and 117 * set_data_vio_compression_status() methods. It should not be accessed directly. 118 */ 119 atomic_t status; 120 121 /* The compressed size of this block */ 122 u16 size; 123 124 /* The packer input or output bin slot which holds the enclosing data_vio */ 125 slot_number_t slot; 126 127 /* The packer bin to which the enclosing data_vio has been assigned */ 128 struct packer_bin *bin; 129 130 /* A link in the chain of data_vios which have been packed together */ 131 struct data_vio *next_in_batch; 132 133 /* A vio which is blocked in the packer while holding a lock this vio needs. */ 134 struct data_vio *lock_holder; 135 136 /* 137 * The compressed block used to hold the compressed form of this block and that of any 138 * other blocks for which this data_vio is the compressed write agent. 139 */ 140 struct compressed_block *block; 141}; 142 143/* Fields supporting allocation of data blocks. */ 144struct allocation { 145 /* The physical zone in which to allocate a physical block */ 146 struct physical_zone *zone; 147 148 /* The block allocated to this vio */ 149 physical_block_number_t pbn; 150 151 /* 152 * If non-NULL, the pooled PBN lock held on the allocated block. Must be a write lock until 153 * the block has been written, after which it will become a read lock. 154 */ 155 struct pbn_lock *lock; 156 157 /* The type of write lock to obtain on the allocated block */ 158 enum pbn_lock_type write_lock_type; 159 160 /* The zone which was the start of the current allocation cycle */ 161 zone_count_t first_allocation_zone; 162 163 /* Whether this vio should wait for a clean slab */ 164 bool wait_for_clean_slab; 165}; 166 167struct reference_updater { 168 enum journal_operation operation; 169 bool increment; 170 struct zoned_pbn zpbn; 171 struct pbn_lock *lock; 172 struct vdo_waiter waiter; 173}; 174 175/* A vio for processing user data requests. */ 176struct data_vio { 177 /* The vdo_wait_queue entry structure */ 178 struct vdo_waiter waiter; 179 180 /* The logical block of this request */ 181 struct lbn_lock logical; 182 183 /* The state for traversing the block map tree */ 184 struct tree_lock tree_lock; 185 186 /* The current partition address of this block */ 187 struct zoned_pbn mapped; 188 189 /* The hash of this vio (if not zero) */ 190 struct uds_record_name record_name; 191 192 /* Used for logging and debugging */ 193 enum async_operation_number last_async_operation; 194 195 /* The operations to record in the recovery and slab journals */ 196 struct reference_updater increment_updater; 197 struct reference_updater decrement_updater; 198 199 u16 read : 1; 200 u16 write : 1; 201 u16 fua : 1; 202 u16 is_zero : 1; 203 u16 is_discard : 1; 204 u16 is_partial : 1; 205 u16 is_duplicate : 1; 206 u16 first_reference_operation_complete : 1; 207 u16 downgrade_allocation_lock : 1; 208 209 struct allocation allocation; 210 211 /* 212 * Whether this vio has received an allocation. This field is examined from threads not in 213 * the allocation zone. 214 */ 215 bool allocation_succeeded; 216 217 /* The new partition address of this block after the vio write completes */ 218 struct zoned_pbn new_mapped; 219 220 /* The hash zone responsible for the name (NULL if is_zero_block) */ 221 struct hash_zone *hash_zone; 222 223 /* The lock this vio holds or shares with other vios with the same data */ 224 struct hash_lock *hash_lock; 225 226 /* All data_vios sharing a hash lock are kept in a list linking these list entries */ 227 struct list_head hash_lock_entry; 228 229 /* The block number in the partition of the UDS deduplication advice */ 230 struct zoned_pbn duplicate; 231 232 /* 233 * The sequence number of the recovery journal block containing the increment entry for 234 * this vio. 235 */ 236 sequence_number_t recovery_sequence_number; 237 238 /* The point in the recovery journal where this write last made an entry */ 239 struct journal_point recovery_journal_point; 240 241 /* The list of vios in user initiated write requests */ 242 struct list_head write_entry; 243 244 /* The generation number of the VDO that this vio belongs to */ 245 sequence_number_t flush_generation; 246 247 /* The completion to use for fetching block map pages for this vio */ 248 struct vdo_page_completion page_completion; 249 250 /* The user bio that initiated this VIO */ 251 struct bio *user_bio; 252 253 /* partial block support */ 254 block_size_t offset; 255 256 /* 257 * The number of bytes to be discarded. For discards, this field will always be positive, 258 * whereas for non-discards it will always be 0. Hence it can be used to determine whether 259 * a data_vio is processing a discard, even after the user_bio has been acknowledged. 260 */ 261 u32 remaining_discard; 262 263 struct dedupe_context *dedupe_context; 264 265 /* Fields beyond this point will not be reset when a pooled data_vio is reused. */ 266 267 struct vio vio; 268 269 /* The completion for making reference count decrements */ 270 struct vdo_completion decrement_completion; 271 272 /* All of the fields necessary for the compression path */ 273 struct compression_state compression; 274 275 /* A block used as output during compression or uncompression */ 276 char *scratch_block; 277 278 struct list_head pool_entry; 279}; 280 281static inline struct data_vio *vio_as_data_vio(struct vio *vio) 282{ 283 VDO_ASSERT_LOG_ONLY((vio->type == VIO_TYPE_DATA), "vio is a data_vio"); 284 return container_of(vio, struct data_vio, vio); 285} 286 287static inline struct data_vio *as_data_vio(struct vdo_completion *completion) 288{ 289 return vio_as_data_vio(as_vio(completion)); 290} 291 292static inline struct data_vio *vdo_waiter_as_data_vio(struct vdo_waiter *waiter) 293{ 294 if (waiter == NULL) 295 return NULL; 296 297 return container_of(waiter, struct data_vio, waiter); 298} 299 300static inline struct data_vio *data_vio_from_reference_updater(struct reference_updater *updater) 301{ 302 if (updater->increment) 303 return container_of(updater, struct data_vio, increment_updater); 304 305 return container_of(updater, struct data_vio, decrement_updater); 306} 307 308static inline bool data_vio_has_flush_generation_lock(struct data_vio *data_vio) 309{ 310 return !list_empty(&data_vio->write_entry); 311} 312 313static inline struct vdo *vdo_from_data_vio(struct data_vio *data_vio) 314{ 315 return data_vio->vio.completion.vdo; 316} 317 318static inline bool data_vio_has_allocation(struct data_vio *data_vio) 319{ 320 return (data_vio->allocation.pbn != VDO_ZERO_BLOCK); 321} 322 323struct data_vio_compression_status __must_check 324advance_data_vio_compression_stage(struct data_vio *data_vio); 325struct data_vio_compression_status __must_check 326get_data_vio_compression_status(struct data_vio *data_vio); 327bool cancel_data_vio_compression(struct data_vio *data_vio); 328 329struct data_vio_pool; 330 331int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, 332 data_vio_count_t discard_limit, struct data_vio_pool **pool_ptr); 333void free_data_vio_pool(struct data_vio_pool *pool); 334void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio); 335void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion); 336void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion); 337 338void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios); 339data_vio_count_t get_data_vio_pool_active_discards(struct data_vio_pool *pool); 340data_vio_count_t get_data_vio_pool_discard_limit(struct data_vio_pool *pool); 341data_vio_count_t get_data_vio_pool_maximum_discards(struct data_vio_pool *pool); 342int __must_check set_data_vio_pool_discard_limit(struct data_vio_pool *pool, 343 data_vio_count_t limit); 344data_vio_count_t get_data_vio_pool_active_requests(struct data_vio_pool *pool); 345data_vio_count_t get_data_vio_pool_request_limit(struct data_vio_pool *pool); 346data_vio_count_t get_data_vio_pool_maximum_requests(struct data_vio_pool *pool); 347 348void complete_data_vio(struct vdo_completion *completion); 349void handle_data_vio_error(struct vdo_completion *completion); 350 351static inline void continue_data_vio(struct data_vio *data_vio) 352{ 353 vdo_launch_completion(&data_vio->vio.completion); 354} 355 356/** 357 * continue_data_vio_with_error() - Set an error code and then continue processing a data_vio. 358 * 359 * This will not mask older errors. This function can be called with a success code, but it is more 360 * efficient to call continue_data_vio() if the caller knows the result was a success. 361 */ 362static inline void continue_data_vio_with_error(struct data_vio *data_vio, int result) 363{ 364 vdo_continue_completion(&data_vio->vio.completion, result); 365} 366 367const char * __must_check get_data_vio_operation_name(struct data_vio *data_vio); 368 369static inline void assert_data_vio_in_hash_zone(struct data_vio *data_vio) 370{ 371 thread_id_t expected = data_vio->hash_zone->thread_id; 372 thread_id_t thread_id = vdo_get_callback_thread_id(); 373 /* 374 * It's odd to use the LBN, but converting the record name to hex is a bit clunky for an 375 * inline, and the LBN better than nothing as an identifier. 376 */ 377 VDO_ASSERT_LOG_ONLY((expected == thread_id), 378 "data_vio for logical block %llu on thread %u, should be on hash zone thread %u", 379 (unsigned long long) data_vio->logical.lbn, thread_id, expected); 380} 381 382static inline void set_data_vio_hash_zone_callback(struct data_vio *data_vio, 383 vdo_action_fn callback) 384{ 385 vdo_set_completion_callback(&data_vio->vio.completion, callback, 386 data_vio->hash_zone->thread_id); 387} 388 389/** 390 * launch_data_vio_hash_zone_callback() - Set a callback as a hash zone operation and invoke it 391 * immediately. 392 */ 393static inline void launch_data_vio_hash_zone_callback(struct data_vio *data_vio, 394 vdo_action_fn callback) 395{ 396 set_data_vio_hash_zone_callback(data_vio, callback); 397 vdo_launch_completion(&data_vio->vio.completion); 398} 399 400static inline void assert_data_vio_in_logical_zone(struct data_vio *data_vio) 401{ 402 thread_id_t expected = data_vio->logical.zone->thread_id; 403 thread_id_t thread_id = vdo_get_callback_thread_id(); 404 405 VDO_ASSERT_LOG_ONLY((expected == thread_id), 406 "data_vio for logical block %llu on thread %u, should be on thread %u", 407 (unsigned long long) data_vio->logical.lbn, thread_id, expected); 408} 409 410static inline void set_data_vio_logical_callback(struct data_vio *data_vio, 411 vdo_action_fn callback) 412{ 413 vdo_set_completion_callback(&data_vio->vio.completion, callback, 414 data_vio->logical.zone->thread_id); 415} 416 417/** 418 * launch_data_vio_logical_callback() - Set a callback as a logical block operation and invoke it 419 * immediately. 420 */ 421static inline void launch_data_vio_logical_callback(struct data_vio *data_vio, 422 vdo_action_fn callback) 423{ 424 set_data_vio_logical_callback(data_vio, callback); 425 vdo_launch_completion(&data_vio->vio.completion); 426} 427 428static inline void assert_data_vio_in_allocated_zone(struct data_vio *data_vio) 429{ 430 thread_id_t expected = data_vio->allocation.zone->thread_id; 431 thread_id_t thread_id = vdo_get_callback_thread_id(); 432 433 VDO_ASSERT_LOG_ONLY((expected == thread_id), 434 "struct data_vio for allocated physical block %llu on thread %u, should be on thread %u", 435 (unsigned long long) data_vio->allocation.pbn, thread_id, 436 expected); 437} 438 439static inline void set_data_vio_allocated_zone_callback(struct data_vio *data_vio, 440 vdo_action_fn callback) 441{ 442 vdo_set_completion_callback(&data_vio->vio.completion, callback, 443 data_vio->allocation.zone->thread_id); 444} 445 446/** 447 * launch_data_vio_allocated_zone_callback() - Set a callback as a physical block operation in a 448 * data_vio's allocated zone and queue the data_vio and 449 * invoke it immediately. 450 */ 451static inline void launch_data_vio_allocated_zone_callback(struct data_vio *data_vio, 452 vdo_action_fn callback) 453{ 454 set_data_vio_allocated_zone_callback(data_vio, callback); 455 vdo_launch_completion(&data_vio->vio.completion); 456} 457 458static inline void assert_data_vio_in_duplicate_zone(struct data_vio *data_vio) 459{ 460 thread_id_t expected = data_vio->duplicate.zone->thread_id; 461 thread_id_t thread_id = vdo_get_callback_thread_id(); 462 463 VDO_ASSERT_LOG_ONLY((expected == thread_id), 464 "data_vio for duplicate physical block %llu on thread %u, should be on thread %u", 465 (unsigned long long) data_vio->duplicate.pbn, thread_id, 466 expected); 467} 468 469static inline void set_data_vio_duplicate_zone_callback(struct data_vio *data_vio, 470 vdo_action_fn callback) 471{ 472 vdo_set_completion_callback(&data_vio->vio.completion, callback, 473 data_vio->duplicate.zone->thread_id); 474} 475 476/** 477 * launch_data_vio_duplicate_zone_callback() - Set a callback as a physical block operation in a 478 * data_vio's duplicate zone and queue the data_vio and 479 * invoke it immediately. 480 */ 481static inline void launch_data_vio_duplicate_zone_callback(struct data_vio *data_vio, 482 vdo_action_fn callback) 483{ 484 set_data_vio_duplicate_zone_callback(data_vio, callback); 485 vdo_launch_completion(&data_vio->vio.completion); 486} 487 488static inline void assert_data_vio_in_mapped_zone(struct data_vio *data_vio) 489{ 490 thread_id_t expected = data_vio->mapped.zone->thread_id; 491 thread_id_t thread_id = vdo_get_callback_thread_id(); 492 493 VDO_ASSERT_LOG_ONLY((expected == thread_id), 494 "data_vio for mapped physical block %llu on thread %u, should be on thread %u", 495 (unsigned long long) data_vio->mapped.pbn, thread_id, expected); 496} 497 498static inline void set_data_vio_mapped_zone_callback(struct data_vio *data_vio, 499 vdo_action_fn callback) 500{ 501 vdo_set_completion_callback(&data_vio->vio.completion, callback, 502 data_vio->mapped.zone->thread_id); 503} 504 505static inline void assert_data_vio_in_new_mapped_zone(struct data_vio *data_vio) 506{ 507 thread_id_t expected = data_vio->new_mapped.zone->thread_id; 508 thread_id_t thread_id = vdo_get_callback_thread_id(); 509 510 VDO_ASSERT_LOG_ONLY((expected == thread_id), 511 "data_vio for new_mapped physical block %llu on thread %u, should be on thread %u", 512 (unsigned long long) data_vio->new_mapped.pbn, thread_id, 513 expected); 514} 515 516static inline void set_data_vio_new_mapped_zone_callback(struct data_vio *data_vio, 517 vdo_action_fn callback) 518{ 519 vdo_set_completion_callback(&data_vio->vio.completion, callback, 520 data_vio->new_mapped.zone->thread_id); 521} 522 523static inline void assert_data_vio_in_journal_zone(struct data_vio *data_vio) 524{ 525 thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread; 526 thread_id_t thread_id = vdo_get_callback_thread_id(); 527 528 VDO_ASSERT_LOG_ONLY((journal_thread == thread_id), 529 "data_vio for logical block %llu on thread %u, should be on journal thread %u", 530 (unsigned long long) data_vio->logical.lbn, thread_id, 531 journal_thread); 532} 533 534static inline void set_data_vio_journal_callback(struct data_vio *data_vio, 535 vdo_action_fn callback) 536{ 537 thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread; 538 539 vdo_set_completion_callback(&data_vio->vio.completion, callback, journal_thread); 540} 541 542/** 543 * launch_data_vio_journal_callback() - Set a callback as a journal operation and invoke it 544 * immediately. 545 */ 546static inline void launch_data_vio_journal_callback(struct data_vio *data_vio, 547 vdo_action_fn callback) 548{ 549 set_data_vio_journal_callback(data_vio, callback); 550 vdo_launch_completion(&data_vio->vio.completion); 551} 552 553static inline void assert_data_vio_in_packer_zone(struct data_vio *data_vio) 554{ 555 thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread; 556 thread_id_t thread_id = vdo_get_callback_thread_id(); 557 558 VDO_ASSERT_LOG_ONLY((packer_thread == thread_id), 559 "data_vio for logical block %llu on thread %u, should be on packer thread %u", 560 (unsigned long long) data_vio->logical.lbn, thread_id, 561 packer_thread); 562} 563 564static inline void set_data_vio_packer_callback(struct data_vio *data_vio, 565 vdo_action_fn callback) 566{ 567 thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread; 568 569 vdo_set_completion_callback(&data_vio->vio.completion, callback, packer_thread); 570} 571 572/** 573 * launch_data_vio_packer_callback() - Set a callback as a packer operation and invoke it 574 * immediately. 575 */ 576static inline void launch_data_vio_packer_callback(struct data_vio *data_vio, 577 vdo_action_fn callback) 578{ 579 set_data_vio_packer_callback(data_vio, callback); 580 vdo_launch_completion(&data_vio->vio.completion); 581} 582 583static inline void assert_data_vio_on_cpu_thread(struct data_vio *data_vio) 584{ 585 thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread; 586 thread_id_t thread_id = vdo_get_callback_thread_id(); 587 588 VDO_ASSERT_LOG_ONLY((cpu_thread == thread_id), 589 "data_vio for logical block %llu on thread %u, should be on cpu thread %u", 590 (unsigned long long) data_vio->logical.lbn, thread_id, 591 cpu_thread); 592} 593 594static inline void set_data_vio_cpu_callback(struct data_vio *data_vio, 595 vdo_action_fn callback) 596{ 597 thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread; 598 599 vdo_set_completion_callback(&data_vio->vio.completion, callback, cpu_thread); 600} 601 602/** 603 * launch_data_vio_cpu_callback() - Set a callback to run on the CPU queues and invoke it 604 * immediately. 605 */ 606static inline void launch_data_vio_cpu_callback(struct data_vio *data_vio, 607 vdo_action_fn callback, 608 enum vdo_completion_priority priority) 609{ 610 set_data_vio_cpu_callback(data_vio, callback); 611 vdo_launch_completion_with_priority(&data_vio->vio.completion, priority); 612} 613 614static inline void set_data_vio_bio_zone_callback(struct data_vio *data_vio, 615 vdo_action_fn callback) 616{ 617 vdo_set_completion_callback(&data_vio->vio.completion, callback, 618 get_vio_bio_zone_thread_id(&data_vio->vio)); 619} 620 621/** 622 * launch_data_vio_bio_zone_callback() - Set a callback as a bio zone operation and invoke it 623 * immediately. 624 */ 625static inline void launch_data_vio_bio_zone_callback(struct data_vio *data_vio, 626 vdo_action_fn callback) 627{ 628 set_data_vio_bio_zone_callback(data_vio, callback); 629 vdo_launch_completion_with_priority(&data_vio->vio.completion, 630 BIO_Q_DATA_PRIORITY); 631} 632 633/** 634 * launch_data_vio_on_bio_ack_queue() - If the vdo uses a bio_ack queue, set a callback to run on 635 * it and invoke it immediately, otherwise, just run the 636 * callback on the current thread. 637 */ 638static inline void launch_data_vio_on_bio_ack_queue(struct data_vio *data_vio, 639 vdo_action_fn callback) 640{ 641 struct vdo_completion *completion = &data_vio->vio.completion; 642 struct vdo *vdo = completion->vdo; 643 644 if (!vdo_uses_bio_ack_queue(vdo)) { 645 callback(completion); 646 return; 647 } 648 649 vdo_set_completion_callback(completion, callback, 650 vdo->thread_config.bio_ack_thread); 651 vdo_launch_completion_with_priority(completion, BIO_ACK_Q_ACK_PRIORITY); 652} 653 654void data_vio_allocate_data_block(struct data_vio *data_vio, 655 enum pbn_lock_type write_lock_type, 656 vdo_action_fn callback, vdo_action_fn error_handler); 657 658void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset); 659 660int __must_check uncompress_data_vio(struct data_vio *data_vio, 661 enum block_mapping_state mapping_state, 662 char *buffer); 663 664void update_metadata_for_data_vio_write(struct data_vio *data_vio, 665 struct pbn_lock *lock); 666void write_data_vio(struct data_vio *data_vio); 667void launch_compress_data_vio(struct data_vio *data_vio); 668void continue_data_vio_with_block_map_slot(struct vdo_completion *completion); 669 670#endif /* DATA_VIO_H */ 671