1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 24 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 25 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 26 * Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved. 27 * Copyright (c) 2015 by Chunwei Chen. All rights reserved. 28 * Copyright (c) 2019 Datto Inc. 29 * Copyright (c) 2019, Klara Inc. 30 * Copyright (c) 2019, Allan Jude 31 */ 32 33#include <sys/dmu.h> 34#include <sys/dmu_impl.h> 35#include <sys/dmu_tx.h> 36#include <sys/dbuf.h> 37#include <sys/dnode.h> 38#include <sys/zfs_context.h> 39#include <sys/dmu_objset.h> 40#include <sys/dmu_traverse.h> 41#include <sys/dsl_dataset.h> 42#include <sys/dsl_dir.h> 43#include <sys/dsl_pool.h> 44#include <sys/dsl_synctask.h> 45#include <sys/dsl_prop.h> 46#include <sys/dmu_zfetch.h> 47#include <sys/zfs_ioctl.h> 48#include <sys/zap.h> 49#include <sys/zio_checksum.h> 50#include <sys/zio_compress.h> 51#include <sys/sa.h> 52#include <sys/zfeature.h> 53#include <sys/abd.h> 54#include <sys/trace_zfs.h> 55#include <sys/zfs_racct.h> 56#include <sys/zfs_rlock.h> 57#ifdef _KERNEL 58#include <sys/vmsystm.h> 59#include <sys/zfs_znode.h> 60#endif 61 62/* 63 * Enable/disable nopwrite feature. 64 */ 65int zfs_nopwrite_enabled = 1; 66 67/* 68 * Tunable to control percentage of dirtied L1 blocks from frees allowed into 69 * one TXG. After this threshold is crossed, additional dirty blocks from frees 70 * will wait until the next TXG. 71 * A value of zero will disable this throttle. 72 */ 73unsigned long zfs_per_txg_dirty_frees_percent = 5; 74 75/* 76 * Enable/disable forcing txg sync when dirty in dmu_offset_next. 77 */ 78int zfs_dmu_offset_next_sync = 0; 79 80/* 81 * Limit the amount we can prefetch with one call to this amount. This 82 * helps to limit the amount of memory that can be used by prefetching. 83 * Larger objects should be prefetched a bit at a time. 84 */ 85int dmu_prefetch_max = 8 * SPA_MAXBLOCKSIZE; 86 87const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = { 88 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "unallocated" }, 89 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "object directory" }, 90 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "object array" }, 91 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "packed nvlist" }, 92 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "packed nvlist size" }, 93 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj" }, 94 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj header" }, 95 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map header" }, 96 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map" }, 97 {DMU_BSWAP_UINT64, TRUE, FALSE, TRUE, "ZIL intent log" }, 98 {DMU_BSWAP_DNODE, TRUE, FALSE, TRUE, "DMU dnode" }, 99 {DMU_BSWAP_OBJSET, TRUE, TRUE, FALSE, "DMU objset" }, 100 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL directory" }, 101 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL directory child map"}, 102 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset snap map" }, 103 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL props" }, 104 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL dataset" }, 105 {DMU_BSWAP_ZNODE, TRUE, FALSE, FALSE, "ZFS znode" }, 106 {DMU_BSWAP_OLDACL, TRUE, FALSE, TRUE, "ZFS V0 ACL" }, 107 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "ZFS plain file" }, 108 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS directory" }, 109 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "ZFS master node" }, 110 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS delete queue" }, 111 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "zvol object" }, 112 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "zvol prop" }, 113 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "other uint8[]" }, 114 {DMU_BSWAP_UINT64, FALSE, FALSE, TRUE, "other uint64[]" }, 115 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "other ZAP" }, 116 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "persistent error log" }, 117 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "SPA history" }, 118 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA history offsets" }, 119 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "Pool properties" }, 120 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL permissions" }, 121 {DMU_BSWAP_ACL, TRUE, FALSE, TRUE, "ZFS ACL" }, 122 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "ZFS SYSACL" }, 123 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "FUID table" }, 124 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "FUID table size" }, 125 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset next clones"}, 126 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan work queue" }, 127 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project used" }, 128 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project quota"}, 129 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "snapshot refcount tags"}, 130 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT ZAP algorithm" }, 131 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT statistics" }, 132 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "System attributes" }, 133 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA master node" }, 134 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr registration" }, 135 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr layouts" }, 136 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan translations" }, 137 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "deduplicated block" }, 138 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL deadlist map" }, 139 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL deadlist map hdr" }, 140 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dir clones" }, 141 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj subobj" } 142}; 143 144const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = { 145 { byteswap_uint8_array, "uint8" }, 146 { byteswap_uint16_array, "uint16" }, 147 { byteswap_uint32_array, "uint32" }, 148 { byteswap_uint64_array, "uint64" }, 149 { zap_byteswap, "zap" }, 150 { dnode_buf_byteswap, "dnode" }, 151 { dmu_objset_byteswap, "objset" }, 152 { zfs_znode_byteswap, "znode" }, 153 { zfs_oldacl_byteswap, "oldacl" }, 154 { zfs_acl_byteswap, "acl" } 155}; 156 157static int 158dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset, 159 void *tag, dmu_buf_t **dbp) 160{ 161 uint64_t blkid; 162 dmu_buf_impl_t *db; 163 164 rw_enter(&dn->dn_struct_rwlock, RW_READER); 165 blkid = dbuf_whichblock(dn, 0, offset); 166 db = dbuf_hold(dn, blkid, tag); 167 rw_exit(&dn->dn_struct_rwlock); 168 169 if (db == NULL) { 170 *dbp = NULL; 171 return (SET_ERROR(EIO)); 172 } 173 174 *dbp = &db->db; 175 return (0); 176} 177int 178dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset, 179 void *tag, dmu_buf_t **dbp) 180{ 181 dnode_t *dn; 182 uint64_t blkid; 183 dmu_buf_impl_t *db; 184 int err; 185 186 err = dnode_hold(os, object, FTAG, &dn); 187 if (err) 188 return (err); 189 rw_enter(&dn->dn_struct_rwlock, RW_READER); 190 blkid = dbuf_whichblock(dn, 0, offset); 191 db = dbuf_hold(dn, blkid, tag); 192 rw_exit(&dn->dn_struct_rwlock); 193 dnode_rele(dn, FTAG); 194 195 if (db == NULL) { 196 *dbp = NULL; 197 return (SET_ERROR(EIO)); 198 } 199 200 *dbp = &db->db; 201 return (err); 202} 203 204int 205dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset, 206 void *tag, dmu_buf_t **dbp, int flags) 207{ 208 int err; 209 int db_flags = DB_RF_CANFAIL; 210 211 if (flags & DMU_READ_NO_PREFETCH) 212 db_flags |= DB_RF_NOPREFETCH; 213 if (flags & DMU_READ_NO_DECRYPT) 214 db_flags |= DB_RF_NO_DECRYPT; 215 216 err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp); 217 if (err == 0) { 218 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp); 219 err = dbuf_read(db, NULL, db_flags); 220 if (err != 0) { 221 dbuf_rele(db, tag); 222 *dbp = NULL; 223 } 224 } 225 226 return (err); 227} 228 229int 230dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset, 231 void *tag, dmu_buf_t **dbp, int flags) 232{ 233 int err; 234 int db_flags = DB_RF_CANFAIL; 235 236 if (flags & DMU_READ_NO_PREFETCH) 237 db_flags |= DB_RF_NOPREFETCH; 238 if (flags & DMU_READ_NO_DECRYPT) 239 db_flags |= DB_RF_NO_DECRYPT; 240 241 err = dmu_buf_hold_noread(os, object, offset, tag, dbp); 242 if (err == 0) { 243 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp); 244 err = dbuf_read(db, NULL, db_flags); 245 if (err != 0) { 246 dbuf_rele(db, tag); 247 *dbp = NULL; 248 } 249 } 250 251 return (err); 252} 253 254int 255dmu_bonus_max(void) 256{ 257 return (DN_OLD_MAX_BONUSLEN); 258} 259 260int 261dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx) 262{ 263 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 264 dnode_t *dn; 265 int error; 266 267 DB_DNODE_ENTER(db); 268 dn = DB_DNODE(db); 269 270 if (dn->dn_bonus != db) { 271 error = SET_ERROR(EINVAL); 272 } else if (newsize < 0 || newsize > db_fake->db_size) { 273 error = SET_ERROR(EINVAL); 274 } else { 275 dnode_setbonuslen(dn, newsize, tx); 276 error = 0; 277 } 278 279 DB_DNODE_EXIT(db); 280 return (error); 281} 282 283int 284dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx) 285{ 286 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 287 dnode_t *dn; 288 int error; 289 290 DB_DNODE_ENTER(db); 291 dn = DB_DNODE(db); 292 293 if (!DMU_OT_IS_VALID(type)) { 294 error = SET_ERROR(EINVAL); 295 } else if (dn->dn_bonus != db) { 296 error = SET_ERROR(EINVAL); 297 } else { 298 dnode_setbonus_type(dn, type, tx); 299 error = 0; 300 } 301 302 DB_DNODE_EXIT(db); 303 return (error); 304} 305 306dmu_object_type_t 307dmu_get_bonustype(dmu_buf_t *db_fake) 308{ 309 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 310 dnode_t *dn; 311 dmu_object_type_t type; 312 313 DB_DNODE_ENTER(db); 314 dn = DB_DNODE(db); 315 type = dn->dn_bonustype; 316 DB_DNODE_EXIT(db); 317 318 return (type); 319} 320 321int 322dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx) 323{ 324 dnode_t *dn; 325 int error; 326 327 error = dnode_hold(os, object, FTAG, &dn); 328 dbuf_rm_spill(dn, tx); 329 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 330 dnode_rm_spill(dn, tx); 331 rw_exit(&dn->dn_struct_rwlock); 332 dnode_rele(dn, FTAG); 333 return (error); 334} 335 336/* 337 * Lookup and hold the bonus buffer for the provided dnode. If the dnode 338 * has not yet been allocated a new bonus dbuf a will be allocated. 339 * Returns ENOENT, EIO, or 0. 340 */ 341int dmu_bonus_hold_by_dnode(dnode_t *dn, void *tag, dmu_buf_t **dbp, 342 uint32_t flags) 343{ 344 dmu_buf_impl_t *db; 345 int error; 346 uint32_t db_flags = DB_RF_MUST_SUCCEED; 347 348 if (flags & DMU_READ_NO_PREFETCH) 349 db_flags |= DB_RF_NOPREFETCH; 350 if (flags & DMU_READ_NO_DECRYPT) 351 db_flags |= DB_RF_NO_DECRYPT; 352 353 rw_enter(&dn->dn_struct_rwlock, RW_READER); 354 if (dn->dn_bonus == NULL) { 355 rw_exit(&dn->dn_struct_rwlock); 356 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 357 if (dn->dn_bonus == NULL) 358 dbuf_create_bonus(dn); 359 } 360 db = dn->dn_bonus; 361 362 /* as long as the bonus buf is held, the dnode will be held */ 363 if (zfs_refcount_add(&db->db_holds, tag) == 1) { 364 VERIFY(dnode_add_ref(dn, db)); 365 atomic_inc_32(&dn->dn_dbufs_count); 366 } 367 368 /* 369 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's 370 * hold and incrementing the dbuf count to ensure that dnode_move() sees 371 * a dnode hold for every dbuf. 372 */ 373 rw_exit(&dn->dn_struct_rwlock); 374 375 error = dbuf_read(db, NULL, db_flags); 376 if (error) { 377 dnode_evict_bonus(dn); 378 dbuf_rele(db, tag); 379 *dbp = NULL; 380 return (error); 381 } 382 383 *dbp = &db->db; 384 return (0); 385} 386 387int 388dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp) 389{ 390 dnode_t *dn; 391 int error; 392 393 error = dnode_hold(os, object, FTAG, &dn); 394 if (error) 395 return (error); 396 397 error = dmu_bonus_hold_by_dnode(dn, tag, dbp, DMU_READ_NO_PREFETCH); 398 dnode_rele(dn, FTAG); 399 400 return (error); 401} 402 403/* 404 * returns ENOENT, EIO, or 0. 405 * 406 * This interface will allocate a blank spill dbuf when a spill blk 407 * doesn't already exist on the dnode. 408 * 409 * if you only want to find an already existing spill db, then 410 * dmu_spill_hold_existing() should be used. 411 */ 412int 413dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp) 414{ 415 dmu_buf_impl_t *db = NULL; 416 int err; 417 418 if ((flags & DB_RF_HAVESTRUCT) == 0) 419 rw_enter(&dn->dn_struct_rwlock, RW_READER); 420 421 db = dbuf_hold(dn, DMU_SPILL_BLKID, tag); 422 423 if ((flags & DB_RF_HAVESTRUCT) == 0) 424 rw_exit(&dn->dn_struct_rwlock); 425 426 if (db == NULL) { 427 *dbp = NULL; 428 return (SET_ERROR(EIO)); 429 } 430 err = dbuf_read(db, NULL, flags); 431 if (err == 0) 432 *dbp = &db->db; 433 else { 434 dbuf_rele(db, tag); 435 *dbp = NULL; 436 } 437 return (err); 438} 439 440int 441dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp) 442{ 443 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus; 444 dnode_t *dn; 445 int err; 446 447 DB_DNODE_ENTER(db); 448 dn = DB_DNODE(db); 449 450 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) { 451 err = SET_ERROR(EINVAL); 452 } else { 453 rw_enter(&dn->dn_struct_rwlock, RW_READER); 454 455 if (!dn->dn_have_spill) { 456 err = SET_ERROR(ENOENT); 457 } else { 458 err = dmu_spill_hold_by_dnode(dn, 459 DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp); 460 } 461 462 rw_exit(&dn->dn_struct_rwlock); 463 } 464 465 DB_DNODE_EXIT(db); 466 return (err); 467} 468 469int 470dmu_spill_hold_by_bonus(dmu_buf_t *bonus, uint32_t flags, void *tag, 471 dmu_buf_t **dbp) 472{ 473 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus; 474 dnode_t *dn; 475 int err; 476 uint32_t db_flags = DB_RF_CANFAIL; 477 478 if (flags & DMU_READ_NO_DECRYPT) 479 db_flags |= DB_RF_NO_DECRYPT; 480 481 DB_DNODE_ENTER(db); 482 dn = DB_DNODE(db); 483 err = dmu_spill_hold_by_dnode(dn, db_flags, tag, dbp); 484 DB_DNODE_EXIT(db); 485 486 return (err); 487} 488 489/* 490 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces 491 * to take a held dnode rather than <os, object> -- the lookup is wasteful, 492 * and can induce severe lock contention when writing to several files 493 * whose dnodes are in the same block. 494 */ 495int 496dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length, 497 boolean_t read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags) 498{ 499 dmu_buf_t **dbp; 500 zstream_t *zs = NULL; 501 uint64_t blkid, nblks, i; 502 uint32_t dbuf_flags; 503 int err; 504 zio_t *zio = NULL; 505 boolean_t missed = B_FALSE; 506 507 ASSERT(length <= DMU_MAX_ACCESS); 508 509 /* 510 * Note: We directly notify the prefetch code of this read, so that 511 * we can tell it about the multi-block read. dbuf_read() only knows 512 * about the one block it is accessing. 513 */ 514 dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT | 515 DB_RF_NOPREFETCH; 516 517 rw_enter(&dn->dn_struct_rwlock, RW_READER); 518 if (dn->dn_datablkshift) { 519 int blkshift = dn->dn_datablkshift; 520 nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) - 521 P2ALIGN(offset, 1ULL << blkshift)) >> blkshift; 522 } else { 523 if (offset + length > dn->dn_datablksz) { 524 zfs_panic_recover("zfs: accessing past end of object " 525 "%llx/%llx (size=%u access=%llu+%llu)", 526 (longlong_t)dn->dn_objset-> 527 os_dsl_dataset->ds_object, 528 (longlong_t)dn->dn_object, dn->dn_datablksz, 529 (longlong_t)offset, (longlong_t)length); 530 rw_exit(&dn->dn_struct_rwlock); 531 return (SET_ERROR(EIO)); 532 } 533 nblks = 1; 534 } 535 dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP); 536 537 if (read) 538 zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, 539 ZIO_FLAG_CANFAIL); 540 blkid = dbuf_whichblock(dn, 0, offset); 541 if ((flags & DMU_READ_NO_PREFETCH) == 0 && 542 DNODE_META_IS_CACHEABLE(dn) && length <= zfetch_array_rd_sz) { 543 /* 544 * Prepare the zfetch before initiating the demand reads, so 545 * that if multiple threads block on same indirect block, we 546 * base predictions on the original less racy request order. 547 */ 548 zs = dmu_zfetch_prepare(&dn->dn_zfetch, blkid, nblks, 549 read && DNODE_IS_CACHEABLE(dn), B_TRUE); 550 } 551 for (i = 0; i < nblks; i++) { 552 dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag); 553 if (db == NULL) { 554 if (zs) 555 dmu_zfetch_run(zs, missed, B_TRUE); 556 rw_exit(&dn->dn_struct_rwlock); 557 dmu_buf_rele_array(dbp, nblks, tag); 558 if (read) 559 zio_nowait(zio); 560 return (SET_ERROR(EIO)); 561 } 562 563 /* 564 * Initiate async demand data read. 565 * We check the db_state after calling dbuf_read() because 566 * (1) dbuf_read() may change the state to CACHED due to a 567 * hit in the ARC, and (2) on a cache miss, a child will 568 * have been added to "zio" but not yet completed, so the 569 * state will not yet be CACHED. 570 */ 571 if (read) { 572 (void) dbuf_read(db, zio, dbuf_flags); 573 if (db->db_state != DB_CACHED) 574 missed = B_TRUE; 575 } 576 dbp[i] = &db->db; 577 } 578 579 if (!read) 580 zfs_racct_write(length, nblks); 581 582 if (zs) 583 dmu_zfetch_run(zs, missed, B_TRUE); 584 rw_exit(&dn->dn_struct_rwlock); 585 586 if (read) { 587 /* wait for async read i/o */ 588 err = zio_wait(zio); 589 if (err) { 590 dmu_buf_rele_array(dbp, nblks, tag); 591 return (err); 592 } 593 594 /* wait for other io to complete */ 595 for (i = 0; i < nblks; i++) { 596 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i]; 597 mutex_enter(&db->db_mtx); 598 while (db->db_state == DB_READ || 599 db->db_state == DB_FILL) 600 cv_wait(&db->db_changed, &db->db_mtx); 601 if (db->db_state == DB_UNCACHED) 602 err = SET_ERROR(EIO); 603 mutex_exit(&db->db_mtx); 604 if (err) { 605 dmu_buf_rele_array(dbp, nblks, tag); 606 return (err); 607 } 608 } 609 } 610 611 *numbufsp = nblks; 612 *dbpp = dbp; 613 return (0); 614} 615 616static int 617dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset, 618 uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp) 619{ 620 dnode_t *dn; 621 int err; 622 623 err = dnode_hold(os, object, FTAG, &dn); 624 if (err) 625 return (err); 626 627 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag, 628 numbufsp, dbpp, DMU_READ_PREFETCH); 629 630 dnode_rele(dn, FTAG); 631 632 return (err); 633} 634 635int 636dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset, 637 uint64_t length, boolean_t read, void *tag, int *numbufsp, 638 dmu_buf_t ***dbpp) 639{ 640 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 641 dnode_t *dn; 642 int err; 643 644 DB_DNODE_ENTER(db); 645 dn = DB_DNODE(db); 646 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag, 647 numbufsp, dbpp, DMU_READ_PREFETCH); 648 DB_DNODE_EXIT(db); 649 650 return (err); 651} 652 653void 654dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag) 655{ 656 int i; 657 dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake; 658 659 if (numbufs == 0) 660 return; 661 662 for (i = 0; i < numbufs; i++) { 663 if (dbp[i]) 664 dbuf_rele(dbp[i], tag); 665 } 666 667 kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs); 668} 669 670/* 671 * Issue prefetch i/os for the given blocks. If level is greater than 0, the 672 * indirect blocks prefetched will be those that point to the blocks containing 673 * the data starting at offset, and continuing to offset + len. 674 * 675 * Note that if the indirect blocks above the blocks being prefetched are not 676 * in cache, they will be asynchronously read in. 677 */ 678void 679dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset, 680 uint64_t len, zio_priority_t pri) 681{ 682 dnode_t *dn; 683 uint64_t blkid; 684 int nblks, err; 685 686 if (len == 0) { /* they're interested in the bonus buffer */ 687 dn = DMU_META_DNODE(os); 688 689 if (object == 0 || object >= DN_MAX_OBJECT) 690 return; 691 692 rw_enter(&dn->dn_struct_rwlock, RW_READER); 693 blkid = dbuf_whichblock(dn, level, 694 object * sizeof (dnode_phys_t)); 695 dbuf_prefetch(dn, level, blkid, pri, 0); 696 rw_exit(&dn->dn_struct_rwlock); 697 return; 698 } 699 700 /* 701 * See comment before the definition of dmu_prefetch_max. 702 */ 703 len = MIN(len, dmu_prefetch_max); 704 705 /* 706 * XXX - Note, if the dnode for the requested object is not 707 * already cached, we will do a *synchronous* read in the 708 * dnode_hold() call. The same is true for any indirects. 709 */ 710 err = dnode_hold(os, object, FTAG, &dn); 711 if (err != 0) 712 return; 713 714 /* 715 * offset + len - 1 is the last byte we want to prefetch for, and offset 716 * is the first. Then dbuf_whichblk(dn, level, off + len - 1) is the 717 * last block we want to prefetch, and dbuf_whichblock(dn, level, 718 * offset) is the first. Then the number we need to prefetch is the 719 * last - first + 1. 720 */ 721 rw_enter(&dn->dn_struct_rwlock, RW_READER); 722 if (level > 0 || dn->dn_datablkshift != 0) { 723 nblks = dbuf_whichblock(dn, level, offset + len - 1) - 724 dbuf_whichblock(dn, level, offset) + 1; 725 } else { 726 nblks = (offset < dn->dn_datablksz); 727 } 728 729 if (nblks != 0) { 730 blkid = dbuf_whichblock(dn, level, offset); 731 for (int i = 0; i < nblks; i++) 732 dbuf_prefetch(dn, level, blkid + i, pri, 0); 733 } 734 rw_exit(&dn->dn_struct_rwlock); 735 736 dnode_rele(dn, FTAG); 737} 738 739/* 740 * Get the next "chunk" of file data to free. We traverse the file from 741 * the end so that the file gets shorter over time (if we crashes in the 742 * middle, this will leave us in a better state). We find allocated file 743 * data by simply searching the allocated level 1 indirects. 744 * 745 * On input, *start should be the first offset that does not need to be 746 * freed (e.g. "offset + length"). On return, *start will be the first 747 * offset that should be freed and l1blks is set to the number of level 1 748 * indirect blocks found within the chunk. 749 */ 750static int 751get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum, uint64_t *l1blks) 752{ 753 uint64_t blks; 754 uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1); 755 /* bytes of data covered by a level-1 indirect block */ 756 uint64_t iblkrange = (uint64_t)dn->dn_datablksz * 757 EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT); 758 759 ASSERT3U(minimum, <=, *start); 760 761 /* 762 * Check if we can free the entire range assuming that all of the 763 * L1 blocks in this range have data. If we can, we use this 764 * worst case value as an estimate so we can avoid having to look 765 * at the object's actual data. 766 */ 767 uint64_t total_l1blks = 768 (roundup(*start, iblkrange) - (minimum / iblkrange * iblkrange)) / 769 iblkrange; 770 if (total_l1blks <= maxblks) { 771 *l1blks = total_l1blks; 772 *start = minimum; 773 return (0); 774 } 775 ASSERT(ISP2(iblkrange)); 776 777 for (blks = 0; *start > minimum && blks < maxblks; blks++) { 778 int err; 779 780 /* 781 * dnode_next_offset(BACKWARDS) will find an allocated L1 782 * indirect block at or before the input offset. We must 783 * decrement *start so that it is at the end of the region 784 * to search. 785 */ 786 (*start)--; 787 788 err = dnode_next_offset(dn, 789 DNODE_FIND_BACKWARDS, start, 2, 1, 0); 790 791 /* if there are no indirect blocks before start, we are done */ 792 if (err == ESRCH) { 793 *start = minimum; 794 break; 795 } else if (err != 0) { 796 *l1blks = blks; 797 return (err); 798 } 799 800 /* set start to the beginning of this L1 indirect */ 801 *start = P2ALIGN(*start, iblkrange); 802 } 803 if (*start < minimum) 804 *start = minimum; 805 *l1blks = blks; 806 807 return (0); 808} 809 810/* 811 * If this objset is of type OST_ZFS return true if vfs's unmounted flag is set, 812 * otherwise return false. 813 * Used below in dmu_free_long_range_impl() to enable abort when unmounting 814 */ 815/*ARGSUSED*/ 816static boolean_t 817dmu_objset_zfs_unmounting(objset_t *os) 818{ 819#ifdef _KERNEL 820 if (dmu_objset_type(os) == DMU_OST_ZFS) 821 return (zfs_get_vfs_flag_unmounted(os)); 822#endif 823 return (B_FALSE); 824} 825 826static int 827dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset, 828 uint64_t length) 829{ 830 uint64_t object_size; 831 int err; 832 uint64_t dirty_frees_threshold; 833 dsl_pool_t *dp = dmu_objset_pool(os); 834 835 if (dn == NULL) 836 return (SET_ERROR(EINVAL)); 837 838 object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz; 839 if (offset >= object_size) 840 return (0); 841 842 if (zfs_per_txg_dirty_frees_percent <= 100) 843 dirty_frees_threshold = 844 zfs_per_txg_dirty_frees_percent * zfs_dirty_data_max / 100; 845 else 846 dirty_frees_threshold = zfs_dirty_data_max / 20; 847 848 if (length == DMU_OBJECT_END || offset + length > object_size) 849 length = object_size - offset; 850 851 while (length != 0) { 852 uint64_t chunk_end, chunk_begin, chunk_len; 853 uint64_t l1blks; 854 dmu_tx_t *tx; 855 856 if (dmu_objset_zfs_unmounting(dn->dn_objset)) 857 return (SET_ERROR(EINTR)); 858 859 chunk_end = chunk_begin = offset + length; 860 861 /* move chunk_begin backwards to the beginning of this chunk */ 862 err = get_next_chunk(dn, &chunk_begin, offset, &l1blks); 863 if (err) 864 return (err); 865 ASSERT3U(chunk_begin, >=, offset); 866 ASSERT3U(chunk_begin, <=, chunk_end); 867 868 chunk_len = chunk_end - chunk_begin; 869 870 tx = dmu_tx_create(os); 871 dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len); 872 873 /* 874 * Mark this transaction as typically resulting in a net 875 * reduction in space used. 876 */ 877 dmu_tx_mark_netfree(tx); 878 err = dmu_tx_assign(tx, TXG_WAIT); 879 if (err) { 880 dmu_tx_abort(tx); 881 return (err); 882 } 883 884 uint64_t txg = dmu_tx_get_txg(tx); 885 886 mutex_enter(&dp->dp_lock); 887 uint64_t long_free_dirty = 888 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK]; 889 mutex_exit(&dp->dp_lock); 890 891 /* 892 * To avoid filling up a TXG with just frees, wait for 893 * the next TXG to open before freeing more chunks if 894 * we have reached the threshold of frees. 895 */ 896 if (dirty_frees_threshold != 0 && 897 long_free_dirty >= dirty_frees_threshold) { 898 DMU_TX_STAT_BUMP(dmu_tx_dirty_frees_delay); 899 dmu_tx_commit(tx); 900 txg_wait_open(dp, 0, B_TRUE); 901 continue; 902 } 903 904 /* 905 * In order to prevent unnecessary write throttling, for each 906 * TXG, we track the cumulative size of L1 blocks being dirtied 907 * in dnode_free_range() below. We compare this number to a 908 * tunable threshold, past which we prevent new L1 dirty freeing 909 * blocks from being added into the open TXG. See 910 * dmu_free_long_range_impl() for details. The threshold 911 * prevents write throttle activation due to dirty freeing L1 912 * blocks taking up a large percentage of zfs_dirty_data_max. 913 */ 914 mutex_enter(&dp->dp_lock); 915 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] += 916 l1blks << dn->dn_indblkshift; 917 mutex_exit(&dp->dp_lock); 918 DTRACE_PROBE3(free__long__range, 919 uint64_t, long_free_dirty, uint64_t, chunk_len, 920 uint64_t, txg); 921 dnode_free_range(dn, chunk_begin, chunk_len, tx); 922 923 dmu_tx_commit(tx); 924 925 length -= chunk_len; 926 } 927 return (0); 928} 929 930int 931dmu_free_long_range(objset_t *os, uint64_t object, 932 uint64_t offset, uint64_t length) 933{ 934 dnode_t *dn; 935 int err; 936 937 err = dnode_hold(os, object, FTAG, &dn); 938 if (err != 0) 939 return (err); 940 err = dmu_free_long_range_impl(os, dn, offset, length); 941 942 /* 943 * It is important to zero out the maxblkid when freeing the entire 944 * file, so that (a) subsequent calls to dmu_free_long_range_impl() 945 * will take the fast path, and (b) dnode_reallocate() can verify 946 * that the entire file has been freed. 947 */ 948 if (err == 0 && offset == 0 && length == DMU_OBJECT_END) 949 dn->dn_maxblkid = 0; 950 951 dnode_rele(dn, FTAG); 952 return (err); 953} 954 955int 956dmu_free_long_object(objset_t *os, uint64_t object) 957{ 958 dmu_tx_t *tx; 959 int err; 960 961 err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END); 962 if (err != 0) 963 return (err); 964 965 tx = dmu_tx_create(os); 966 dmu_tx_hold_bonus(tx, object); 967 dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END); 968 dmu_tx_mark_netfree(tx); 969 err = dmu_tx_assign(tx, TXG_WAIT); 970 if (err == 0) { 971 if (err == 0) 972 err = dmu_object_free(os, object, tx); 973 974 dmu_tx_commit(tx); 975 } else { 976 dmu_tx_abort(tx); 977 } 978 979 return (err); 980} 981 982int 983dmu_free_range(objset_t *os, uint64_t object, uint64_t offset, 984 uint64_t size, dmu_tx_t *tx) 985{ 986 dnode_t *dn; 987 int err = dnode_hold(os, object, FTAG, &dn); 988 if (err) 989 return (err); 990 ASSERT(offset < UINT64_MAX); 991 ASSERT(size == DMU_OBJECT_END || size <= UINT64_MAX - offset); 992 dnode_free_range(dn, offset, size, tx); 993 dnode_rele(dn, FTAG); 994 return (0); 995} 996 997static int 998dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size, 999 void *buf, uint32_t flags) 1000{ 1001 dmu_buf_t **dbp; 1002 int numbufs, err = 0; 1003 1004 /* 1005 * Deal with odd block sizes, where there can't be data past the first 1006 * block. If we ever do the tail block optimization, we will need to 1007 * handle that here as well. 1008 */ 1009 if (dn->dn_maxblkid == 0) { 1010 uint64_t newsz = offset > dn->dn_datablksz ? 0 : 1011 MIN(size, dn->dn_datablksz - offset); 1012 bzero((char *)buf + newsz, size - newsz); 1013 size = newsz; 1014 } 1015 1016 while (size > 0) { 1017 uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2); 1018 int i; 1019 1020 /* 1021 * NB: we could do this block-at-a-time, but it's nice 1022 * to be reading in parallel. 1023 */ 1024 err = dmu_buf_hold_array_by_dnode(dn, offset, mylen, 1025 TRUE, FTAG, &numbufs, &dbp, flags); 1026 if (err) 1027 break; 1028 1029 for (i = 0; i < numbufs; i++) { 1030 uint64_t tocpy; 1031 int64_t bufoff; 1032 dmu_buf_t *db = dbp[i]; 1033 1034 ASSERT(size > 0); 1035 1036 bufoff = offset - db->db_offset; 1037 tocpy = MIN(db->db_size - bufoff, size); 1038 1039 (void) memcpy(buf, (char *)db->db_data + bufoff, tocpy); 1040 1041 offset += tocpy; 1042 size -= tocpy; 1043 buf = (char *)buf + tocpy; 1044 } 1045 dmu_buf_rele_array(dbp, numbufs, FTAG); 1046 } 1047 return (err); 1048} 1049 1050int 1051dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1052 void *buf, uint32_t flags) 1053{ 1054 dnode_t *dn; 1055 int err; 1056 1057 err = dnode_hold(os, object, FTAG, &dn); 1058 if (err != 0) 1059 return (err); 1060 1061 err = dmu_read_impl(dn, offset, size, buf, flags); 1062 dnode_rele(dn, FTAG); 1063 return (err); 1064} 1065 1066int 1067dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf, 1068 uint32_t flags) 1069{ 1070 return (dmu_read_impl(dn, offset, size, buf, flags)); 1071} 1072 1073static void 1074dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size, 1075 const void *buf, dmu_tx_t *tx) 1076{ 1077 int i; 1078 1079 for (i = 0; i < numbufs; i++) { 1080 uint64_t tocpy; 1081 int64_t bufoff; 1082 dmu_buf_t *db = dbp[i]; 1083 1084 ASSERT(size > 0); 1085 1086 bufoff = offset - db->db_offset; 1087 tocpy = MIN(db->db_size - bufoff, size); 1088 1089 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 1090 1091 if (tocpy == db->db_size) 1092 dmu_buf_will_fill(db, tx); 1093 else 1094 dmu_buf_will_dirty(db, tx); 1095 1096 (void) memcpy((char *)db->db_data + bufoff, buf, tocpy); 1097 1098 if (tocpy == db->db_size) 1099 dmu_buf_fill_done(db, tx); 1100 1101 offset += tocpy; 1102 size -= tocpy; 1103 buf = (char *)buf + tocpy; 1104 } 1105} 1106 1107void 1108dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1109 const void *buf, dmu_tx_t *tx) 1110{ 1111 dmu_buf_t **dbp; 1112 int numbufs; 1113 1114 if (size == 0) 1115 return; 1116 1117 VERIFY0(dmu_buf_hold_array(os, object, offset, size, 1118 FALSE, FTAG, &numbufs, &dbp)); 1119 dmu_write_impl(dbp, numbufs, offset, size, buf, tx); 1120 dmu_buf_rele_array(dbp, numbufs, FTAG); 1121} 1122 1123/* 1124 * Note: Lustre is an external consumer of this interface. 1125 */ 1126void 1127dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, 1128 const void *buf, dmu_tx_t *tx) 1129{ 1130 dmu_buf_t **dbp; 1131 int numbufs; 1132 1133 if (size == 0) 1134 return; 1135 1136 VERIFY0(dmu_buf_hold_array_by_dnode(dn, offset, size, 1137 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH)); 1138 dmu_write_impl(dbp, numbufs, offset, size, buf, tx); 1139 dmu_buf_rele_array(dbp, numbufs, FTAG); 1140} 1141 1142void 1143dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1144 dmu_tx_t *tx) 1145{ 1146 dmu_buf_t **dbp; 1147 int numbufs, i; 1148 1149 if (size == 0) 1150 return; 1151 1152 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size, 1153 FALSE, FTAG, &numbufs, &dbp)); 1154 1155 for (i = 0; i < numbufs; i++) { 1156 dmu_buf_t *db = dbp[i]; 1157 1158 dmu_buf_will_not_fill(db, tx); 1159 } 1160 dmu_buf_rele_array(dbp, numbufs, FTAG); 1161} 1162 1163void 1164dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset, 1165 void *data, uint8_t etype, uint8_t comp, int uncompressed_size, 1166 int compressed_size, int byteorder, dmu_tx_t *tx) 1167{ 1168 dmu_buf_t *db; 1169 1170 ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES); 1171 ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS); 1172 VERIFY0(dmu_buf_hold_noread(os, object, offset, 1173 FTAG, &db)); 1174 1175 dmu_buf_write_embedded(db, 1176 data, (bp_embedded_type_t)etype, (enum zio_compress)comp, 1177 uncompressed_size, compressed_size, byteorder, tx); 1178 1179 dmu_buf_rele(db, FTAG); 1180} 1181 1182void 1183dmu_redact(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1184 dmu_tx_t *tx) 1185{ 1186 int numbufs, i; 1187 dmu_buf_t **dbp; 1188 1189 VERIFY0(dmu_buf_hold_array(os, object, offset, size, FALSE, FTAG, 1190 &numbufs, &dbp)); 1191 for (i = 0; i < numbufs; i++) 1192 dmu_buf_redact(dbp[i], tx); 1193 dmu_buf_rele_array(dbp, numbufs, FTAG); 1194} 1195 1196#ifdef _KERNEL 1197int 1198dmu_read_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size) 1199{ 1200 dmu_buf_t **dbp; 1201 int numbufs, i, err; 1202 1203 /* 1204 * NB: we could do this block-at-a-time, but it's nice 1205 * to be reading in parallel. 1206 */ 1207 err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), size, 1208 TRUE, FTAG, &numbufs, &dbp, 0); 1209 if (err) 1210 return (err); 1211 1212 for (i = 0; i < numbufs; i++) { 1213 uint64_t tocpy; 1214 int64_t bufoff; 1215 dmu_buf_t *db = dbp[i]; 1216 1217 ASSERT(size > 0); 1218 1219 bufoff = zfs_uio_offset(uio) - db->db_offset; 1220 tocpy = MIN(db->db_size - bufoff, size); 1221 1222 err = zfs_uio_fault_move((char *)db->db_data + bufoff, tocpy, 1223 UIO_READ, uio); 1224 1225 if (err) 1226 break; 1227 1228 size -= tocpy; 1229 } 1230 dmu_buf_rele_array(dbp, numbufs, FTAG); 1231 1232 return (err); 1233} 1234 1235/* 1236 * Read 'size' bytes into the uio buffer. 1237 * From object zdb->db_object. 1238 * Starting at zfs_uio_offset(uio). 1239 * 1240 * If the caller already has a dbuf in the target object 1241 * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(), 1242 * because we don't have to find the dnode_t for the object. 1243 */ 1244int 1245dmu_read_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size) 1246{ 1247 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb; 1248 dnode_t *dn; 1249 int err; 1250 1251 if (size == 0) 1252 return (0); 1253 1254 DB_DNODE_ENTER(db); 1255 dn = DB_DNODE(db); 1256 err = dmu_read_uio_dnode(dn, uio, size); 1257 DB_DNODE_EXIT(db); 1258 1259 return (err); 1260} 1261 1262/* 1263 * Read 'size' bytes into the uio buffer. 1264 * From the specified object 1265 * Starting at offset zfs_uio_offset(uio). 1266 */ 1267int 1268dmu_read_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size) 1269{ 1270 dnode_t *dn; 1271 int err; 1272 1273 if (size == 0) 1274 return (0); 1275 1276 err = dnode_hold(os, object, FTAG, &dn); 1277 if (err) 1278 return (err); 1279 1280 err = dmu_read_uio_dnode(dn, uio, size); 1281 1282 dnode_rele(dn, FTAG); 1283 1284 return (err); 1285} 1286 1287int 1288dmu_write_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size, dmu_tx_t *tx) 1289{ 1290 dmu_buf_t **dbp; 1291 int numbufs; 1292 int err = 0; 1293 int i; 1294 1295 err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), size, 1296 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH); 1297 if (err) 1298 return (err); 1299 1300 for (i = 0; i < numbufs; i++) { 1301 uint64_t tocpy; 1302 int64_t bufoff; 1303 dmu_buf_t *db = dbp[i]; 1304 1305 ASSERT(size > 0); 1306 1307 bufoff = zfs_uio_offset(uio) - db->db_offset; 1308 tocpy = MIN(db->db_size - bufoff, size); 1309 1310 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 1311 1312 if (tocpy == db->db_size) 1313 dmu_buf_will_fill(db, tx); 1314 else 1315 dmu_buf_will_dirty(db, tx); 1316 1317 /* 1318 * XXX zfs_uiomove could block forever (eg.nfs-backed 1319 * pages). There needs to be a uiolockdown() function 1320 * to lock the pages in memory, so that zfs_uiomove won't 1321 * block. 1322 */ 1323 err = zfs_uio_fault_move((char *)db->db_data + bufoff, 1324 tocpy, UIO_WRITE, uio); 1325 1326 if (tocpy == db->db_size) 1327 dmu_buf_fill_done(db, tx); 1328 1329 if (err) 1330 break; 1331 1332 size -= tocpy; 1333 } 1334 1335 dmu_buf_rele_array(dbp, numbufs, FTAG); 1336 return (err); 1337} 1338 1339/* 1340 * Write 'size' bytes from the uio buffer. 1341 * To object zdb->db_object. 1342 * Starting at offset zfs_uio_offset(uio). 1343 * 1344 * If the caller already has a dbuf in the target object 1345 * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(), 1346 * because we don't have to find the dnode_t for the object. 1347 */ 1348int 1349dmu_write_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size, 1350 dmu_tx_t *tx) 1351{ 1352 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb; 1353 dnode_t *dn; 1354 int err; 1355 1356 if (size == 0) 1357 return (0); 1358 1359 DB_DNODE_ENTER(db); 1360 dn = DB_DNODE(db); 1361 err = dmu_write_uio_dnode(dn, uio, size, tx); 1362 DB_DNODE_EXIT(db); 1363 1364 return (err); 1365} 1366 1367/* 1368 * Write 'size' bytes from the uio buffer. 1369 * To the specified object. 1370 * Starting at offset zfs_uio_offset(uio). 1371 */ 1372int 1373dmu_write_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size, 1374 dmu_tx_t *tx) 1375{ 1376 dnode_t *dn; 1377 int err; 1378 1379 if (size == 0) 1380 return (0); 1381 1382 err = dnode_hold(os, object, FTAG, &dn); 1383 if (err) 1384 return (err); 1385 1386 err = dmu_write_uio_dnode(dn, uio, size, tx); 1387 1388 dnode_rele(dn, FTAG); 1389 1390 return (err); 1391} 1392#endif /* _KERNEL */ 1393 1394/* 1395 * Allocate a loaned anonymous arc buffer. 1396 */ 1397arc_buf_t * 1398dmu_request_arcbuf(dmu_buf_t *handle, int size) 1399{ 1400 dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle; 1401 1402 return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size)); 1403} 1404 1405/* 1406 * Free a loaned arc buffer. 1407 */ 1408void 1409dmu_return_arcbuf(arc_buf_t *buf) 1410{ 1411 arc_return_buf(buf, FTAG); 1412 arc_buf_destroy(buf, FTAG); 1413} 1414 1415/* 1416 * A "lightweight" write is faster than a regular write (e.g. 1417 * dmu_write_by_dnode() or dmu_assign_arcbuf_by_dnode()), because it avoids the 1418 * CPU cost of creating a dmu_buf_impl_t and arc_buf_[hdr_]_t. However, the 1419 * data can not be read or overwritten until the transaction's txg has been 1420 * synced. This makes it appropriate for workloads that are known to be 1421 * (temporarily) write-only, like "zfs receive". 1422 * 1423 * A single block is written, starting at the specified offset in bytes. If 1424 * the call is successful, it returns 0 and the provided abd has been 1425 * consumed (the caller should not free it). 1426 */ 1427int 1428dmu_lightweight_write_by_dnode(dnode_t *dn, uint64_t offset, abd_t *abd, 1429 const zio_prop_t *zp, enum zio_flag flags, dmu_tx_t *tx) 1430{ 1431 dbuf_dirty_record_t *dr = 1432 dbuf_dirty_lightweight(dn, dbuf_whichblock(dn, 0, offset), tx); 1433 if (dr == NULL) 1434 return (SET_ERROR(EIO)); 1435 dr->dt.dll.dr_abd = abd; 1436 dr->dt.dll.dr_props = *zp; 1437 dr->dt.dll.dr_flags = flags; 1438 return (0); 1439} 1440 1441/* 1442 * When possible directly assign passed loaned arc buffer to a dbuf. 1443 * If this is not possible copy the contents of passed arc buf via 1444 * dmu_write(). 1445 */ 1446int 1447dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf, 1448 dmu_tx_t *tx) 1449{ 1450 dmu_buf_impl_t *db; 1451 objset_t *os = dn->dn_objset; 1452 uint64_t object = dn->dn_object; 1453 uint32_t blksz = (uint32_t)arc_buf_lsize(buf); 1454 uint64_t blkid; 1455 1456 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1457 blkid = dbuf_whichblock(dn, 0, offset); 1458 db = dbuf_hold(dn, blkid, FTAG); 1459 if (db == NULL) 1460 return (SET_ERROR(EIO)); 1461 rw_exit(&dn->dn_struct_rwlock); 1462 1463 /* 1464 * We can only assign if the offset is aligned and the arc buf is the 1465 * same size as the dbuf. 1466 */ 1467 if (offset == db->db.db_offset && blksz == db->db.db_size) { 1468 zfs_racct_write(blksz, 1); 1469 dbuf_assign_arcbuf(db, buf, tx); 1470 dbuf_rele(db, FTAG); 1471 } else { 1472 /* compressed bufs must always be assignable to their dbuf */ 1473 ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF); 1474 ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED)); 1475 1476 dbuf_rele(db, FTAG); 1477 dmu_write(os, object, offset, blksz, buf->b_data, tx); 1478 dmu_return_arcbuf(buf); 1479 } 1480 1481 return (0); 1482} 1483 1484int 1485dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf, 1486 dmu_tx_t *tx) 1487{ 1488 int err; 1489 dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle; 1490 1491 DB_DNODE_ENTER(dbuf); 1492 err = dmu_assign_arcbuf_by_dnode(DB_DNODE(dbuf), offset, buf, tx); 1493 DB_DNODE_EXIT(dbuf); 1494 1495 return (err); 1496} 1497 1498typedef struct { 1499 dbuf_dirty_record_t *dsa_dr; 1500 dmu_sync_cb_t *dsa_done; 1501 zgd_t *dsa_zgd; 1502 dmu_tx_t *dsa_tx; 1503} dmu_sync_arg_t; 1504 1505/* ARGSUSED */ 1506static void 1507dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg) 1508{ 1509 dmu_sync_arg_t *dsa = varg; 1510 dmu_buf_t *db = dsa->dsa_zgd->zgd_db; 1511 blkptr_t *bp = zio->io_bp; 1512 1513 if (zio->io_error == 0) { 1514 if (BP_IS_HOLE(bp)) { 1515 /* 1516 * A block of zeros may compress to a hole, but the 1517 * block size still needs to be known for replay. 1518 */ 1519 BP_SET_LSIZE(bp, db->db_size); 1520 } else if (!BP_IS_EMBEDDED(bp)) { 1521 ASSERT(BP_GET_LEVEL(bp) == 0); 1522 BP_SET_FILL(bp, 1); 1523 } 1524 } 1525} 1526 1527static void 1528dmu_sync_late_arrival_ready(zio_t *zio) 1529{ 1530 dmu_sync_ready(zio, NULL, zio->io_private); 1531} 1532 1533/* ARGSUSED */ 1534static void 1535dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg) 1536{ 1537 dmu_sync_arg_t *dsa = varg; 1538 dbuf_dirty_record_t *dr = dsa->dsa_dr; 1539 dmu_buf_impl_t *db = dr->dr_dbuf; 1540 zgd_t *zgd = dsa->dsa_zgd; 1541 1542 /* 1543 * Record the vdev(s) backing this blkptr so they can be flushed after 1544 * the writes for the lwb have completed. 1545 */ 1546 if (zio->io_error == 0) { 1547 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp); 1548 } 1549 1550 mutex_enter(&db->db_mtx); 1551 ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC); 1552 if (zio->io_error == 0) { 1553 dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE); 1554 if (dr->dt.dl.dr_nopwrite) { 1555 blkptr_t *bp = zio->io_bp; 1556 blkptr_t *bp_orig = &zio->io_bp_orig; 1557 uint8_t chksum = BP_GET_CHECKSUM(bp_orig); 1558 1559 ASSERT(BP_EQUAL(bp, bp_orig)); 1560 VERIFY(BP_EQUAL(bp, db->db_blkptr)); 1561 ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF); 1562 VERIFY(zio_checksum_table[chksum].ci_flags & 1563 ZCHECKSUM_FLAG_NOPWRITE); 1564 } 1565 dr->dt.dl.dr_overridden_by = *zio->io_bp; 1566 dr->dt.dl.dr_override_state = DR_OVERRIDDEN; 1567 dr->dt.dl.dr_copies = zio->io_prop.zp_copies; 1568 1569 /* 1570 * Old style holes are filled with all zeros, whereas 1571 * new-style holes maintain their lsize, type, level, 1572 * and birth time (see zio_write_compress). While we 1573 * need to reset the BP_SET_LSIZE() call that happened 1574 * in dmu_sync_ready for old style holes, we do *not* 1575 * want to wipe out the information contained in new 1576 * style holes. Thus, only zero out the block pointer if 1577 * it's an old style hole. 1578 */ 1579 if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) && 1580 dr->dt.dl.dr_overridden_by.blk_birth == 0) 1581 BP_ZERO(&dr->dt.dl.dr_overridden_by); 1582 } else { 1583 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1584 } 1585 cv_broadcast(&db->db_changed); 1586 mutex_exit(&db->db_mtx); 1587 1588 dsa->dsa_done(dsa->dsa_zgd, zio->io_error); 1589 1590 kmem_free(dsa, sizeof (*dsa)); 1591} 1592 1593static void 1594dmu_sync_late_arrival_done(zio_t *zio) 1595{ 1596 blkptr_t *bp = zio->io_bp; 1597 dmu_sync_arg_t *dsa = zio->io_private; 1598 zgd_t *zgd = dsa->dsa_zgd; 1599 1600 if (zio->io_error == 0) { 1601 /* 1602 * Record the vdev(s) backing this blkptr so they can be 1603 * flushed after the writes for the lwb have completed. 1604 */ 1605 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp); 1606 1607 if (!BP_IS_HOLE(bp)) { 1608 blkptr_t *bp_orig __maybe_unused = &zio->io_bp_orig; 1609 ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE)); 1610 ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig)); 1611 ASSERT(zio->io_bp->blk_birth == zio->io_txg); 1612 ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa)); 1613 zio_free(zio->io_spa, zio->io_txg, zio->io_bp); 1614 } 1615 } 1616 1617 dmu_tx_commit(dsa->dsa_tx); 1618 1619 dsa->dsa_done(dsa->dsa_zgd, zio->io_error); 1620 1621 abd_free(zio->io_abd); 1622 kmem_free(dsa, sizeof (*dsa)); 1623} 1624 1625static int 1626dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd, 1627 zio_prop_t *zp, zbookmark_phys_t *zb) 1628{ 1629 dmu_sync_arg_t *dsa; 1630 dmu_tx_t *tx; 1631 1632 tx = dmu_tx_create(os); 1633 dmu_tx_hold_space(tx, zgd->zgd_db->db_size); 1634 if (dmu_tx_assign(tx, TXG_WAIT) != 0) { 1635 dmu_tx_abort(tx); 1636 /* Make zl_get_data do txg_waited_synced() */ 1637 return (SET_ERROR(EIO)); 1638 } 1639 1640 /* 1641 * In order to prevent the zgd's lwb from being free'd prior to 1642 * dmu_sync_late_arrival_done() being called, we have to ensure 1643 * the lwb's "max txg" takes this tx's txg into account. 1644 */ 1645 zil_lwb_add_txg(zgd->zgd_lwb, dmu_tx_get_txg(tx)); 1646 1647 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); 1648 dsa->dsa_dr = NULL; 1649 dsa->dsa_done = done; 1650 dsa->dsa_zgd = zgd; 1651 dsa->dsa_tx = tx; 1652 1653 /* 1654 * Since we are currently syncing this txg, it's nontrivial to 1655 * determine what BP to nopwrite against, so we disable nopwrite. 1656 * 1657 * When syncing, the db_blkptr is initially the BP of the previous 1658 * txg. We can not nopwrite against it because it will be changed 1659 * (this is similar to the non-late-arrival case where the dbuf is 1660 * dirty in a future txg). 1661 * 1662 * Then dbuf_write_ready() sets bp_blkptr to the location we will write. 1663 * We can not nopwrite against it because although the BP will not 1664 * (typically) be changed, the data has not yet been persisted to this 1665 * location. 1666 * 1667 * Finally, when dbuf_write_done() is called, it is theoretically 1668 * possible to always nopwrite, because the data that was written in 1669 * this txg is the same data that we are trying to write. However we 1670 * would need to check that this dbuf is not dirty in any future 1671 * txg's (as we do in the normal dmu_sync() path). For simplicity, we 1672 * don't nopwrite in this case. 1673 */ 1674 zp->zp_nopwrite = B_FALSE; 1675 1676 zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp, 1677 abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size), 1678 zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp, 1679 dmu_sync_late_arrival_ready, NULL, NULL, dmu_sync_late_arrival_done, 1680 dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb)); 1681 1682 return (0); 1683} 1684 1685/* 1686 * Intent log support: sync the block associated with db to disk. 1687 * N.B. and XXX: the caller is responsible for making sure that the 1688 * data isn't changing while dmu_sync() is writing it. 1689 * 1690 * Return values: 1691 * 1692 * EEXIST: this txg has already been synced, so there's nothing to do. 1693 * The caller should not log the write. 1694 * 1695 * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do. 1696 * The caller should not log the write. 1697 * 1698 * EALREADY: this block is already in the process of being synced. 1699 * The caller should track its progress (somehow). 1700 * 1701 * EIO: could not do the I/O. 1702 * The caller should do a txg_wait_synced(). 1703 * 1704 * 0: the I/O has been initiated. 1705 * The caller should log this blkptr in the done callback. 1706 * It is possible that the I/O will fail, in which case 1707 * the error will be reported to the done callback and 1708 * propagated to pio from zio_done(). 1709 */ 1710int 1711dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd) 1712{ 1713 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db; 1714 objset_t *os = db->db_objset; 1715 dsl_dataset_t *ds = os->os_dsl_dataset; 1716 dbuf_dirty_record_t *dr, *dr_next; 1717 dmu_sync_arg_t *dsa; 1718 zbookmark_phys_t zb; 1719 zio_prop_t zp; 1720 dnode_t *dn; 1721 1722 ASSERT(pio != NULL); 1723 ASSERT(txg != 0); 1724 1725 SET_BOOKMARK(&zb, ds->ds_object, 1726 db->db.db_object, db->db_level, db->db_blkid); 1727 1728 DB_DNODE_ENTER(db); 1729 dn = DB_DNODE(db); 1730 dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp); 1731 DB_DNODE_EXIT(db); 1732 1733 /* 1734 * If we're frozen (running ziltest), we always need to generate a bp. 1735 */ 1736 if (txg > spa_freeze_txg(os->os_spa)) 1737 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb)); 1738 1739 /* 1740 * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf() 1741 * and us. If we determine that this txg is not yet syncing, 1742 * but it begins to sync a moment later, that's OK because the 1743 * sync thread will block in dbuf_sync_leaf() until we drop db_mtx. 1744 */ 1745 mutex_enter(&db->db_mtx); 1746 1747 if (txg <= spa_last_synced_txg(os->os_spa)) { 1748 /* 1749 * This txg has already synced. There's nothing to do. 1750 */ 1751 mutex_exit(&db->db_mtx); 1752 return (SET_ERROR(EEXIST)); 1753 } 1754 1755 if (txg <= spa_syncing_txg(os->os_spa)) { 1756 /* 1757 * This txg is currently syncing, so we can't mess with 1758 * the dirty record anymore; just write a new log block. 1759 */ 1760 mutex_exit(&db->db_mtx); 1761 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb)); 1762 } 1763 1764 dr = dbuf_find_dirty_eq(db, txg); 1765 1766 if (dr == NULL) { 1767 /* 1768 * There's no dr for this dbuf, so it must have been freed. 1769 * There's no need to log writes to freed blocks, so we're done. 1770 */ 1771 mutex_exit(&db->db_mtx); 1772 return (SET_ERROR(ENOENT)); 1773 } 1774 1775 dr_next = list_next(&db->db_dirty_records, dr); 1776 ASSERT(dr_next == NULL || dr_next->dr_txg < txg); 1777 1778 if (db->db_blkptr != NULL) { 1779 /* 1780 * We need to fill in zgd_bp with the current blkptr so that 1781 * the nopwrite code can check if we're writing the same 1782 * data that's already on disk. We can only nopwrite if we 1783 * are sure that after making the copy, db_blkptr will not 1784 * change until our i/o completes. We ensure this by 1785 * holding the db_mtx, and only allowing nopwrite if the 1786 * block is not already dirty (see below). This is verified 1787 * by dmu_sync_done(), which VERIFYs that the db_blkptr has 1788 * not changed. 1789 */ 1790 *zgd->zgd_bp = *db->db_blkptr; 1791 } 1792 1793 /* 1794 * Assume the on-disk data is X, the current syncing data (in 1795 * txg - 1) is Y, and the current in-memory data is Z (currently 1796 * in dmu_sync). 1797 * 1798 * We usually want to perform a nopwrite if X and Z are the 1799 * same. However, if Y is different (i.e. the BP is going to 1800 * change before this write takes effect), then a nopwrite will 1801 * be incorrect - we would override with X, which could have 1802 * been freed when Y was written. 1803 * 1804 * (Note that this is not a concern when we are nop-writing from 1805 * syncing context, because X and Y must be identical, because 1806 * all previous txgs have been synced.) 1807 * 1808 * Therefore, we disable nopwrite if the current BP could change 1809 * before this TXG. There are two ways it could change: by 1810 * being dirty (dr_next is non-NULL), or by being freed 1811 * (dnode_block_freed()). This behavior is verified by 1812 * zio_done(), which VERIFYs that the override BP is identical 1813 * to the on-disk BP. 1814 */ 1815 DB_DNODE_ENTER(db); 1816 dn = DB_DNODE(db); 1817 if (dr_next != NULL || dnode_block_freed(dn, db->db_blkid)) 1818 zp.zp_nopwrite = B_FALSE; 1819 DB_DNODE_EXIT(db); 1820 1821 ASSERT(dr->dr_txg == txg); 1822 if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC || 1823 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 1824 /* 1825 * We have already issued a sync write for this buffer, 1826 * or this buffer has already been synced. It could not 1827 * have been dirtied since, or we would have cleared the state. 1828 */ 1829 mutex_exit(&db->db_mtx); 1830 return (SET_ERROR(EALREADY)); 1831 } 1832 1833 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 1834 dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC; 1835 mutex_exit(&db->db_mtx); 1836 1837 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); 1838 dsa->dsa_dr = dr; 1839 dsa->dsa_done = done; 1840 dsa->dsa_zgd = zgd; 1841 dsa->dsa_tx = NULL; 1842 1843 zio_nowait(arc_write(pio, os->os_spa, txg, 1844 zgd->zgd_bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db), 1845 &zp, dmu_sync_ready, NULL, NULL, dmu_sync_done, dsa, 1846 ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb)); 1847 1848 return (0); 1849} 1850 1851int 1852dmu_object_set_nlevels(objset_t *os, uint64_t object, int nlevels, dmu_tx_t *tx) 1853{ 1854 dnode_t *dn; 1855 int err; 1856 1857 err = dnode_hold(os, object, FTAG, &dn); 1858 if (err) 1859 return (err); 1860 err = dnode_set_nlevels(dn, nlevels, tx); 1861 dnode_rele(dn, FTAG); 1862 return (err); 1863} 1864 1865int 1866dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs, 1867 dmu_tx_t *tx) 1868{ 1869 dnode_t *dn; 1870 int err; 1871 1872 err = dnode_hold(os, object, FTAG, &dn); 1873 if (err) 1874 return (err); 1875 err = dnode_set_blksz(dn, size, ibs, tx); 1876 dnode_rele(dn, FTAG); 1877 return (err); 1878} 1879 1880int 1881dmu_object_set_maxblkid(objset_t *os, uint64_t object, uint64_t maxblkid, 1882 dmu_tx_t *tx) 1883{ 1884 dnode_t *dn; 1885 int err; 1886 1887 err = dnode_hold(os, object, FTAG, &dn); 1888 if (err) 1889 return (err); 1890 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1891 dnode_new_blkid(dn, maxblkid, tx, B_FALSE, B_TRUE); 1892 rw_exit(&dn->dn_struct_rwlock); 1893 dnode_rele(dn, FTAG); 1894 return (0); 1895} 1896 1897void 1898dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum, 1899 dmu_tx_t *tx) 1900{ 1901 dnode_t *dn; 1902 1903 /* 1904 * Send streams include each object's checksum function. This 1905 * check ensures that the receiving system can understand the 1906 * checksum function transmitted. 1907 */ 1908 ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS); 1909 1910 VERIFY0(dnode_hold(os, object, FTAG, &dn)); 1911 ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS); 1912 dn->dn_checksum = checksum; 1913 dnode_setdirty(dn, tx); 1914 dnode_rele(dn, FTAG); 1915} 1916 1917void 1918dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress, 1919 dmu_tx_t *tx) 1920{ 1921 dnode_t *dn; 1922 1923 /* 1924 * Send streams include each object's compression function. This 1925 * check ensures that the receiving system can understand the 1926 * compression function transmitted. 1927 */ 1928 ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS); 1929 1930 VERIFY0(dnode_hold(os, object, FTAG, &dn)); 1931 dn->dn_compress = compress; 1932 dnode_setdirty(dn, tx); 1933 dnode_rele(dn, FTAG); 1934} 1935 1936/* 1937 * When the "redundant_metadata" property is set to "most", only indirect 1938 * blocks of this level and higher will have an additional ditto block. 1939 */ 1940int zfs_redundant_metadata_most_ditto_level = 2; 1941 1942void 1943dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp) 1944{ 1945 dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET; 1946 boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) || 1947 (wp & WP_SPILL)); 1948 enum zio_checksum checksum = os->os_checksum; 1949 enum zio_compress compress = os->os_compress; 1950 uint8_t complevel = os->os_complevel; 1951 enum zio_checksum dedup_checksum = os->os_dedup_checksum; 1952 boolean_t dedup = B_FALSE; 1953 boolean_t nopwrite = B_FALSE; 1954 boolean_t dedup_verify = os->os_dedup_verify; 1955 boolean_t encrypt = B_FALSE; 1956 int copies = os->os_copies; 1957 1958 /* 1959 * We maintain different write policies for each of the following 1960 * types of data: 1961 * 1. metadata 1962 * 2. preallocated blocks (i.e. level-0 blocks of a dump device) 1963 * 3. all other level 0 blocks 1964 */ 1965 if (ismd) { 1966 /* 1967 * XXX -- we should design a compression algorithm 1968 * that specializes in arrays of bps. 1969 */ 1970 compress = zio_compress_select(os->os_spa, 1971 ZIO_COMPRESS_ON, ZIO_COMPRESS_ON); 1972 1973 /* 1974 * Metadata always gets checksummed. If the data 1975 * checksum is multi-bit correctable, and it's not a 1976 * ZBT-style checksum, then it's suitable for metadata 1977 * as well. Otherwise, the metadata checksum defaults 1978 * to fletcher4. 1979 */ 1980 if (!(zio_checksum_table[checksum].ci_flags & 1981 ZCHECKSUM_FLAG_METADATA) || 1982 (zio_checksum_table[checksum].ci_flags & 1983 ZCHECKSUM_FLAG_EMBEDDED)) 1984 checksum = ZIO_CHECKSUM_FLETCHER_4; 1985 1986 if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL || 1987 (os->os_redundant_metadata == 1988 ZFS_REDUNDANT_METADATA_MOST && 1989 (level >= zfs_redundant_metadata_most_ditto_level || 1990 DMU_OT_IS_METADATA(type) || (wp & WP_SPILL)))) 1991 copies++; 1992 } else if (wp & WP_NOFILL) { 1993 ASSERT(level == 0); 1994 1995 /* 1996 * If we're writing preallocated blocks, we aren't actually 1997 * writing them so don't set any policy properties. These 1998 * blocks are currently only used by an external subsystem 1999 * outside of zfs (i.e. dump) and not written by the zio 2000 * pipeline. 2001 */ 2002 compress = ZIO_COMPRESS_OFF; 2003 checksum = ZIO_CHECKSUM_OFF; 2004 } else { 2005 compress = zio_compress_select(os->os_spa, dn->dn_compress, 2006 compress); 2007 complevel = zio_complevel_select(os->os_spa, compress, 2008 complevel, complevel); 2009 2010 checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ? 2011 zio_checksum_select(dn->dn_checksum, checksum) : 2012 dedup_checksum; 2013 2014 /* 2015 * Determine dedup setting. If we are in dmu_sync(), 2016 * we won't actually dedup now because that's all 2017 * done in syncing context; but we do want to use the 2018 * dedup checksum. If the checksum is not strong 2019 * enough to ensure unique signatures, force 2020 * dedup_verify. 2021 */ 2022 if (dedup_checksum != ZIO_CHECKSUM_OFF) { 2023 dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE; 2024 if (!(zio_checksum_table[checksum].ci_flags & 2025 ZCHECKSUM_FLAG_DEDUP)) 2026 dedup_verify = B_TRUE; 2027 } 2028 2029 /* 2030 * Enable nopwrite if we have secure enough checksum 2031 * algorithm (see comment in zio_nop_write) and 2032 * compression is enabled. We don't enable nopwrite if 2033 * dedup is enabled as the two features are mutually 2034 * exclusive. 2035 */ 2036 nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags & 2037 ZCHECKSUM_FLAG_NOPWRITE) && 2038 compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled); 2039 } 2040 2041 /* 2042 * All objects in an encrypted objset are protected from modification 2043 * via a MAC. Encrypted objects store their IV and salt in the last DVA 2044 * in the bp, so we cannot use all copies. Encrypted objects are also 2045 * not subject to nopwrite since writing the same data will still 2046 * result in a new ciphertext. Only encrypted blocks can be dedup'd 2047 * to avoid ambiguity in the dedup code since the DDT does not store 2048 * object types. 2049 */ 2050 if (os->os_encrypted && (wp & WP_NOFILL) == 0) { 2051 encrypt = B_TRUE; 2052 2053 if (DMU_OT_IS_ENCRYPTED(type)) { 2054 copies = MIN(copies, SPA_DVAS_PER_BP - 1); 2055 nopwrite = B_FALSE; 2056 } else { 2057 dedup = B_FALSE; 2058 } 2059 2060 if (level <= 0 && 2061 (type == DMU_OT_DNODE || type == DMU_OT_OBJSET)) { 2062 compress = ZIO_COMPRESS_EMPTY; 2063 } 2064 } 2065 2066 zp->zp_compress = compress; 2067 zp->zp_complevel = complevel; 2068 zp->zp_checksum = checksum; 2069 zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type; 2070 zp->zp_level = level; 2071 zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa)); 2072 zp->zp_dedup = dedup; 2073 zp->zp_dedup_verify = dedup && dedup_verify; 2074 zp->zp_nopwrite = nopwrite; 2075 zp->zp_encrypt = encrypt; 2076 zp->zp_byteorder = ZFS_HOST_BYTEORDER; 2077 bzero(zp->zp_salt, ZIO_DATA_SALT_LEN); 2078 bzero(zp->zp_iv, ZIO_DATA_IV_LEN); 2079 bzero(zp->zp_mac, ZIO_DATA_MAC_LEN); 2080 zp->zp_zpl_smallblk = DMU_OT_IS_FILE(zp->zp_type) ? 2081 os->os_zpl_special_smallblock : 0; 2082 2083 ASSERT3U(zp->zp_compress, !=, ZIO_COMPRESS_INHERIT); 2084} 2085 2086/* 2087 * This function is only called from zfs_holey_common() for zpl_llseek() 2088 * in order to determine the location of holes. In order to accurately 2089 * report holes all dirty data must be synced to disk. This causes extremely 2090 * poor performance when seeking for holes in a dirty file. As a compromise, 2091 * only provide hole data when the dnode is clean. When a dnode is dirty 2092 * report the dnode as having no holes which is always a safe thing to do. 2093 */ 2094int 2095dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off) 2096{ 2097 dnode_t *dn; 2098 int i, err; 2099 boolean_t clean = B_TRUE; 2100 2101 err = dnode_hold(os, object, FTAG, &dn); 2102 if (err) 2103 return (err); 2104 2105 /* 2106 * Check if dnode is dirty 2107 */ 2108 for (i = 0; i < TXG_SIZE; i++) { 2109 if (multilist_link_active(&dn->dn_dirty_link[i])) { 2110 clean = B_FALSE; 2111 break; 2112 } 2113 } 2114 2115 /* 2116 * If compatibility option is on, sync any current changes before 2117 * we go trundling through the block pointers. 2118 */ 2119 if (!clean && zfs_dmu_offset_next_sync) { 2120 clean = B_TRUE; 2121 dnode_rele(dn, FTAG); 2122 txg_wait_synced(dmu_objset_pool(os), 0); 2123 err = dnode_hold(os, object, FTAG, &dn); 2124 if (err) 2125 return (err); 2126 } 2127 2128 if (clean) 2129 err = dnode_next_offset(dn, 2130 (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0); 2131 else 2132 err = SET_ERROR(EBUSY); 2133 2134 dnode_rele(dn, FTAG); 2135 2136 return (err); 2137} 2138 2139void 2140__dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi) 2141{ 2142 dnode_phys_t *dnp = dn->dn_phys; 2143 2144 doi->doi_data_block_size = dn->dn_datablksz; 2145 doi->doi_metadata_block_size = dn->dn_indblkshift ? 2146 1ULL << dn->dn_indblkshift : 0; 2147 doi->doi_type = dn->dn_type; 2148 doi->doi_bonus_type = dn->dn_bonustype; 2149 doi->doi_bonus_size = dn->dn_bonuslen; 2150 doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT; 2151 doi->doi_indirection = dn->dn_nlevels; 2152 doi->doi_checksum = dn->dn_checksum; 2153 doi->doi_compress = dn->dn_compress; 2154 doi->doi_nblkptr = dn->dn_nblkptr; 2155 doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9; 2156 doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz; 2157 doi->doi_fill_count = 0; 2158 for (int i = 0; i < dnp->dn_nblkptr; i++) 2159 doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]); 2160} 2161 2162void 2163dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi) 2164{ 2165 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2166 mutex_enter(&dn->dn_mtx); 2167 2168 __dmu_object_info_from_dnode(dn, doi); 2169 2170 mutex_exit(&dn->dn_mtx); 2171 rw_exit(&dn->dn_struct_rwlock); 2172} 2173 2174/* 2175 * Get information on a DMU object. 2176 * If doi is NULL, just indicates whether the object exists. 2177 */ 2178int 2179dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi) 2180{ 2181 dnode_t *dn; 2182 int err = dnode_hold(os, object, FTAG, &dn); 2183 2184 if (err) 2185 return (err); 2186 2187 if (doi != NULL) 2188 dmu_object_info_from_dnode(dn, doi); 2189 2190 dnode_rele(dn, FTAG); 2191 return (0); 2192} 2193 2194/* 2195 * As above, but faster; can be used when you have a held dbuf in hand. 2196 */ 2197void 2198dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi) 2199{ 2200 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2201 2202 DB_DNODE_ENTER(db); 2203 dmu_object_info_from_dnode(DB_DNODE(db), doi); 2204 DB_DNODE_EXIT(db); 2205} 2206 2207/* 2208 * Faster still when you only care about the size. 2209 */ 2210void 2211dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize, 2212 u_longlong_t *nblk512) 2213{ 2214 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2215 dnode_t *dn; 2216 2217 DB_DNODE_ENTER(db); 2218 dn = DB_DNODE(db); 2219 2220 *blksize = dn->dn_datablksz; 2221 /* add in number of slots used for the dnode itself */ 2222 *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >> 2223 SPA_MINBLOCKSHIFT) + dn->dn_num_slots; 2224 DB_DNODE_EXIT(db); 2225} 2226 2227void 2228dmu_object_dnsize_from_db(dmu_buf_t *db_fake, int *dnsize) 2229{ 2230 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2231 dnode_t *dn; 2232 2233 DB_DNODE_ENTER(db); 2234 dn = DB_DNODE(db); 2235 *dnsize = dn->dn_num_slots << DNODE_SHIFT; 2236 DB_DNODE_EXIT(db); 2237} 2238 2239void 2240byteswap_uint64_array(void *vbuf, size_t size) 2241{ 2242 uint64_t *buf = vbuf; 2243 size_t count = size >> 3; 2244 int i; 2245 2246 ASSERT((size & 7) == 0); 2247 2248 for (i = 0; i < count; i++) 2249 buf[i] = BSWAP_64(buf[i]); 2250} 2251 2252void 2253byteswap_uint32_array(void *vbuf, size_t size) 2254{ 2255 uint32_t *buf = vbuf; 2256 size_t count = size >> 2; 2257 int i; 2258 2259 ASSERT((size & 3) == 0); 2260 2261 for (i = 0; i < count; i++) 2262 buf[i] = BSWAP_32(buf[i]); 2263} 2264 2265void 2266byteswap_uint16_array(void *vbuf, size_t size) 2267{ 2268 uint16_t *buf = vbuf; 2269 size_t count = size >> 1; 2270 int i; 2271 2272 ASSERT((size & 1) == 0); 2273 2274 for (i = 0; i < count; i++) 2275 buf[i] = BSWAP_16(buf[i]); 2276} 2277 2278/* ARGSUSED */ 2279void 2280byteswap_uint8_array(void *vbuf, size_t size) 2281{ 2282} 2283 2284void 2285dmu_init(void) 2286{ 2287 abd_init(); 2288 zfs_dbgmsg_init(); 2289 sa_cache_init(); 2290 dmu_objset_init(); 2291 dnode_init(); 2292 zfetch_init(); 2293 dmu_tx_init(); 2294 l2arc_init(); 2295 arc_init(); 2296 dbuf_init(); 2297} 2298 2299void 2300dmu_fini(void) 2301{ 2302 arc_fini(); /* arc depends on l2arc, so arc must go first */ 2303 l2arc_fini(); 2304 dmu_tx_fini(); 2305 zfetch_fini(); 2306 dbuf_fini(); 2307 dnode_fini(); 2308 dmu_objset_fini(); 2309 sa_cache_fini(); 2310 zfs_dbgmsg_fini(); 2311 abd_fini(); 2312} 2313 2314EXPORT_SYMBOL(dmu_bonus_hold); 2315EXPORT_SYMBOL(dmu_bonus_hold_by_dnode); 2316EXPORT_SYMBOL(dmu_buf_hold_array_by_bonus); 2317EXPORT_SYMBOL(dmu_buf_rele_array); 2318EXPORT_SYMBOL(dmu_prefetch); 2319EXPORT_SYMBOL(dmu_free_range); 2320EXPORT_SYMBOL(dmu_free_long_range); 2321EXPORT_SYMBOL(dmu_free_long_object); 2322EXPORT_SYMBOL(dmu_read); 2323EXPORT_SYMBOL(dmu_read_by_dnode); 2324EXPORT_SYMBOL(dmu_write); 2325EXPORT_SYMBOL(dmu_write_by_dnode); 2326EXPORT_SYMBOL(dmu_prealloc); 2327EXPORT_SYMBOL(dmu_object_info); 2328EXPORT_SYMBOL(dmu_object_info_from_dnode); 2329EXPORT_SYMBOL(dmu_object_info_from_db); 2330EXPORT_SYMBOL(dmu_object_size_from_db); 2331EXPORT_SYMBOL(dmu_object_dnsize_from_db); 2332EXPORT_SYMBOL(dmu_object_set_nlevels); 2333EXPORT_SYMBOL(dmu_object_set_blocksize); 2334EXPORT_SYMBOL(dmu_object_set_maxblkid); 2335EXPORT_SYMBOL(dmu_object_set_checksum); 2336EXPORT_SYMBOL(dmu_object_set_compress); 2337EXPORT_SYMBOL(dmu_offset_next); 2338EXPORT_SYMBOL(dmu_write_policy); 2339EXPORT_SYMBOL(dmu_sync); 2340EXPORT_SYMBOL(dmu_request_arcbuf); 2341EXPORT_SYMBOL(dmu_return_arcbuf); 2342EXPORT_SYMBOL(dmu_assign_arcbuf_by_dnode); 2343EXPORT_SYMBOL(dmu_assign_arcbuf_by_dbuf); 2344EXPORT_SYMBOL(dmu_buf_hold); 2345EXPORT_SYMBOL(dmu_ot); 2346 2347/* BEGIN CSTYLED */ 2348ZFS_MODULE_PARAM(zfs, zfs_, nopwrite_enabled, INT, ZMOD_RW, 2349 "Enable NOP writes"); 2350 2351ZFS_MODULE_PARAM(zfs, zfs_, per_txg_dirty_frees_percent, ULONG, ZMOD_RW, 2352 "Percentage of dirtied blocks from frees in one TXG"); 2353 2354ZFS_MODULE_PARAM(zfs, zfs_, dmu_offset_next_sync, INT, ZMOD_RW, 2355 "Enable forcing txg sync to find holes"); 2356 2357ZFS_MODULE_PARAM(zfs, , dmu_prefetch_max, INT, ZMOD_RW, 2358 "Limit one prefetch call to this size"); 2359/* END CSTYLED */ 2360