ddt.c revision 219089
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26#include <sys/zfs_context.h> 27#include <sys/spa.h> 28#include <sys/spa_impl.h> 29#include <sys/zio.h> 30#include <sys/ddt.h> 31#include <sys/zap.h> 32#include <sys/dmu_tx.h> 33#include <sys/arc.h> 34#include <sys/dsl_pool.h> 35#include <sys/zio_checksum.h> 36#include <sys/zio_compress.h> 37#include <sys/dsl_scan.h> 38 39/* 40 * Enable/disable prefetching of dedup-ed blocks which are going to be freed. 41 */ 42int zfs_dedup_prefetch = 1; 43 44SYSCTL_DECL(_vfs_zfs); 45SYSCTL_NODE(_vfs_zfs, OID_AUTO, dedup, CTLFLAG_RW, 0, "ZFS DEDUP"); 46TUNABLE_INT("vfs.zfs.dedup.prefetch", &zfs_dedup_prefetch); 47SYSCTL_INT(_vfs_zfs_dedup, OID_AUTO, prefetch, CTLFLAG_RW, &zfs_dedup_prefetch, 48 0, "Enable/disable prefetching of dedup-ed blocks which are going to be freed"); 49 50static const ddt_ops_t *ddt_ops[DDT_TYPES] = { 51 &ddt_zap_ops, 52}; 53 54static const char *ddt_class_name[DDT_CLASSES] = { 55 "ditto", 56 "duplicate", 57 "unique", 58}; 59 60static void 61ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 62 dmu_tx_t *tx) 63{ 64 spa_t *spa = ddt->ddt_spa; 65 objset_t *os = ddt->ddt_os; 66 uint64_t *objectp = &ddt->ddt_object[type][class]; 67 boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup; 68 char name[DDT_NAMELEN]; 69 70 ddt_object_name(ddt, type, class, name); 71 72 ASSERT(*objectp == 0); 73 VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0); 74 ASSERT(*objectp != 0); 75 76 VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name, 77 sizeof (uint64_t), 1, objectp, tx) == 0); 78 79 VERIFY(zap_add(os, spa->spa_ddt_stat_object, name, 80 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 81 &ddt->ddt_histogram[type][class], tx) == 0); 82} 83 84static void 85ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 86 dmu_tx_t *tx) 87{ 88 spa_t *spa = ddt->ddt_spa; 89 objset_t *os = ddt->ddt_os; 90 uint64_t *objectp = &ddt->ddt_object[type][class]; 91 char name[DDT_NAMELEN]; 92 93 ddt_object_name(ddt, type, class, name); 94 95 ASSERT(*objectp != 0); 96 ASSERT(ddt_object_count(ddt, type, class) == 0); 97 ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class])); 98 VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0); 99 VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0); 100 VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0); 101 bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t)); 102 103 *objectp = 0; 104} 105 106static int 107ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 108{ 109 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 110 dmu_object_info_t doi; 111 char name[DDT_NAMELEN]; 112 int error; 113 114 ddt_object_name(ddt, type, class, name); 115 116 error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name, 117 sizeof (uint64_t), 1, &ddt->ddt_object[type][class]); 118 119 if (error) 120 return (error); 121 122 error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 123 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 124 &ddt->ddt_histogram[type][class]); 125 126 /* 127 * Seed the cached statistics. 128 */ 129 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 130 131 ddo->ddo_count = ddt_object_count(ddt, type, class); 132 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 133 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 134 135 ASSERT(error == 0); 136 return (error); 137} 138 139static void 140ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 141 dmu_tx_t *tx) 142{ 143 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 144 dmu_object_info_t doi; 145 char name[DDT_NAMELEN]; 146 147 ddt_object_name(ddt, type, class, name); 148 149 VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 150 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 151 &ddt->ddt_histogram[type][class], tx) == 0); 152 153 /* 154 * Cache DDT statistics; this is the only time they'll change. 155 */ 156 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 157 158 ddo->ddo_count = ddt_object_count(ddt, type, class); 159 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 160 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 161} 162 163static int 164ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 165 ddt_entry_t *dde) 166{ 167 if (!ddt_object_exists(ddt, type, class)) 168 return (ENOENT); 169 170 return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os, 171 ddt->ddt_object[type][class], dde)); 172} 173 174static void 175ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 176 ddt_entry_t *dde) 177{ 178 if (!ddt_object_exists(ddt, type, class)) 179 return; 180 181 ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os, 182 ddt->ddt_object[type][class], dde); 183} 184 185int 186ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 187 ddt_entry_t *dde, dmu_tx_t *tx) 188{ 189 ASSERT(ddt_object_exists(ddt, type, class)); 190 191 return (ddt_ops[type]->ddt_op_update(ddt->ddt_os, 192 ddt->ddt_object[type][class], dde, tx)); 193} 194 195static int 196ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 197 ddt_entry_t *dde, dmu_tx_t *tx) 198{ 199 ASSERT(ddt_object_exists(ddt, type, class)); 200 201 return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os, 202 ddt->ddt_object[type][class], dde, tx)); 203} 204 205int 206ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 207 uint64_t *walk, ddt_entry_t *dde) 208{ 209 ASSERT(ddt_object_exists(ddt, type, class)); 210 211 return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os, 212 ddt->ddt_object[type][class], dde, walk)); 213} 214 215uint64_t 216ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 217{ 218 ASSERT(ddt_object_exists(ddt, type, class)); 219 220 return (ddt_ops[type]->ddt_op_count(ddt->ddt_os, 221 ddt->ddt_object[type][class])); 222} 223 224int 225ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 226 dmu_object_info_t *doi) 227{ 228 if (!ddt_object_exists(ddt, type, class)) 229 return (ENOENT); 230 231 return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class], 232 doi)); 233} 234 235boolean_t 236ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 237{ 238 return (!!ddt->ddt_object[type][class]); 239} 240 241void 242ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 243 char *name) 244{ 245 (void) sprintf(name, DMU_POOL_DDT, 246 zio_checksum_table[ddt->ddt_checksum].ci_name, 247 ddt_ops[type]->ddt_op_name, ddt_class_name[class]); 248} 249 250void 251ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg) 252{ 253 ASSERT(txg != 0); 254 255 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 256 bp->blk_dva[d] = ddp->ddp_dva[d]; 257 BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth); 258} 259 260void 261ddt_bp_create(enum zio_checksum checksum, 262 const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp) 263{ 264 BP_ZERO(bp); 265 266 if (ddp != NULL) 267 ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth); 268 269 bp->blk_cksum = ddk->ddk_cksum; 270 bp->blk_fill = 1; 271 272 BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk)); 273 BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk)); 274 BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk)); 275 BP_SET_CHECKSUM(bp, checksum); 276 BP_SET_TYPE(bp, DMU_OT_DEDUP); 277 BP_SET_LEVEL(bp, 0); 278 BP_SET_DEDUP(bp, 0); 279 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 280} 281 282void 283ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp) 284{ 285 ddk->ddk_cksum = bp->blk_cksum; 286 ddk->ddk_prop = 0; 287 288 DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp)); 289 DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp)); 290 DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp)); 291} 292 293void 294ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp) 295{ 296 ASSERT(ddp->ddp_phys_birth == 0); 297 298 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 299 ddp->ddp_dva[d] = bp->blk_dva[d]; 300 ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp); 301} 302 303void 304ddt_phys_clear(ddt_phys_t *ddp) 305{ 306 bzero(ddp, sizeof (*ddp)); 307} 308 309void 310ddt_phys_addref(ddt_phys_t *ddp) 311{ 312 ddp->ddp_refcnt++; 313} 314 315void 316ddt_phys_decref(ddt_phys_t *ddp) 317{ 318 ASSERT((int64_t)ddp->ddp_refcnt > 0); 319 ddp->ddp_refcnt--; 320} 321 322void 323ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg) 324{ 325 blkptr_t blk; 326 327 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 328 ddt_phys_clear(ddp); 329 zio_free(ddt->ddt_spa, txg, &blk); 330} 331 332ddt_phys_t * 333ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp) 334{ 335 ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys; 336 337 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 338 if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) && 339 BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth) 340 return (ddp); 341 } 342 return (NULL); 343} 344 345uint64_t 346ddt_phys_total_refcnt(const ddt_entry_t *dde) 347{ 348 uint64_t refcnt = 0; 349 350 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) 351 refcnt += dde->dde_phys[p].ddp_refcnt; 352 353 return (refcnt); 354} 355 356static void 357ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) 358{ 359 spa_t *spa = ddt->ddt_spa; 360 ddt_phys_t *ddp = dde->dde_phys; 361 ddt_key_t *ddk = &dde->dde_key; 362 uint64_t lsize = DDK_GET_LSIZE(ddk); 363 uint64_t psize = DDK_GET_PSIZE(ddk); 364 365 bzero(dds, sizeof (*dds)); 366 367 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 368 uint64_t dsize = 0; 369 uint64_t refcnt = ddp->ddp_refcnt; 370 371 if (ddp->ddp_phys_birth == 0) 372 continue; 373 374 for (int d = 0; d < SPA_DVAS_PER_BP; d++) 375 dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); 376 377 dds->dds_blocks += 1; 378 dds->dds_lsize += lsize; 379 dds->dds_psize += psize; 380 dds->dds_dsize += dsize; 381 382 dds->dds_ref_blocks += refcnt; 383 dds->dds_ref_lsize += lsize * refcnt; 384 dds->dds_ref_psize += psize * refcnt; 385 dds->dds_ref_dsize += dsize * refcnt; 386 } 387} 388 389void 390ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) 391{ 392 const uint64_t *s = (const uint64_t *)src; 393 uint64_t *d = (uint64_t *)dst; 394 uint64_t *d_end = (uint64_t *)(dst + 1); 395 396 ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ 397 398 while (d < d_end) 399 *d++ += (*s++ ^ neg) - neg; 400} 401 402static void 403ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) 404{ 405 ddt_stat_t dds; 406 ddt_histogram_t *ddh; 407 int bucket; 408 409 ddt_stat_generate(ddt, dde, &dds); 410 411 bucket = highbit(dds.dds_ref_blocks) - 1; 412 ASSERT(bucket >= 0); 413 414 ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; 415 416 ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); 417} 418 419void 420ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) 421{ 422 for (int h = 0; h < 64; h++) 423 ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); 424} 425 426void 427ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) 428{ 429 bzero(dds, sizeof (*dds)); 430 431 for (int h = 0; h < 64; h++) 432 ddt_stat_add(dds, &ddh->ddh_stat[h], 0); 433} 434 435boolean_t 436ddt_histogram_empty(const ddt_histogram_t *ddh) 437{ 438 const uint64_t *s = (const uint64_t *)ddh; 439 const uint64_t *s_end = (const uint64_t *)(ddh + 1); 440 441 while (s < s_end) 442 if (*s++ != 0) 443 return (B_FALSE); 444 445 return (B_TRUE); 446} 447 448void 449ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total) 450{ 451 /* Sum the statistics we cached in ddt_object_sync(). */ 452 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 453 ddt_t *ddt = spa->spa_ddt[c]; 454 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 455 for (enum ddt_class class = 0; class < DDT_CLASSES; 456 class++) { 457 ddt_object_t *ddo = 458 &ddt->ddt_object_stats[type][class]; 459 ddo_total->ddo_count += ddo->ddo_count; 460 ddo_total->ddo_dspace += ddo->ddo_dspace; 461 ddo_total->ddo_mspace += ddo->ddo_mspace; 462 } 463 } 464 } 465 466 /* ... and compute the averages. */ 467 if (ddo_total->ddo_count != 0) { 468 ddo_total->ddo_dspace /= ddo_total->ddo_count; 469 ddo_total->ddo_mspace /= ddo_total->ddo_count; 470 } 471} 472 473void 474ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) 475{ 476 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 477 ddt_t *ddt = spa->spa_ddt[c]; 478 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 479 for (enum ddt_class class = 0; class < DDT_CLASSES; 480 class++) { 481 ddt_histogram_add(ddh, 482 &ddt->ddt_histogram_cache[type][class]); 483 } 484 } 485 } 486} 487 488void 489ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) 490{ 491 ddt_histogram_t *ddh_total; 492 493 ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); 494 ddt_get_dedup_histogram(spa, ddh_total); 495 ddt_histogram_stat(dds_total, ddh_total); 496 kmem_free(ddh_total, sizeof (ddt_histogram_t)); 497} 498 499uint64_t 500ddt_get_dedup_dspace(spa_t *spa) 501{ 502 ddt_stat_t dds_total = { 0 }; 503 504 ddt_get_dedup_stats(spa, &dds_total); 505 return (dds_total.dds_ref_dsize - dds_total.dds_dsize); 506} 507 508uint64_t 509ddt_get_pool_dedup_ratio(spa_t *spa) 510{ 511 ddt_stat_t dds_total = { 0 }; 512 513 ddt_get_dedup_stats(spa, &dds_total); 514 if (dds_total.dds_dsize == 0) 515 return (100); 516 517 return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); 518} 519 520int 521ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref) 522{ 523 spa_t *spa = ddt->ddt_spa; 524 uint64_t total_refcnt = 0; 525 uint64_t ditto = spa->spa_dedup_ditto; 526 int total_copies = 0; 527 int desired_copies = 0; 528 529 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 530 ddt_phys_t *ddp = &dde->dde_phys[p]; 531 zio_t *zio = dde->dde_lead_zio[p]; 532 uint64_t refcnt = ddp->ddp_refcnt; /* committed refs */ 533 if (zio != NULL) 534 refcnt += zio->io_parent_count; /* pending refs */ 535 if (ddp == ddp_willref) 536 refcnt++; /* caller's ref */ 537 if (refcnt != 0) { 538 total_refcnt += refcnt; 539 total_copies += p; 540 } 541 } 542 543 if (ditto == 0 || ditto > UINT32_MAX) 544 ditto = UINT32_MAX; 545 546 if (total_refcnt >= 1) 547 desired_copies++; 548 if (total_refcnt >= ditto) 549 desired_copies++; 550 if (total_refcnt >= ditto * ditto) 551 desired_copies++; 552 553 return (MAX(desired_copies, total_copies) - total_copies); 554} 555 556int 557ddt_ditto_copies_present(ddt_entry_t *dde) 558{ 559 ddt_phys_t *ddp = &dde->dde_phys[DDT_PHYS_DITTO]; 560 dva_t *dva = ddp->ddp_dva; 561 int copies = 0 - DVA_GET_GANG(dva); 562 563 for (int d = 0; d < SPA_DVAS_PER_BP; d++, dva++) 564 if (DVA_IS_VALID(dva)) 565 copies++; 566 567 ASSERT(copies >= 0 && copies < SPA_DVAS_PER_BP); 568 569 return (copies); 570} 571 572size_t 573ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len) 574{ 575 uchar_t *version = dst++; 576 int cpfunc = ZIO_COMPRESS_ZLE; 577 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 578 size_t c_len; 579 580 ASSERT(d_len >= s_len + 1); /* no compression plus version byte */ 581 582 c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level); 583 584 if (c_len == s_len) { 585 cpfunc = ZIO_COMPRESS_OFF; 586 bcopy(src, dst, s_len); 587 } 588 589 *version = (ZFS_HOST_BYTEORDER & DDT_COMPRESS_BYTEORDER_MASK) | cpfunc; 590 591 return (c_len + 1); 592} 593 594void 595ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len) 596{ 597 uchar_t version = *src++; 598 int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK; 599 zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 600 601 if (ci->ci_decompress != NULL) 602 (void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level); 603 else 604 bcopy(src, dst, d_len); 605 606 if ((version ^ ZFS_HOST_BYTEORDER) & DDT_COMPRESS_BYTEORDER_MASK) 607 byteswap_uint64_array(dst, d_len); 608} 609 610ddt_t * 611ddt_select_by_checksum(spa_t *spa, enum zio_checksum c) 612{ 613 return (spa->spa_ddt[c]); 614} 615 616ddt_t * 617ddt_select(spa_t *spa, const blkptr_t *bp) 618{ 619 return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]); 620} 621 622void 623ddt_enter(ddt_t *ddt) 624{ 625 mutex_enter(&ddt->ddt_lock); 626} 627 628void 629ddt_exit(ddt_t *ddt) 630{ 631 mutex_exit(&ddt->ddt_lock); 632} 633 634static ddt_entry_t * 635ddt_alloc(const ddt_key_t *ddk) 636{ 637 ddt_entry_t *dde; 638 639 dde = kmem_zalloc(sizeof (ddt_entry_t), KM_SLEEP); 640 cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL); 641 642 dde->dde_key = *ddk; 643 644 return (dde); 645} 646 647static void 648ddt_free(ddt_entry_t *dde) 649{ 650 ASSERT(!dde->dde_loading); 651 652 for (int p = 0; p < DDT_PHYS_TYPES; p++) 653 ASSERT(dde->dde_lead_zio[p] == NULL); 654 655 if (dde->dde_repair_data != NULL) 656 zio_buf_free(dde->dde_repair_data, 657 DDK_GET_PSIZE(&dde->dde_key)); 658 659 cv_destroy(&dde->dde_cv); 660 kmem_free(dde, sizeof (*dde)); 661} 662 663void 664ddt_remove(ddt_t *ddt, ddt_entry_t *dde) 665{ 666 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 667 668 avl_remove(&ddt->ddt_tree, dde); 669 ddt_free(dde); 670} 671 672ddt_entry_t * 673ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add) 674{ 675 ddt_entry_t *dde, dde_search; 676 enum ddt_type type; 677 enum ddt_class class; 678 avl_index_t where; 679 int error; 680 681 ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 682 683 ddt_key_fill(&dde_search.dde_key, bp); 684 685 dde = avl_find(&ddt->ddt_tree, &dde_search, &where); 686 if (dde == NULL) { 687 if (!add) 688 return (NULL); 689 dde = ddt_alloc(&dde_search.dde_key); 690 avl_insert(&ddt->ddt_tree, dde, where); 691 } 692 693 while (dde->dde_loading) 694 cv_wait(&dde->dde_cv, &ddt->ddt_lock); 695 696 if (dde->dde_loaded) 697 return (dde); 698 699 dde->dde_loading = B_TRUE; 700 701 ddt_exit(ddt); 702 703 error = ENOENT; 704 705 for (type = 0; type < DDT_TYPES; type++) { 706 for (class = 0; class < DDT_CLASSES; class++) { 707 error = ddt_object_lookup(ddt, type, class, dde); 708 if (error != ENOENT) 709 break; 710 } 711 if (error != ENOENT) 712 break; 713 } 714 715 ASSERT(error == 0 || error == ENOENT); 716 717 ddt_enter(ddt); 718 719 ASSERT(dde->dde_loaded == B_FALSE); 720 ASSERT(dde->dde_loading == B_TRUE); 721 722 dde->dde_type = type; /* will be DDT_TYPES if no entry found */ 723 dde->dde_class = class; /* will be DDT_CLASSES if no entry found */ 724 dde->dde_loaded = B_TRUE; 725 dde->dde_loading = B_FALSE; 726 727 if (error == 0) 728 ddt_stat_update(ddt, dde, -1ULL); 729 730 cv_broadcast(&dde->dde_cv); 731 732 return (dde); 733} 734 735void 736ddt_prefetch(spa_t *spa, const blkptr_t *bp) 737{ 738 ddt_t *ddt; 739 ddt_entry_t dde; 740 741 if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp)) 742 return; 743 744 /* 745 * We only remove the DDT once all tables are empty and only 746 * prefetch dedup blocks when there are entries in the DDT. 747 * Thus no locking is required as the DDT can't disappear on us. 748 */ 749 ddt = ddt_select(spa, bp); 750 ddt_key_fill(&dde.dde_key, bp); 751 752 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 753 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 754 ddt_object_prefetch(ddt, type, class, &dde); 755 } 756 } 757} 758 759int 760ddt_entry_compare(const void *x1, const void *x2) 761{ 762 const ddt_entry_t *dde1 = x1; 763 const ddt_entry_t *dde2 = x2; 764 const uint64_t *u1 = (const uint64_t *)&dde1->dde_key; 765 const uint64_t *u2 = (const uint64_t *)&dde2->dde_key; 766 767 for (int i = 0; i < DDT_KEY_WORDS; i++) { 768 if (u1[i] < u2[i]) 769 return (-1); 770 if (u1[i] > u2[i]) 771 return (1); 772 } 773 774 return (0); 775} 776 777static ddt_t * 778ddt_table_alloc(spa_t *spa, enum zio_checksum c) 779{ 780 ddt_t *ddt; 781 782 ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP); 783 784 mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL); 785 avl_create(&ddt->ddt_tree, ddt_entry_compare, 786 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 787 avl_create(&ddt->ddt_repair_tree, ddt_entry_compare, 788 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 789 ddt->ddt_checksum = c; 790 ddt->ddt_spa = spa; 791 ddt->ddt_os = spa->spa_meta_objset; 792 793 return (ddt); 794} 795 796static void 797ddt_table_free(ddt_t *ddt) 798{ 799 ASSERT(avl_numnodes(&ddt->ddt_tree) == 0); 800 ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0); 801 avl_destroy(&ddt->ddt_tree); 802 avl_destroy(&ddt->ddt_repair_tree); 803 mutex_destroy(&ddt->ddt_lock); 804 kmem_free(ddt, sizeof (*ddt)); 805} 806 807void 808ddt_create(spa_t *spa) 809{ 810 spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM; 811 812 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) 813 spa->spa_ddt[c] = ddt_table_alloc(spa, c); 814} 815 816int 817ddt_load(spa_t *spa) 818{ 819 int error; 820 821 ddt_create(spa); 822 823 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 824 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 825 &spa->spa_ddt_stat_object); 826 827 if (error) 828 return (error == ENOENT ? 0 : error); 829 830 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 831 ddt_t *ddt = spa->spa_ddt[c]; 832 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 833 for (enum ddt_class class = 0; class < DDT_CLASSES; 834 class++) { 835 error = ddt_object_load(ddt, type, class); 836 if (error != 0 && error != ENOENT) 837 return (error); 838 } 839 } 840 841 /* 842 * Seed the cached histograms. 843 */ 844 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 845 sizeof (ddt->ddt_histogram)); 846 } 847 848 return (0); 849} 850 851void 852ddt_unload(spa_t *spa) 853{ 854 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 855 if (spa->spa_ddt[c]) { 856 ddt_table_free(spa->spa_ddt[c]); 857 spa->spa_ddt[c] = NULL; 858 } 859 } 860} 861 862boolean_t 863ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp) 864{ 865 ddt_t *ddt; 866 ddt_entry_t dde; 867 868 if (!BP_GET_DEDUP(bp)) 869 return (B_FALSE); 870 871 if (max_class == DDT_CLASS_UNIQUE) 872 return (B_TRUE); 873 874 ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)]; 875 876 ddt_key_fill(&dde.dde_key, bp); 877 878 for (enum ddt_type type = 0; type < DDT_TYPES; type++) 879 for (enum ddt_class class = 0; class <= max_class; class++) 880 if (ddt_object_lookup(ddt, type, class, &dde) == 0) 881 return (B_TRUE); 882 883 return (B_FALSE); 884} 885 886ddt_entry_t * 887ddt_repair_start(ddt_t *ddt, const blkptr_t *bp) 888{ 889 ddt_key_t ddk; 890 ddt_entry_t *dde; 891 892 ddt_key_fill(&ddk, bp); 893 894 dde = ddt_alloc(&ddk); 895 896 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 897 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 898 /* 899 * We can only do repair if there are multiple copies 900 * of the block. For anything in the UNIQUE class, 901 * there's definitely only one copy, so don't even try. 902 */ 903 if (class != DDT_CLASS_UNIQUE && 904 ddt_object_lookup(ddt, type, class, dde) == 0) 905 return (dde); 906 } 907 } 908 909 bzero(dde->dde_phys, sizeof (dde->dde_phys)); 910 911 return (dde); 912} 913 914void 915ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde) 916{ 917 avl_index_t where; 918 919 ddt_enter(ddt); 920 921 if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) && 922 avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL) 923 avl_insert(&ddt->ddt_repair_tree, dde, where); 924 else 925 ddt_free(dde); 926 927 ddt_exit(ddt); 928} 929 930static void 931ddt_repair_entry_done(zio_t *zio) 932{ 933 ddt_entry_t *rdde = zio->io_private; 934 935 ddt_free(rdde); 936} 937 938static void 939ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio) 940{ 941 ddt_phys_t *ddp = dde->dde_phys; 942 ddt_phys_t *rddp = rdde->dde_phys; 943 ddt_key_t *ddk = &dde->dde_key; 944 ddt_key_t *rddk = &rdde->dde_key; 945 zio_t *zio; 946 blkptr_t blk; 947 948 zio = zio_null(rio, rio->io_spa, NULL, 949 ddt_repair_entry_done, rdde, rio->io_flags); 950 951 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) { 952 if (ddp->ddp_phys_birth == 0 || 953 ddp->ddp_phys_birth != rddp->ddp_phys_birth || 954 bcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva))) 955 continue; 956 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 957 zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk, 958 rdde->dde_repair_data, DDK_GET_PSIZE(rddk), NULL, NULL, 959 ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL)); 960 } 961 962 zio_nowait(zio); 963} 964 965static void 966ddt_repair_table(ddt_t *ddt, zio_t *rio) 967{ 968 spa_t *spa = ddt->ddt_spa; 969 ddt_entry_t *dde, *rdde_next, *rdde; 970 avl_tree_t *t = &ddt->ddt_repair_tree; 971 blkptr_t blk; 972 973 if (spa_sync_pass(spa) > 1) 974 return; 975 976 ddt_enter(ddt); 977 for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) { 978 rdde_next = AVL_NEXT(t, rdde); 979 avl_remove(&ddt->ddt_repair_tree, rdde); 980 ddt_exit(ddt); 981 ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk); 982 dde = ddt_repair_start(ddt, &blk); 983 ddt_repair_entry(ddt, dde, rdde, rio); 984 ddt_repair_done(ddt, dde); 985 ddt_enter(ddt); 986 } 987 ddt_exit(ddt); 988} 989 990static void 991ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg) 992{ 993 dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool; 994 ddt_phys_t *ddp = dde->dde_phys; 995 ddt_key_t *ddk = &dde->dde_key; 996 enum ddt_type otype = dde->dde_type; 997 enum ddt_type ntype = DDT_TYPE_CURRENT; 998 enum ddt_class oclass = dde->dde_class; 999 enum ddt_class nclass; 1000 uint64_t total_refcnt = 0; 1001 1002 ASSERT(dde->dde_loaded); 1003 ASSERT(!dde->dde_loading); 1004 1005 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1006 ASSERT(dde->dde_lead_zio[p] == NULL); 1007 ASSERT((int64_t)ddp->ddp_refcnt >= 0); 1008 if (ddp->ddp_phys_birth == 0) { 1009 ASSERT(ddp->ddp_refcnt == 0); 1010 continue; 1011 } 1012 if (p == DDT_PHYS_DITTO) { 1013 if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0) 1014 ddt_phys_free(ddt, ddk, ddp, txg); 1015 continue; 1016 } 1017 if (ddp->ddp_refcnt == 0) 1018 ddt_phys_free(ddt, ddk, ddp, txg); 1019 total_refcnt += ddp->ddp_refcnt; 1020 } 1021 1022 if (dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth != 0) 1023 nclass = DDT_CLASS_DITTO; 1024 else if (total_refcnt > 1) 1025 nclass = DDT_CLASS_DUPLICATE; 1026 else 1027 nclass = DDT_CLASS_UNIQUE; 1028 1029 if (otype != DDT_TYPES && 1030 (otype != ntype || oclass != nclass || total_refcnt == 0)) { 1031 VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0); 1032 ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT); 1033 } 1034 1035 if (total_refcnt != 0) { 1036 dde->dde_type = ntype; 1037 dde->dde_class = nclass; 1038 ddt_stat_update(ddt, dde, 0); 1039 if (!ddt_object_exists(ddt, ntype, nclass)) 1040 ddt_object_create(ddt, ntype, nclass, tx); 1041 VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0); 1042 1043 /* 1044 * If the class changes, the order that we scan this bp 1045 * changes. If it decreases, we could miss it, so 1046 * scan it right now. (This covers both class changing 1047 * while we are doing ddt_walk(), and when we are 1048 * traversing.) 1049 */ 1050 if (nclass < oclass) { 1051 dsl_scan_ddt_entry(dp->dp_scan, 1052 ddt->ddt_checksum, dde, tx); 1053 } 1054 } 1055} 1056 1057static void 1058ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg) 1059{ 1060 spa_t *spa = ddt->ddt_spa; 1061 ddt_entry_t *dde; 1062 void *cookie = NULL; 1063 1064 if (avl_numnodes(&ddt->ddt_tree) == 0) 1065 return; 1066 1067 ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP); 1068 1069 if (spa->spa_ddt_stat_object == 0) { 1070 spa->spa_ddt_stat_object = zap_create(ddt->ddt_os, 1071 DMU_OT_DDT_STATS, DMU_OT_NONE, 0, tx); 1072 VERIFY(zap_add(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, 1073 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 1074 &spa->spa_ddt_stat_object, tx) == 0); 1075 } 1076 1077 while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) { 1078 ddt_sync_entry(ddt, dde, tx, txg); 1079 ddt_free(dde); 1080 } 1081 1082 for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 1083 uint64_t count = 0; 1084 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1085 if (ddt_object_exists(ddt, type, class)) { 1086 ddt_object_sync(ddt, type, class, tx); 1087 count += ddt_object_count(ddt, type, class); 1088 } 1089 } 1090 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1091 if (count == 0 && ddt_object_exists(ddt, type, class)) 1092 ddt_object_destroy(ddt, type, class, tx); 1093 } 1094 } 1095 1096 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 1097 sizeof (ddt->ddt_histogram)); 1098} 1099 1100void 1101ddt_sync(spa_t *spa, uint64_t txg) 1102{ 1103 dmu_tx_t *tx; 1104 zio_t *rio = zio_root(spa, NULL, NULL, 1105 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 1106 1107 ASSERT(spa_syncing_txg(spa) == txg); 1108 1109 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 1110 1111 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 1112 ddt_t *ddt = spa->spa_ddt[c]; 1113 if (ddt == NULL) 1114 continue; 1115 ddt_sync_table(ddt, tx, txg); 1116 ddt_repair_table(ddt, rio); 1117 } 1118 1119 (void) zio_wait(rio); 1120 1121 dmu_tx_commit(tx); 1122} 1123 1124int 1125ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) 1126{ 1127 do { 1128 do { 1129 do { 1130 ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum]; 1131 int error = ENOENT; 1132 if (ddt_object_exists(ddt, ddb->ddb_type, 1133 ddb->ddb_class)) { 1134 error = ddt_object_walk(ddt, 1135 ddb->ddb_type, ddb->ddb_class, 1136 &ddb->ddb_cursor, dde); 1137 } 1138 dde->dde_type = ddb->ddb_type; 1139 dde->dde_class = ddb->ddb_class; 1140 if (error == 0) 1141 return (0); 1142 if (error != ENOENT) 1143 return (error); 1144 ddb->ddb_cursor = 0; 1145 } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS); 1146 ddb->ddb_checksum = 0; 1147 } while (++ddb->ddb_type < DDT_TYPES); 1148 ddb->ddb_type = 0; 1149 } while (++ddb->ddb_class < DDT_CLASSES); 1150 1151 return (ENOENT); 1152} 1153