zap.c revision 256281
156893Sfenner/* 256893Sfenner * CDDL HEADER START 356893Sfenner * 4127668Sbms * The contents of this file are subject to the terms of the 556893Sfenner * Common Development and Distribution License (the "License"). 656893Sfenner * You may not use this file except in compliance with the License. 756893Sfenner * 856893Sfenner * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 956893Sfenner * or http://www.opensolaris.org/os/licensing. 1056893Sfenner * See the License for the specific language governing permissions 1156893Sfenner * and limitations under the License. 1256893Sfenner * 1356893Sfenner * When distributing Covered Code, include this CDDL HEADER in each 1456893Sfenner * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1556893Sfenner * If applicable, add the following below this CDDL HEADER, with the 16127668Sbms * fields enclosed by brackets "[]" replaced with your own identifying 1756893Sfenner * information: Portions Copyright [yyyy] [name of copyright owner] 1856893Sfenner * 1956893Sfenner * CDDL HEADER END 2056893Sfenner */ 2156893Sfenner/* 2256893Sfenner * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 2356893Sfenner * Copyright (c) 2013 by Delphix. All rights reserved. 2456893Sfenner */ 2556893Sfenner 2656893Sfenner/* 2756893Sfenner * This file contains the top half of the zfs directory structure 2856893Sfenner * implementation. The bottom half is in zap_leaf.c. 29111726Sfenner * 30127668Sbms * The zdir is an extendable hash data structure. There is a table of 31276788Sdelphij * pointers to buckets (zap_t->zd_data->zd_leafs). The buckets are 32251158Sdelphij * each a constant size and hold a variable number of directory entries. 33251158Sdelphij * The buckets (aka "leaf nodes") are implemented in zap_leaf.c. 34251158Sdelphij * 35251158Sdelphij * The pointer table holds a power of 2 number of pointers. 36251158Sdelphij * (1<<zap_t->zd_data->zd_phys->zd_prefix_len). The bucket pointed to 37251158Sdelphij * by the pointer at index i in the table holds entries whose hash value 38251158Sdelphij * has a zd_prefix_len - bit prefix 39276788Sdelphij */ 40251158Sdelphij 41111726Sfenner#include <sys/spa.h> 4256893Sfenner#include <sys/dmu.h> 43276788Sdelphij#include <sys/zfs_context.h> 4456893Sfenner#include <sys/zfs_znode.h> 4556893Sfenner#include <sys/fs/zfs.h> 4656893Sfenner#include <sys/zap.h> 4756893Sfenner#include <sys/refcount.h> 48127668Sbms#include <sys/zap_impl.h> 4956893Sfenner#include <sys/zap_leaf.h> 5056893Sfenner 5156893Sfennerint fzap_default_block_shift = 14; /* 16k blocksize */ 5256893Sfenner 5356893Sfennerstatic void zap_leaf_pageout(dmu_buf_t *db, void *vl); 5456893Sfennerstatic uint64_t zap_allocate_blocks(zap_t *zap, int nblocks); 55127668Sbms 5656893Sfenner 57127668Sbmsvoid 58276788Sdelphijfzap_byteswap(void *vbuf, size_t size) 59127668Sbms{ 60111726Sfenner uint64_t block_type; 61111726Sfenner 62111726Sfenner block_type = *(uint64_t *)vbuf; 63111726Sfenner 64111726Sfenner if (block_type == ZBT_LEAF || block_type == BSWAP_64(ZBT_LEAF)) 65111726Sfenner zap_leaf_byteswap(vbuf, size); 6656893Sfenner else { 67111726Sfenner /* it's a ptrtbl block */ 68127668Sbms byteswap_uint64_array(vbuf, size); 69127668Sbms } 70127668Sbms} 71127668Sbms 72127668Sbmsvoid 73127668Sbmsfzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags) 74111726Sfenner{ 75127668Sbms dmu_buf_t *db; 76127668Sbms zap_leaf_t *l; 77127668Sbms int i; 78111726Sfenner zap_phys_t *zp; 79127668Sbms 80127668Sbms ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 81190207Srpaulo zap->zap_ismicro = FALSE; 82190207Srpaulo 8356893Sfenner (void) dmu_buf_update_user(zap->zap_dbuf, zap, zap, 84276788Sdelphij &zap->zap_f.zap_phys, zap_evict); 85276788Sdelphij 86276788Sdelphij mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, 0, 0); 87276788Sdelphij zap->zap_f.zap_block_shift = highbit(zap->zap_dbuf->db_size) - 1; 88276788Sdelphij 89276788Sdelphij zp = zap->zap_f.zap_phys; 90276788Sdelphij /* 91276788Sdelphij * explicitly zero it since it might be coming from an 92276788Sdelphij * initialized microzap 93276788Sdelphij */ 94276788Sdelphij bzero(zap->zap_dbuf->db_data, zap->zap_dbuf->db_size); 95276788Sdelphij zp->zap_block_type = ZBT_HEADER; 96276788Sdelphij zp->zap_magic = ZAP_MAGIC; 97276788Sdelphij 98276788Sdelphij zp->zap_ptrtbl.zt_shift = ZAP_EMBEDDED_PTRTBL_SHIFT(zap); 99276788Sdelphij 100276788Sdelphij zp->zap_freeblk = 2; /* block 1 will be the first leaf */ 101276788Sdelphij zp->zap_num_leafs = 1; 102276788Sdelphij zp->zap_num_entries = 0; 103111726Sfenner zp->zap_salt = zap->zap_salt; 104111726Sfenner zp->zap_normflags = zap->zap_normflags; 105111726Sfenner zp->zap_flags = flags; 106276788Sdelphij 107276788Sdelphij /* block 1 will be the first leaf */ 108111726Sfenner for (i = 0; i < (1<<zp->zap_ptrtbl.zt_shift); i++) 109111726Sfenner ZAP_EMBEDDED_PTRTBL_ENT(zap, i) = 1; 110127668Sbms 111111726Sfenner /* 112111726Sfenner * set up block 1 - the first leaf 113111726Sfenner */ 11456893Sfenner VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 115127668Sbms 1<<FZAP_BLOCK_SHIFT(zap), FTAG, &db, DMU_READ_NO_PREFETCH)); 116127668Sbms dmu_buf_will_dirty(db, tx); 117276788Sdelphij 118276788Sdelphij l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP); 119276788Sdelphij l->l_dbuf = db; 120276788Sdelphij l->l_phys = db->db_data; 121127668Sbms 122127668Sbms zap_leaf_init(l, zp->zap_normflags != 0); 123127668Sbms 124127668Sbms kmem_free(l, sizeof (zap_leaf_t)); 125127668Sbms dmu_buf_rele(db, FTAG); 126127668Sbms} 127127668Sbms 128172683Smlaierstatic int 129172683Smlaierzap_tryupgradedir(zap_t *zap, dmu_tx_t *tx) 130127668Sbms{ 131127668Sbms if (RW_WRITE_HELD(&zap->zap_rwlock)) 132127668Sbms return (1); 133127668Sbms if (rw_tryupgrade(&zap->zap_rwlock)) { 134127668Sbms dmu_buf_will_dirty(zap->zap_dbuf, tx); 135127668Sbms return (1); 136127668Sbms } 137146773Ssam return (0); 138146773Ssam} 139146773Ssam 140146773Ssam/* 141146773Ssam * Generic routines for dealing with the pointer & cookie tables. 142146773Ssam */ 143127668Sbms 144127668Sbmsstatic int 145127668Sbmszap_table_grow(zap_t *zap, zap_table_phys_t *tbl, 146127668Sbms void (*transfer_func)(const uint64_t *src, uint64_t *dst, int n), 147127668Sbms dmu_tx_t *tx) 148127668Sbms{ 149127668Sbms uint64_t b, newblk; 150127668Sbms dmu_buf_t *db_old, *db_new; 151127668Sbms int err; 152190207Srpaulo int bs = FZAP_BLOCK_SHIFT(zap); 153190207Srpaulo int hepb = 1<<(bs-4); 154190207Srpaulo /* hepb = half the number of entries in a block */ 155190207Srpaulo 156127668Sbms ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 157127668Sbms ASSERT(tbl->zt_blk != 0); 158127668Sbms ASSERT(tbl->zt_numblks > 0); 159127668Sbms 160127668Sbms if (tbl->zt_nextblk != 0) { 161127668Sbms newblk = tbl->zt_nextblk; 162127668Sbms } else { 163127668Sbms newblk = zap_allocate_blocks(zap, tbl->zt_numblks * 2); 164127668Sbms tbl->zt_nextblk = newblk; 165276788Sdelphij ASSERT0(tbl->zt_blks_copied); 166276788Sdelphij dmu_prefetch(zap->zap_objset, zap->zap_object, 167146773Ssam tbl->zt_blk << bs, tbl->zt_numblks << bs); 168146773Ssam } 169172683Smlaier 170172683Smlaier /* 171172683Smlaier * Copy the ptrtbl from the old to new location. 172172683Smlaier */ 173276788Sdelphij 174172683Smlaier b = tbl->zt_blks_copied; 175172683Smlaier err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 176172683Smlaier (tbl->zt_blk + b) << bs, FTAG, &db_old, DMU_READ_NO_PREFETCH); 177172683Smlaier if (err) 178172683Smlaier return (err); 179172683Smlaier 180172683Smlaier /* first half of entries in old[b] go to new[2*b+0] */ 181190207Srpaulo VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 182190207Srpaulo (newblk + 2*b+0) << bs, FTAG, &db_new, DMU_READ_NO_PREFETCH)); 183190207Srpaulo dmu_buf_will_dirty(db_new, tx); 184190207Srpaulo transfer_func(db_old->db_data, db_new->db_data, hepb); 185190207Srpaulo dmu_buf_rele(db_new, FTAG); 186190207Srpaulo 187190207Srpaulo /* second half of entries in old[b] go to new[2*b+1] */ 188190207Srpaulo VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 189190207Srpaulo (newblk + 2*b+1) << bs, FTAG, &db_new, DMU_READ_NO_PREFETCH)); 190276788Sdelphij dmu_buf_will_dirty(db_new, tx); 191276788Sdelphij transfer_func((uint64_t *)db_old->db_data + hepb, 192276788Sdelphij db_new->db_data, hepb); 193276788Sdelphij dmu_buf_rele(db_new, FTAG); 194251158Sdelphij 195127668Sbms dmu_buf_rele(db_old, FTAG); 196276788Sdelphij 197276788Sdelphij tbl->zt_blks_copied++; 198276788Sdelphij 199276788Sdelphij dprintf("copied block %llu of %llu\n", 200276788Sdelphij tbl->zt_blks_copied, tbl->zt_numblks); 201276788Sdelphij 202276788Sdelphij if (tbl->zt_blks_copied == tbl->zt_numblks) { 203276788Sdelphij (void) dmu_free_range(zap->zap_objset, zap->zap_object, 204276788Sdelphij tbl->zt_blk << bs, tbl->zt_numblks << bs, tx); 205276788Sdelphij 206276788Sdelphij tbl->zt_blk = newblk; 207276788Sdelphij tbl->zt_numblks *= 2; 208276788Sdelphij tbl->zt_shift++; 209276788Sdelphij tbl->zt_nextblk = 0; 210276788Sdelphij tbl->zt_blks_copied = 0; 211276788Sdelphij 212276788Sdelphij dprintf("finished; numblocks now %llu (%lluk entries)\n", 213276788Sdelphij tbl->zt_numblks, 1<<(tbl->zt_shift-10)); 214276788Sdelphij } 215276788Sdelphij 216276788Sdelphij return (0); 217276788Sdelphij} 218276788Sdelphij 219276788Sdelphijstatic int 220276788Sdelphijzap_table_store(zap_t *zap, zap_table_phys_t *tbl, uint64_t idx, uint64_t val, 221276788Sdelphij dmu_tx_t *tx) 222276788Sdelphij{ 223276788Sdelphij int err; 224276788Sdelphij uint64_t blk, off; 225276788Sdelphij int bs = FZAP_BLOCK_SHIFT(zap); 226276788Sdelphij dmu_buf_t *db; 227276788Sdelphij 228276788Sdelphij ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 229276788Sdelphij ASSERT(tbl->zt_blk != 0); 230276788Sdelphij 231276788Sdelphij dprintf("storing %llx at index %llx\n", val, idx); 232276788Sdelphij 233276788Sdelphij blk = idx >> (bs-3); 234276788Sdelphij off = idx & ((1<<(bs-3))-1); 235276788Sdelphij 236276788Sdelphij err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 237276788Sdelphij (tbl->zt_blk + blk) << bs, FTAG, &db, DMU_READ_NO_PREFETCH); 238276788Sdelphij if (err) 239276788Sdelphij return (err); 240276788Sdelphij dmu_buf_will_dirty(db, tx); 241276788Sdelphij 242276788Sdelphij if (tbl->zt_nextblk != 0) { 243276788Sdelphij uint64_t idx2 = idx * 2; 244276788Sdelphij uint64_t blk2 = idx2 >> (bs-3); 245276788Sdelphij uint64_t off2 = idx2 & ((1<<(bs-3))-1); 246276788Sdelphij dmu_buf_t *db2; 247276788Sdelphij 248276788Sdelphij err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 249276788Sdelphij (tbl->zt_nextblk + blk2) << bs, FTAG, &db2, 250276788Sdelphij DMU_READ_NO_PREFETCH); 251276788Sdelphij if (err) { 252276788Sdelphij dmu_buf_rele(db, FTAG); 253276788Sdelphij return (err); 254276788Sdelphij } 255276788Sdelphij dmu_buf_will_dirty(db2, tx); 256276788Sdelphij ((uint64_t *)db2->db_data)[off2] = val; 257276788Sdelphij ((uint64_t *)db2->db_data)[off2+1] = val; 258276788Sdelphij dmu_buf_rele(db2, FTAG); 259276788Sdelphij } 260276788Sdelphij 261276788Sdelphij ((uint64_t *)db->db_data)[off] = val; 262276788Sdelphij dmu_buf_rele(db, FTAG); 263111726Sfenner 264276788Sdelphij return (0); 265276788Sdelphij} 266111726Sfenner 267127668Sbmsstatic int 26856893Sfennerzap_table_load(zap_t *zap, zap_table_phys_t *tbl, uint64_t idx, uint64_t *valp) 269127668Sbms{ 270276788Sdelphij uint64_t blk, off; 271127668Sbms int err; 272276788Sdelphij dmu_buf_t *db; 273127668Sbms int bs = FZAP_BLOCK_SHIFT(zap); 274127668Sbms 27556893Sfenner ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 276276788Sdelphij 277276788Sdelphij blk = idx >> (bs-3); 27856893Sfenner off = idx & ((1<<(bs-3))-1); 279276788Sdelphij 280276788Sdelphij err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 281127668Sbms (tbl->zt_blk + blk) << bs, FTAG, &db, DMU_READ_NO_PREFETCH); 282276788Sdelphij if (err) 283111726Sfenner return (err); 284276788Sdelphij *valp = ((uint64_t *)db->db_data)[off]; 285146773Ssam dmu_buf_rele(db, FTAG); 286276788Sdelphij 287276788Sdelphij if (tbl->zt_nextblk != 0) { 288276788Sdelphij /* 289276788Sdelphij * read the nextblk for the sake of i/o error checking, 29056893Sfenner * so that zap_table_load() will catch errors for 29156893Sfenner * zap_table_store. 29256893Sfenner */ 29356893Sfenner blk = (idx*2) >> (bs-3); 294127668Sbms 295111726Sfenner err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 296111726Sfenner (tbl->zt_nextblk + blk) << bs, FTAG, &db, 297276788Sdelphij DMU_READ_NO_PREFETCH); 298127668Sbms if (err == 0) 299127668Sbms dmu_buf_rele(db, FTAG); 300111726Sfenner } 301127668Sbms return (err); 302276788Sdelphij} 303127668Sbms 304127668Sbms/* 305127668Sbms * Routines for growing the ptrtbl. 306111726Sfenner */ 307111726Sfenner 308276788Sdelphijstatic void 309111726Sfennerzap_ptrtbl_transfer(const uint64_t *src, uint64_t *dst, int n) 310111726Sfenner{ 311111726Sfenner int i; 312127668Sbms for (i = 0; i < n; i++) { 313111726Sfenner uint64_t lb = src[i]; 314111726Sfenner dst[2*i+0] = lb; 315276788Sdelphij dst[2*i+1] = lb; 316127668Sbms } 317276788Sdelphij} 318111726Sfenner 319276788Sdelphijstatic int 320111726Sfennerzap_grow_ptrtbl(zap_t *zap, dmu_tx_t *tx) 321276788Sdelphij{ 322111726Sfenner /* 323111726Sfenner * The pointer table should never use more hash bits than we 324276788Sdelphij * have (otherwise we'd be using useless zero bits to index it). 325111726Sfenner * If we are within 2 bits of running out, stop growing, since 326111726Sfenner * this is already an aberrant condition. 327111726Sfenner */ 328111726Sfenner if (zap->zap_f.zap_phys->zap_ptrtbl.zt_shift >= zap_hashbits(zap) - 2) 329276788Sdelphij return (SET_ERROR(ENOSPC)); 330111726Sfenner 331276788Sdelphij if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) { 332111726Sfenner /* 333276788Sdelphij * We are outgrowing the "embedded" ptrtbl (the one 334111726Sfenner * stored in the header block). Give it its own entire 335111726Sfenner * block, which will double the size of the ptrtbl. 336276788Sdelphij */ 337111726Sfenner uint64_t newblk; 338111726Sfenner dmu_buf_t *db_new; 339111726Sfenner int err; 340111726Sfenner 341276788Sdelphij ASSERT3U(zap->zap_f.zap_phys->zap_ptrtbl.zt_shift, ==, 342276788Sdelphij ZAP_EMBEDDED_PTRTBL_SHIFT(zap)); 343111726Sfenner ASSERT0(zap->zap_f.zap_phys->zap_ptrtbl.zt_blk); 344276788Sdelphij 345111726Sfenner newblk = zap_allocate_blocks(zap, 1); 346276788Sdelphij err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 347111726Sfenner newblk << FZAP_BLOCK_SHIFT(zap), FTAG, &db_new, 348111726Sfenner DMU_READ_NO_PREFETCH); 349276788Sdelphij if (err) 350111726Sfenner return (err); 351127668Sbms dmu_buf_will_dirty(db_new, tx); 352127668Sbms zap_ptrtbl_transfer(&ZAP_EMBEDDED_PTRTBL_ENT(zap, 0), 353276788Sdelphij db_new->db_data, 1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap)); 354127668Sbms dmu_buf_rele(db_new, FTAG); 355111726Sfenner 35675115Sfenner zap->zap_f.zap_phys->zap_ptrtbl.zt_blk = newblk; 357172683Smlaier zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks = 1; 358214478Srpaulo zap->zap_f.zap_phys->zap_ptrtbl.zt_shift++; 359214478Srpaulo 360276788Sdelphij ASSERT3U(1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift, ==, 361172683Smlaier zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks << 362172683Smlaier (FZAP_BLOCK_SHIFT(zap)-3)); 363214478Srpaulo 364276788Sdelphij return (0); 365276788Sdelphij } else { 366214478Srpaulo return (zap_table_grow(zap, &zap->zap_f.zap_phys->zap_ptrtbl, 367276788Sdelphij zap_ptrtbl_transfer, tx)); 368214478Srpaulo } 369172683Smlaier} 370276788Sdelphij 371172683Smlaierstatic void 372276788Sdelphijzap_increment_num_entries(zap_t *zap, int delta, dmu_tx_t *tx) 373172683Smlaier{ 374127668Sbms dmu_buf_will_dirty(zap->zap_dbuf, tx); 375190207Srpaulo mutex_enter(&zap->zap_f.zap_num_entries_mtx); 376127668Sbms ASSERT(delta > 0 || zap->zap_f.zap_phys->zap_num_entries >= -delta); 377276788Sdelphij zap->zap_f.zap_phys->zap_num_entries += delta; 378127668Sbms mutex_exit(&zap->zap_f.zap_num_entries_mtx); 379127668Sbms} 380127668Sbms 381127668Sbmsstatic uint64_t 382276788Sdelphijzap_allocate_blocks(zap_t *zap, int nblocks) 383276788Sdelphij{ 384127668Sbms uint64_t newblk; 385276788Sdelphij ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 386127668Sbms newblk = zap->zap_f.zap_phys->zap_freeblk; 387127668Sbms zap->zap_f.zap_phys->zap_freeblk += nblocks; 388127668Sbms return (newblk); 389276788Sdelphij} 390127668Sbms 391127668Sbmsstatic zap_leaf_t * 392214478Srpaulozap_create_leaf(zap_t *zap, dmu_tx_t *tx) 393276788Sdelphij{ 394127668Sbms void *winner; 395127668Sbms zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); 396127668Sbms 397276788Sdelphij ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 398127668Sbms 399127668Sbms rw_init(&l->l_rwlock, 0, 0, 0); 400214478Srpaulo rw_enter(&l->l_rwlock, RW_WRITER); 401276788Sdelphij l->l_blkid = zap_allocate_blocks(zap, 1); 402127668Sbms l->l_dbuf = NULL; 403127668Sbms l->l_phys = NULL; 404276788Sdelphij 405214478Srpaulo VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object, 406276788Sdelphij l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf, 407276788Sdelphij DMU_READ_NO_PREFETCH)); 408127668Sbms winner = dmu_buf_set_user(l->l_dbuf, l, &l->l_phys, zap_leaf_pageout); 409146773Ssam ASSERT(winner == NULL); 410214478Srpaulo dmu_buf_will_dirty(l->l_dbuf, tx); 411276788Sdelphij 412146773Ssam zap_leaf_init(l, zap->zap_normflags != 0); 413146773Ssam 414214478Srpaulo zap->zap_f.zap_phys->zap_num_leafs++; 415214478Srpaulo 416214478Srpaulo return (l); 417146773Ssam} 418276788Sdelphij 419146773Ssamint 420146773Ssamfzap_count(zap_t *zap, uint64_t *count) 421276788Sdelphij{ 422146773Ssam ASSERT(!zap->zap_ismicro); 423146773Ssam mutex_enter(&zap->zap_f.zap_num_entries_mtx); /* unnecessary */ 424276788Sdelphij *count = zap->zap_f.zap_phys->zap_num_entries; 425146773Ssam mutex_exit(&zap->zap_f.zap_num_entries_mtx); 426146773Ssam return (0); 427214478Srpaulo} 428214478Srpaulo 429146773Ssam/* 430146773Ssam * Routines for obtaining zap_leaf_t's 431276788Sdelphij */ 432146773Ssam 433146773Ssamvoid 434276788Sdelphijzap_put_leaf(zap_leaf_t *l) 435146773Ssam{ 436146773Ssam rw_exit(&l->l_rwlock); 437214478Srpaulo dmu_buf_rele(l->l_dbuf, NULL); 438214478Srpaulo} 439146773Ssam 440276788Sdelphij_NOTE(ARGSUSED(0)) 441146773Ssamstatic void 442146773Ssamzap_leaf_pageout(dmu_buf_t *db, void *vl) 443276788Sdelphij{ 444146773Ssam zap_leaf_t *l = vl; 445146773Ssam 446214478Srpaulo rw_destroy(&l->l_rwlock); 447276788Sdelphij kmem_free(l, sizeof (zap_leaf_t)); 448214478Srpaulo} 449276788Sdelphij 450146773Ssamstatic zap_leaf_t * 451146773Ssamzap_open_leaf(uint64_t blkid, dmu_buf_t *db) 452214478Srpaulo{ 453214478Srpaulo zap_leaf_t *l, *winner; 454146773Ssam 455146773Ssam ASSERT(blkid != 0); 456146773Ssam 457146773Ssam l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); 458276788Sdelphij rw_init(&l->l_rwlock, 0, 0, 0); 459146773Ssam rw_enter(&l->l_rwlock, RW_WRITER); 460146773Ssam l->l_blkid = blkid; 461146773Ssam l->l_bs = highbit(db->db_size)-1; 462146773Ssam l->l_dbuf = db; 463276788Sdelphij l->l_phys = NULL; 464146773Ssam 465146773Ssam winner = dmu_buf_set_user(db, l, &l->l_phys, zap_leaf_pageout); 466276788Sdelphij 467276788Sdelphij rw_exit(&l->l_rwlock); 468146773Ssam if (winner != NULL) { 469276788Sdelphij /* someone else set it first */ 470146773Ssam zap_leaf_pageout(NULL, l); 471276788Sdelphij l = winner; 472146773Ssam } 473146773Ssam 474146773Ssam /* 475276788Sdelphij * lhr_pad was previously used for the next leaf in the leaf 476146773Ssam * chain. There should be no chained leafs (as we have removed 477146773Ssam * support for them). 478146773Ssam */ 479146773Ssam ASSERT0(l->l_phys->l_hdr.lh_pad1); 480276788Sdelphij 481146773Ssam /* 482146773Ssam * There should be more hash entries than there can be 483276788Sdelphij * chunks to put in the hash table 484146773Ssam */ 485146773Ssam ASSERT3U(ZAP_LEAF_HASH_NUMENTRIES(l), >, ZAP_LEAF_NUMCHUNKS(l) / 3); 486276788Sdelphij 487146773Ssam /* The chunks should begin at the end of the hash table */ 488146773Ssam ASSERT3P(&ZAP_LEAF_CHUNK(l, 0), ==, 489276788Sdelphij &l->l_phys->l_hash[ZAP_LEAF_HASH_NUMENTRIES(l)]); 490146773Ssam 491276788Sdelphij /* The chunks should end at the end of the block */ 492146773Ssam ASSERT3U((uintptr_t)&ZAP_LEAF_CHUNK(l, ZAP_LEAF_NUMCHUNKS(l)) - 493146773Ssam (uintptr_t)l->l_phys, ==, l->l_dbuf->db_size); 494276788Sdelphij 495146773Ssam return (l); 496146773Ssam} 497146773Ssam 498276788Sdelphijstatic int 499146773Ssamzap_get_leaf_byblk(zap_t *zap, uint64_t blkid, dmu_tx_t *tx, krw_t lt, 500127668Sbms zap_leaf_t **lp) 501276788Sdelphij{ 502127668Sbms dmu_buf_t *db; 503127668Sbms zap_leaf_t *l; 504190207Srpaulo int bs = FZAP_BLOCK_SHIFT(zap); 505127668Sbms int err; 506127668Sbms 507127668Sbms ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 508127668Sbms 509214478Srpaulo err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 510276788Sdelphij blkid << bs, NULL, &db, DMU_READ_NO_PREFETCH); 511127668Sbms if (err) 512276788Sdelphij return (err); 513276788Sdelphij 514127668Sbms ASSERT3U(db->db_object, ==, zap->zap_object); 515127668Sbms ASSERT3U(db->db_offset, ==, blkid << bs); 516127668Sbms ASSERT3U(db->db_size, ==, 1 << bs); 517127668Sbms ASSERT(blkid != 0); 518127668Sbms 519276788Sdelphij l = dmu_buf_get_user(db); 520127668Sbms 521127668Sbms if (l == NULL) 522276788Sdelphij l = zap_open_leaf(blkid, db); 523127668Sbms 524127668Sbms rw_enter(&l->l_rwlock, lt); 525276788Sdelphij /* 526127668Sbms * Must lock before dirtying, otherwise l->l_phys could change, 527127668Sbms * causing ASSERT below to fail. 528127668Sbms */ 529127668Sbms if (lt == RW_WRITER) 530276788Sdelphij dmu_buf_will_dirty(db, tx); 531127668Sbms ASSERT3U(l->l_blkid, ==, blkid); 532127668Sbms ASSERT3P(l->l_dbuf, ==, db); 533276788Sdelphij ASSERT3P(l->l_phys, ==, l->l_dbuf->db_data); 534276788Sdelphij ASSERT3U(l->l_phys->l_hdr.lh_block_type, ==, ZBT_LEAF); 535172683Smlaier ASSERT3U(l->l_phys->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC); 536172683Smlaier 537172683Smlaier *lp = l; 538190207Srpaulo return (0); 539190207Srpaulo} 540111726Sfenner 541276788Sdelphijstatic int 542111726Sfennerzap_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t *valp) 543111726Sfenner{ 544111726Sfenner ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 545111726Sfenner 546276788Sdelphij if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) { 547276788Sdelphij ASSERT3U(idx, <, 548127668Sbms (1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift)); 549276788Sdelphij *valp = ZAP_EMBEDDED_PTRTBL_ENT(zap, idx); 550276788Sdelphij return (0); 551276788Sdelphij } else { 552276788Sdelphij return (zap_table_load(zap, &zap->zap_f.zap_phys->zap_ptrtbl, 553276788Sdelphij idx, valp)); 554276788Sdelphij } 555276788Sdelphij} 556276788Sdelphij 557276788Sdelphijstatic int 558276788Sdelphijzap_set_idx_to_blk(zap_t *zap, uint64_t idx, uint64_t blk, dmu_tx_t *tx) 559127668Sbms{ 560127668Sbms ASSERT(tx != NULL); 561276788Sdelphij ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 562127668Sbms 563127668Sbms if (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk == 0) { 564214478Srpaulo ZAP_EMBEDDED_PTRTBL_ENT(zap, idx) = blk; 565276788Sdelphij return (0); 566127668Sbms } else { 567127668Sbms return (zap_table_store(zap, &zap->zap_f.zap_phys->zap_ptrtbl, 568127668Sbms idx, blk, tx)); 569214478Srpaulo } 570276788Sdelphij} 571127668Sbms 572127668Sbmsstatic int 573214478Srpaulozap_deref_leaf(zap_t *zap, uint64_t h, dmu_tx_t *tx, krw_t lt, zap_leaf_t **lp) 574276788Sdelphij{ 575214478Srpaulo uint64_t idx, blk; 576214478Srpaulo int err; 577276788Sdelphij 578214478Srpaulo ASSERT(zap->zap_dbuf == NULL || 579127668Sbms zap->zap_f.zap_phys == zap->zap_dbuf->db_data); 580276788Sdelphij ASSERT3U(zap->zap_f.zap_phys->zap_magic, ==, ZAP_MAGIC); 581127668Sbms idx = ZAP_HASH_IDX(h, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift); 582276788Sdelphij err = zap_idx_to_blk(zap, idx, &blk); 583127668Sbms if (err != 0) 584190207Srpaulo return (err); 585190207Srpaulo err = zap_get_leaf_byblk(zap, blk, tx, lt, lp); 586276788Sdelphij 587190207Srpaulo ASSERT(err || ZAP_HASH_IDX(h, (*lp)->l_phys->l_hdr.lh_prefix_len) == 588190207Srpaulo (*lp)->l_phys->l_hdr.lh_prefix); 589214478Srpaulo return (err); 590276788Sdelphij} 591190207Srpaulo 592190207Srpaulostatic int 593276788Sdelphijzap_expand_leaf(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx, zap_leaf_t **lp) 594190207Srpaulo{ 595276788Sdelphij zap_t *zap = zn->zn_zap; 596190207Srpaulo uint64_t hash = zn->zn_hash; 597127668Sbms zap_leaf_t *nl; 598214478Srpaulo int prefix_diff, i, err; 599276788Sdelphij uint64_t sibling; 600127668Sbms int old_prefix_len = l->l_phys->l_hdr.lh_prefix_len; 601127668Sbms 602214478Srpaulo ASSERT3U(old_prefix_len, <=, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift); 603276788Sdelphij ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 604276788Sdelphij 605214478Srpaulo ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==, 606276788Sdelphij l->l_phys->l_hdr.lh_prefix); 607214478Srpaulo 608127668Sbms if (zap_tryupgradedir(zap, tx) == 0 || 609276788Sdelphij old_prefix_len == zap->zap_f.zap_phys->zap_ptrtbl.zt_shift) { 610127668Sbms /* We failed to upgrade, or need to grow the pointer table */ 611276788Sdelphij objset_t *os = zap->zap_objset; 612127668Sbms uint64_t object = zap->zap_object; 613146773Ssam 614190207Srpaulo zap_put_leaf(l); 615146773Ssam zap_unlockdir(zap); 616276788Sdelphij err = zap_lockdir(os, object, tx, RW_WRITER, 617146773Ssam FALSE, FALSE, &zn->zn_zap); 618146773Ssam zap = zn->zn_zap; 619214478Srpaulo if (err) 620276788Sdelphij return (err); 621146773Ssam ASSERT(!zap->zap_ismicro); 622190207Srpaulo 623190207Srpaulo while (old_prefix_len == 624276788Sdelphij zap->zap_f.zap_phys->zap_ptrtbl.zt_shift) { 625190207Srpaulo err = zap_grow_ptrtbl(zap, tx); 626190207Srpaulo if (err) 627190207Srpaulo return (err); 628276788Sdelphij } 629190207Srpaulo 630190207Srpaulo err = zap_deref_leaf(zap, hash, tx, RW_WRITER, &l); 631190207Srpaulo if (err) 632190207Srpaulo return (err); 633276788Sdelphij 634276788Sdelphij if (l->l_phys->l_hdr.lh_prefix_len != old_prefix_len) { 635190207Srpaulo /* it split while our locks were down */ 636190207Srpaulo *lp = l; 637190207Srpaulo return (0); 638276788Sdelphij } 639190207Srpaulo } 640190207Srpaulo ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); 641190207Srpaulo ASSERT3U(old_prefix_len, <, zap->zap_f.zap_phys->zap_ptrtbl.zt_shift); 642190207Srpaulo ASSERT3U(ZAP_HASH_IDX(hash, old_prefix_len), ==, 643190207Srpaulo l->l_phys->l_hdr.lh_prefix); 644276788Sdelphij 645190207Srpaulo prefix_diff = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift - 646190207Srpaulo (old_prefix_len + 1); 647276788Sdelphij sibling = (ZAP_HASH_IDX(hash, old_prefix_len + 1) | 1) << prefix_diff; 648190207Srpaulo 649190207Srpaulo /* check for i/o errors before doing zap_leaf_split */ 650276788Sdelphij for (i = 0; i < (1ULL<<prefix_diff); i++) { 651190207Srpaulo uint64_t blk; 652190207Srpaulo err = zap_idx_to_blk(zap, sibling+i, &blk); 653276788Sdelphij if (err) 654190207Srpaulo return (err); 655190207Srpaulo ASSERT3U(blk, ==, l->l_blkid); 656276788Sdelphij } 657190207Srpaulo 658276788Sdelphij nl = zap_create_leaf(zap, tx); 659190207Srpaulo zap_leaf_split(l, nl, zap->zap_normflags != 0); 660190207Srpaulo 661214478Srpaulo /* set sibling pointers */ 662190207Srpaulo for (i = 0; i < (1ULL << prefix_diff); i++) { 663190207Srpaulo err = zap_set_idx_to_blk(zap, sibling+i, nl->l_blkid, tx); 664276788Sdelphij ASSERT0(err); /* we checked for i/o errors above */ 665190207Srpaulo } 666276788Sdelphij 667190207Srpaulo if (hash & (1ULL << (64 - l->l_phys->l_hdr.lh_prefix_len))) { 668190207Srpaulo /* we want the sibling */ 669190207Srpaulo zap_put_leaf(l); 670276788Sdelphij *lp = nl; 671190207Srpaulo } else { 672190207Srpaulo zap_put_leaf(nl); 673190207Srpaulo *lp = l; 674276788Sdelphij } 675190207Srpaulo 676190207Srpaulo return (0); 677190207Srpaulo} 678190207Srpaulo 679276788Sdelphijstatic void 680276788Sdelphijzap_put_leaf_maybe_grow_ptrtbl(zap_name_t *zn, zap_leaf_t *l, dmu_tx_t *tx) 681190207Srpaulo{ 682276788Sdelphij zap_t *zap = zn->zn_zap; 683276788Sdelphij int shift = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift; 684276788Sdelphij int leaffull = (l->l_phys->l_hdr.lh_prefix_len == shift && 685276788Sdelphij l->l_phys->l_hdr.lh_nfree < ZAP_LEAF_LOW_WATER); 686276788Sdelphij 687276788Sdelphij zap_put_leaf(l); 688276788Sdelphij 689276788Sdelphij if (leaffull || zap->zap_f.zap_phys->zap_ptrtbl.zt_nextblk) { 690276788Sdelphij int err; 691276788Sdelphij 692276788Sdelphij /* 693276788Sdelphij * We are in the middle of growing the pointer table, or 694276788Sdelphij * this leaf will soon make us grow it. 695276788Sdelphij */ 696276788Sdelphij if (zap_tryupgradedir(zap, tx) == 0) { 697276788Sdelphij objset_t *os = zap->zap_objset; 698276788Sdelphij uint64_t zapobj = zap->zap_object; 699276788Sdelphij 700276788Sdelphij zap_unlockdir(zap); 701276788Sdelphij err = zap_lockdir(os, zapobj, tx, 702276788Sdelphij RW_WRITER, FALSE, FALSE, &zn->zn_zap); 703276788Sdelphij zap = zn->zn_zap; 704276788Sdelphij if (err) 705276788Sdelphij return; 706276788Sdelphij } 707276788Sdelphij 708276788Sdelphij /* could have finished growing while our locks were down */ 709276788Sdelphij if (zap->zap_f.zap_phys->zap_ptrtbl.zt_shift == shift) 710276788Sdelphij (void) zap_grow_ptrtbl(zap, tx); 711276788Sdelphij } 712276788Sdelphij} 713276788Sdelphij 714276788Sdelphijstatic int 715276788Sdelphijfzap_checkname(zap_name_t *zn) 716276788Sdelphij{ 717276788Sdelphij if (zn->zn_key_orig_numints * zn->zn_key_intlen > ZAP_MAXNAMELEN) 718251158Sdelphij return (SET_ERROR(ENAMETOOLONG)); 719251158Sdelphij return (0); 720276788Sdelphij} 721251158Sdelphij 722251158Sdelphijstatic int 723251158Sdelphijfzap_checksize(uint64_t integer_size, uint64_t num_integers) 724276788Sdelphij{ 725276788Sdelphij /* Only integer sizes supported by C */ 726251158Sdelphij switch (integer_size) { 727251158Sdelphij case 1: 728251158Sdelphij case 2: 729251158Sdelphij case 4: 730276788Sdelphij case 8: 731251158Sdelphij break; 732251158Sdelphij default: 733276788Sdelphij return (SET_ERROR(EINVAL)); 734251158Sdelphij } 735276788Sdelphij 736251158Sdelphij if (integer_size * num_integers > ZAP_MAXVALUELEN) 737251158Sdelphij return (E2BIG); 738251158Sdelphij 739276788Sdelphij return (0); 740251158Sdelphij} 74156893Sfenner 742276788Sdelphijstatic int 74356893Sfennerfzap_check(zap_name_t *zn, uint64_t integer_size, uint64_t num_integers) 74456893Sfenner{ 74556893Sfenner int err; 746111726Sfenner 74756893Sfenner if ((err = fzap_checkname(zn)) != 0) 74856893Sfenner return (err); 74956893Sfenner return (fzap_checksize(integer_size, num_integers)); 75056893Sfenner} 751276788Sdelphij 75256893Sfenner/* 75356893Sfenner * Routines for manipulating attributes. 75456893Sfenner */ 755111726Sfennerint 75656893Sfennerfzap_lookup(zap_name_t *zn, 75756893Sfenner uint64_t integer_size, uint64_t num_integers, void *buf, 758276788Sdelphij char *realname, int rn_len, boolean_t *ncp) 759276788Sdelphij{ 76056893Sfenner zap_leaf_t *l; 761111726Sfenner int err; 762127668Sbms zap_entry_handle_t zeh; 763127668Sbms 76456893Sfenner if ((err = fzap_checkname(zn)) != 0) 765111726Sfenner return (err); 76656893Sfenner 767276788Sdelphij err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, NULL, RW_READER, &l); 76856893Sfenner if (err != 0) 769276788Sdelphij return (err); 770127668Sbms err = zap_leaf_lookup(l, zn, &zeh); 771127668Sbms if (err == 0) { 77256893Sfenner if ((err = fzap_checksize(integer_size, num_integers)) != 0) { 773111726Sfenner zap_put_leaf(l); 774127668Sbms return (err); 775276788Sdelphij } 776276788Sdelphij 777111726Sfenner err = zap_entry_read(&zeh, integer_size, num_integers, buf); 778276788Sdelphij (void) zap_entry_read_name(zn->zn_zap, &zeh, rn_len, realname); 779276788Sdelphij if (ncp) { 780111726Sfenner *ncp = zap_entry_normalization_conflict(&zeh, 781111726Sfenner zn, NULL, zn->zn_zap); 782111726Sfenner } 783111726Sfenner } 784111726Sfenner 785276788Sdelphij zap_put_leaf(l); 786127668Sbms return (err); 787127668Sbms} 788276788Sdelphij 789127668Sbmsint 790276788Sdelphijfzap_add_cd(zap_name_t *zn, 791127668Sbms uint64_t integer_size, uint64_t num_integers, 792127668Sbms const void *val, uint32_t cd, dmu_tx_t *tx) 793127668Sbms{ 794276788Sdelphij zap_leaf_t *l; 795127668Sbms int err; 796127668Sbms zap_entry_handle_t zeh; 797276788Sdelphij zap_t *zap = zn->zn_zap; 798127668Sbms 799127668Sbms ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 800276788Sdelphij ASSERT(!zap->zap_ismicro); 801127668Sbms ASSERT(fzap_check(zn, integer_size, num_integers) == 0); 802276788Sdelphij 803127668Sbms err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l); 804111726Sfenner if (err != 0) 805276788Sdelphij return (err); 80656893Sfennerretry: 80756893Sfenner err = zap_leaf_lookup(l, zn, &zeh); 80856893Sfenner if (err == 0) { 809276788Sdelphij err = SET_ERROR(EEXIST); 81056893Sfenner goto out; 811 } 812 if (err != ENOENT) 813 goto out; 814 815 err = zap_entry_create(l, zn, cd, 816 integer_size, num_integers, val, &zeh); 817 818 if (err == 0) { 819 zap_increment_num_entries(zap, 1, tx); 820 } else if (err == EAGAIN) { 821 err = zap_expand_leaf(zn, l, tx, &l); 822 zap = zn->zn_zap; /* zap_expand_leaf() may change zap */ 823 if (err == 0) 824 goto retry; 825 } 826 827out: 828 if (zap != NULL) 829 zap_put_leaf_maybe_grow_ptrtbl(zn, l, tx); 830 return (err); 831} 832 833int 834fzap_add(zap_name_t *zn, 835 uint64_t integer_size, uint64_t num_integers, 836 const void *val, dmu_tx_t *tx) 837{ 838 int err = fzap_check(zn, integer_size, num_integers); 839 if (err != 0) 840 return (err); 841 842 return (fzap_add_cd(zn, integer_size, num_integers, 843 val, ZAP_NEED_CD, tx)); 844} 845 846int 847fzap_update(zap_name_t *zn, 848 int integer_size, uint64_t num_integers, const void *val, dmu_tx_t *tx) 849{ 850 zap_leaf_t *l; 851 int err, create; 852 zap_entry_handle_t zeh; 853 zap_t *zap = zn->zn_zap; 854 855 ASSERT(RW_LOCK_HELD(&zap->zap_rwlock)); 856 err = fzap_check(zn, integer_size, num_integers); 857 if (err != 0) 858 return (err); 859 860 err = zap_deref_leaf(zap, zn->zn_hash, tx, RW_WRITER, &l); 861 if (err != 0) 862 return (err); 863retry: 864 err = zap_leaf_lookup(l, zn, &zeh); 865 create = (err == ENOENT); 866 ASSERT(err == 0 || err == ENOENT); 867 868 if (create) { 869 err = zap_entry_create(l, zn, ZAP_NEED_CD, 870 integer_size, num_integers, val, &zeh); 871 if (err == 0) 872 zap_increment_num_entries(zap, 1, tx); 873 } else { 874 err = zap_entry_update(&zeh, integer_size, num_integers, val); 875 } 876 877 if (err == EAGAIN) { 878 err = zap_expand_leaf(zn, l, tx, &l); 879 zap = zn->zn_zap; /* zap_expand_leaf() may change zap */ 880 if (err == 0) 881 goto retry; 882 } 883 884 if (zap != NULL) 885 zap_put_leaf_maybe_grow_ptrtbl(zn, l, tx); 886 return (err); 887} 888 889int 890fzap_length(zap_name_t *zn, 891 uint64_t *integer_size, uint64_t *num_integers) 892{ 893 zap_leaf_t *l; 894 int err; 895 zap_entry_handle_t zeh; 896 897 err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, NULL, RW_READER, &l); 898 if (err != 0) 899 return (err); 900 err = zap_leaf_lookup(l, zn, &zeh); 901 if (err != 0) 902 goto out; 903 904 if (integer_size) 905 *integer_size = zeh.zeh_integer_size; 906 if (num_integers) 907 *num_integers = zeh.zeh_num_integers; 908out: 909 zap_put_leaf(l); 910 return (err); 911} 912 913int 914fzap_remove(zap_name_t *zn, dmu_tx_t *tx) 915{ 916 zap_leaf_t *l; 917 int err; 918 zap_entry_handle_t zeh; 919 920 err = zap_deref_leaf(zn->zn_zap, zn->zn_hash, tx, RW_WRITER, &l); 921 if (err != 0) 922 return (err); 923 err = zap_leaf_lookup(l, zn, &zeh); 924 if (err == 0) { 925 zap_entry_remove(&zeh); 926 zap_increment_num_entries(zn->zn_zap, -1, tx); 927 } 928 zap_put_leaf(l); 929 return (err); 930} 931 932void 933fzap_prefetch(zap_name_t *zn) 934{ 935 uint64_t idx, blk; 936 zap_t *zap = zn->zn_zap; 937 int bs; 938 939 idx = ZAP_HASH_IDX(zn->zn_hash, 940 zap->zap_f.zap_phys->zap_ptrtbl.zt_shift); 941 if (zap_idx_to_blk(zap, idx, &blk) != 0) 942 return; 943 bs = FZAP_BLOCK_SHIFT(zap); 944 dmu_prefetch(zap->zap_objset, zap->zap_object, blk << bs, 1 << bs); 945} 946 947/* 948 * Helper functions for consumers. 949 */ 950 951uint64_t 952zap_create_link(objset_t *os, dmu_object_type_t ot, uint64_t parent_obj, 953 const char *name, dmu_tx_t *tx) 954{ 955 uint64_t new_obj; 956 957 VERIFY((new_obj = zap_create(os, ot, DMU_OT_NONE, 0, tx)) > 0); 958 VERIFY(zap_add(os, parent_obj, name, sizeof (uint64_t), 1, &new_obj, 959 tx) == 0); 960 961 return (new_obj); 962} 963 964int 965zap_value_search(objset_t *os, uint64_t zapobj, uint64_t value, uint64_t mask, 966 char *name) 967{ 968 zap_cursor_t zc; 969 zap_attribute_t *za; 970 int err; 971 972 if (mask == 0) 973 mask = -1ULL; 974 975 za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); 976 for (zap_cursor_init(&zc, os, zapobj); 977 (err = zap_cursor_retrieve(&zc, za)) == 0; 978 zap_cursor_advance(&zc)) { 979 if ((za->za_first_integer & mask) == (value & mask)) { 980 (void) strcpy(name, za->za_name); 981 break; 982 } 983 } 984 zap_cursor_fini(&zc); 985 kmem_free(za, sizeof (zap_attribute_t)); 986 return (err); 987} 988 989int 990zap_join(objset_t *os, uint64_t fromobj, uint64_t intoobj, dmu_tx_t *tx) 991{ 992 zap_cursor_t zc; 993 zap_attribute_t za; 994 int err; 995 996 err = 0; 997 for (zap_cursor_init(&zc, os, fromobj); 998 zap_cursor_retrieve(&zc, &za) == 0; 999 (void) zap_cursor_advance(&zc)) { 1000 if (za.za_integer_length != 8 || za.za_num_integers != 1) { 1001 err = SET_ERROR(EINVAL); 1002 break; 1003 } 1004 err = zap_add(os, intoobj, za.za_name, 1005 8, 1, &za.za_first_integer, tx); 1006 if (err) 1007 break; 1008 } 1009 zap_cursor_fini(&zc); 1010 return (err); 1011} 1012 1013int 1014zap_join_key(objset_t *os, uint64_t fromobj, uint64_t intoobj, 1015 uint64_t value, dmu_tx_t *tx) 1016{ 1017 zap_cursor_t zc; 1018 zap_attribute_t za; 1019 int err; 1020 1021 err = 0; 1022 for (zap_cursor_init(&zc, os, fromobj); 1023 zap_cursor_retrieve(&zc, &za) == 0; 1024 (void) zap_cursor_advance(&zc)) { 1025 if (za.za_integer_length != 8 || za.za_num_integers != 1) { 1026 err = SET_ERROR(EINVAL); 1027 break; 1028 } 1029 err = zap_add(os, intoobj, za.za_name, 1030 8, 1, &value, tx); 1031 if (err) 1032 break; 1033 } 1034 zap_cursor_fini(&zc); 1035 return (err); 1036} 1037 1038int 1039zap_join_increment(objset_t *os, uint64_t fromobj, uint64_t intoobj, 1040 dmu_tx_t *tx) 1041{ 1042 zap_cursor_t zc; 1043 zap_attribute_t za; 1044 int err; 1045 1046 err = 0; 1047 for (zap_cursor_init(&zc, os, fromobj); 1048 zap_cursor_retrieve(&zc, &za) == 0; 1049 (void) zap_cursor_advance(&zc)) { 1050 uint64_t delta = 0; 1051 1052 if (za.za_integer_length != 8 || za.za_num_integers != 1) { 1053 err = SET_ERROR(EINVAL); 1054 break; 1055 } 1056 1057 err = zap_lookup(os, intoobj, za.za_name, 8, 1, &delta); 1058 if (err != 0 && err != ENOENT) 1059 break; 1060 delta += za.za_first_integer; 1061 err = zap_update(os, intoobj, za.za_name, 8, 1, &delta, tx); 1062 if (err) 1063 break; 1064 } 1065 zap_cursor_fini(&zc); 1066 return (err); 1067} 1068 1069int 1070zap_add_int(objset_t *os, uint64_t obj, uint64_t value, dmu_tx_t *tx) 1071{ 1072 char name[20]; 1073 1074 (void) snprintf(name, sizeof (name), "%llx", (longlong_t)value); 1075 return (zap_add(os, obj, name, 8, 1, &value, tx)); 1076} 1077 1078int 1079zap_remove_int(objset_t *os, uint64_t obj, uint64_t value, dmu_tx_t *tx) 1080{ 1081 char name[20]; 1082 1083 (void) snprintf(name, sizeof (name), "%llx", (longlong_t)value); 1084 return (zap_remove(os, obj, name, tx)); 1085} 1086 1087int 1088zap_lookup_int(objset_t *os, uint64_t obj, uint64_t value) 1089{ 1090 char name[20]; 1091 1092 (void) snprintf(name, sizeof (name), "%llx", (longlong_t)value); 1093 return (zap_lookup(os, obj, name, 8, 1, &value)); 1094} 1095 1096int 1097zap_add_int_key(objset_t *os, uint64_t obj, 1098 uint64_t key, uint64_t value, dmu_tx_t *tx) 1099{ 1100 char name[20]; 1101 1102 (void) snprintf(name, sizeof (name), "%llx", (longlong_t)key); 1103 return (zap_add(os, obj, name, 8, 1, &value, tx)); 1104} 1105 1106int 1107zap_update_int_key(objset_t *os, uint64_t obj, 1108 uint64_t key, uint64_t value, dmu_tx_t *tx) 1109{ 1110 char name[20]; 1111 1112 (void) snprintf(name, sizeof (name), "%llx", (longlong_t)key); 1113 return (zap_update(os, obj, name, 8, 1, &value, tx)); 1114} 1115 1116int 1117zap_lookup_int_key(objset_t *os, uint64_t obj, uint64_t key, uint64_t *valuep) 1118{ 1119 char name[20]; 1120 1121 (void) snprintf(name, sizeof (name), "%llx", (longlong_t)key); 1122 return (zap_lookup(os, obj, name, 8, 1, valuep)); 1123} 1124 1125int 1126zap_increment(objset_t *os, uint64_t obj, const char *name, int64_t delta, 1127 dmu_tx_t *tx) 1128{ 1129 uint64_t value = 0; 1130 int err; 1131 1132 if (delta == 0) 1133 return (0); 1134 1135 err = zap_lookup(os, obj, name, 8, 1, &value); 1136 if (err != 0 && err != ENOENT) 1137 return (err); 1138 value += delta; 1139 if (value == 0) 1140 err = zap_remove(os, obj, name, tx); 1141 else 1142 err = zap_update(os, obj, name, 8, 1, &value, tx); 1143 return (err); 1144} 1145 1146int 1147zap_increment_int(objset_t *os, uint64_t obj, uint64_t key, int64_t delta, 1148 dmu_tx_t *tx) 1149{ 1150 char name[20]; 1151 1152 (void) snprintf(name, sizeof (name), "%llx", (longlong_t)key); 1153 return (zap_increment(os, obj, name, delta, tx)); 1154} 1155 1156/* 1157 * Routines for iterating over the attributes. 1158 */ 1159 1160int 1161fzap_cursor_retrieve(zap_t *zap, zap_cursor_t *zc, zap_attribute_t *za) 1162{ 1163 int err = ENOENT; 1164 zap_entry_handle_t zeh; 1165 zap_leaf_t *l; 1166 1167 /* retrieve the next entry at or after zc_hash/zc_cd */ 1168 /* if no entry, return ENOENT */ 1169 1170 if (zc->zc_leaf && 1171 (ZAP_HASH_IDX(zc->zc_hash, 1172 zc->zc_leaf->l_phys->l_hdr.lh_prefix_len) != 1173 zc->zc_leaf->l_phys->l_hdr.lh_prefix)) { 1174 rw_enter(&zc->zc_leaf->l_rwlock, RW_READER); 1175 zap_put_leaf(zc->zc_leaf); 1176 zc->zc_leaf = NULL; 1177 } 1178 1179again: 1180 if (zc->zc_leaf == NULL) { 1181 err = zap_deref_leaf(zap, zc->zc_hash, NULL, RW_READER, 1182 &zc->zc_leaf); 1183 if (err != 0) 1184 return (err); 1185 } else { 1186 rw_enter(&zc->zc_leaf->l_rwlock, RW_READER); 1187 } 1188 l = zc->zc_leaf; 1189 1190 err = zap_leaf_lookup_closest(l, zc->zc_hash, zc->zc_cd, &zeh); 1191 1192 if (err == ENOENT) { 1193 uint64_t nocare = 1194 (1ULL << (64 - l->l_phys->l_hdr.lh_prefix_len)) - 1; 1195 zc->zc_hash = (zc->zc_hash & ~nocare) + nocare + 1; 1196 zc->zc_cd = 0; 1197 if (l->l_phys->l_hdr.lh_prefix_len == 0 || zc->zc_hash == 0) { 1198 zc->zc_hash = -1ULL; 1199 } else { 1200 zap_put_leaf(zc->zc_leaf); 1201 zc->zc_leaf = NULL; 1202 goto again; 1203 } 1204 } 1205 1206 if (err == 0) { 1207 zc->zc_hash = zeh.zeh_hash; 1208 zc->zc_cd = zeh.zeh_cd; 1209 za->za_integer_length = zeh.zeh_integer_size; 1210 za->za_num_integers = zeh.zeh_num_integers; 1211 if (zeh.zeh_num_integers == 0) { 1212 za->za_first_integer = 0; 1213 } else { 1214 err = zap_entry_read(&zeh, 8, 1, &za->za_first_integer); 1215 ASSERT(err == 0 || err == EOVERFLOW); 1216 } 1217 err = zap_entry_read_name(zap, &zeh, 1218 sizeof (za->za_name), za->za_name); 1219 ASSERT(err == 0); 1220 1221 za->za_normalization_conflict = 1222 zap_entry_normalization_conflict(&zeh, 1223 NULL, za->za_name, zap); 1224 } 1225 rw_exit(&zc->zc_leaf->l_rwlock); 1226 return (err); 1227} 1228 1229static void 1230zap_stats_ptrtbl(zap_t *zap, uint64_t *tbl, int len, zap_stats_t *zs) 1231{ 1232 int i, err; 1233 uint64_t lastblk = 0; 1234 1235 /* 1236 * NB: if a leaf has more pointers than an entire ptrtbl block 1237 * can hold, then it'll be accounted for more than once, since 1238 * we won't have lastblk. 1239 */ 1240 for (i = 0; i < len; i++) { 1241 zap_leaf_t *l; 1242 1243 if (tbl[i] == lastblk) 1244 continue; 1245 lastblk = tbl[i]; 1246 1247 err = zap_get_leaf_byblk(zap, tbl[i], NULL, RW_READER, &l); 1248 if (err == 0) { 1249 zap_leaf_stats(zap, l, zs); 1250 zap_put_leaf(l); 1251 } 1252 } 1253} 1254 1255int 1256fzap_cursor_move_to_key(zap_cursor_t *zc, zap_name_t *zn) 1257{ 1258 int err; 1259 zap_leaf_t *l; 1260 zap_entry_handle_t zeh; 1261 1262 if (zn->zn_key_orig_numints * zn->zn_key_intlen > ZAP_MAXNAMELEN) 1263 return (SET_ERROR(ENAMETOOLONG)); 1264 1265 err = zap_deref_leaf(zc->zc_zap, zn->zn_hash, NULL, RW_READER, &l); 1266 if (err != 0) 1267 return (err); 1268 1269 err = zap_leaf_lookup(l, zn, &zeh); 1270 if (err != 0) 1271 return (err); 1272 1273 zc->zc_leaf = l; 1274 zc->zc_hash = zeh.zeh_hash; 1275 zc->zc_cd = zeh.zeh_cd; 1276 1277 return (err); 1278} 1279 1280void 1281fzap_get_stats(zap_t *zap, zap_stats_t *zs) 1282{ 1283 int bs = FZAP_BLOCK_SHIFT(zap); 1284 zs->zs_blocksize = 1ULL << bs; 1285 1286 /* 1287 * Set zap_phys_t fields 1288 */ 1289 zs->zs_num_leafs = zap->zap_f.zap_phys->zap_num_leafs; 1290 zs->zs_num_entries = zap->zap_f.zap_phys->zap_num_entries; 1291 zs->zs_num_blocks = zap->zap_f.zap_phys->zap_freeblk; 1292 zs->zs_block_type = zap->zap_f.zap_phys->zap_block_type; 1293 zs->zs_magic = zap->zap_f.zap_phys->zap_magic; 1294 zs->zs_salt = zap->zap_f.zap_phys->zap_salt; 1295 1296 /* 1297 * Set zap_ptrtbl fields 1298 */ 1299 zs->zs_ptrtbl_len = 1ULL << zap->zap_f.zap_phys->zap_ptrtbl.zt_shift; 1300 zs->zs_ptrtbl_nextblk = zap->zap_f.zap_phys->zap_ptrtbl.zt_nextblk; 1301 zs->zs_ptrtbl_blks_copied = 1302 zap->zap_f.zap_phys->zap_ptrtbl.zt_blks_copied; 1303 zs->zs_ptrtbl_zt_blk = zap->zap_f.zap_phys->zap_ptrtbl.zt_blk; 1304 zs->zs_ptrtbl_zt_numblks = zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks; 1305 zs->zs_ptrtbl_zt_shift = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift; 1306 1307 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks == 0) { 1308 /* the ptrtbl is entirely in the header block. */ 1309 zap_stats_ptrtbl(zap, &ZAP_EMBEDDED_PTRTBL_ENT(zap, 0), 1310 1 << ZAP_EMBEDDED_PTRTBL_SHIFT(zap), zs); 1311 } else { 1312 int b; 1313 1314 dmu_prefetch(zap->zap_objset, zap->zap_object, 1315 zap->zap_f.zap_phys->zap_ptrtbl.zt_blk << bs, 1316 zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks << bs); 1317 1318 for (b = 0; b < zap->zap_f.zap_phys->zap_ptrtbl.zt_numblks; 1319 b++) { 1320 dmu_buf_t *db; 1321 int err; 1322 1323 err = dmu_buf_hold(zap->zap_objset, zap->zap_object, 1324 (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk + b) << bs, 1325 FTAG, &db, DMU_READ_NO_PREFETCH); 1326 if (err == 0) { 1327 zap_stats_ptrtbl(zap, db->db_data, 1328 1<<(bs-3), zs); 1329 dmu_buf_rele(db, FTAG); 1330 } 1331 } 1332 } 1333} 1334 1335int 1336fzap_count_write(zap_name_t *zn, int add, uint64_t *towrite, 1337 uint64_t *tooverwrite) 1338{ 1339 zap_t *zap = zn->zn_zap; 1340 zap_leaf_t *l; 1341 int err; 1342 1343 /* 1344 * Account for the header block of the fatzap. 1345 */ 1346 if (!add && dmu_buf_freeable(zap->zap_dbuf)) { 1347 *tooverwrite += zap->zap_dbuf->db_size; 1348 } else { 1349 *towrite += zap->zap_dbuf->db_size; 1350 } 1351 1352 /* 1353 * Account for the pointer table blocks. 1354 * If we are adding we need to account for the following cases : 1355 * - If the pointer table is embedded, this operation could force an 1356 * external pointer table. 1357 * - If this already has an external pointer table this operation 1358 * could extend the table. 1359 */ 1360 if (add) { 1361 if (zap->zap_f.zap_phys->zap_ptrtbl.zt_blk == 0) 1362 *towrite += zap->zap_dbuf->db_size; 1363 else 1364 *towrite += (zap->zap_dbuf->db_size * 3); 1365 } 1366 1367 /* 1368 * Now, check if the block containing leaf is freeable 1369 * and account accordingly. 1370 */ 1371 err = zap_deref_leaf(zap, zn->zn_hash, NULL, RW_READER, &l); 1372 if (err != 0) { 1373 return (err); 1374 } 1375 1376 if (!add && dmu_buf_freeable(l->l_dbuf)) { 1377 *tooverwrite += l->l_dbuf->db_size; 1378 } else { 1379 /* 1380 * If this an add operation, the leaf block could split. 1381 * Hence, we need to account for an additional leaf block. 1382 */ 1383 *towrite += (add ? 2 : 1) * l->l_dbuf->db_size; 1384 } 1385 1386 zap_put_leaf(l); 1387 return (0); 1388} 1389