Lines Matching refs:db
54 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
192 dmu_buf_impl_t *db = vdb;
193 bzero(db, sizeof (dmu_buf_impl_t));
195 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
196 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
197 multilist_link_init(&db->db_cache_link);
198 refcount_create(&db->db_holds);
207 dmu_buf_impl_t *db = vdb;
208 mutex_destroy(&db->db_mtx);
209 cv_destroy(&db->db_changed);
210 ASSERT(!multilist_link_active(&db->db_cache_link));
211 refcount_destroy(&db->db_holds);
232 ((dbuf)->db.db_object == (obj) && \
243 dmu_buf_impl_t *db;
246 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
247 if (DBUF_EQUAL(db, os, obj, level, blkid)) {
248 mutex_enter(&db->db_mtx);
249 if (db->db_state != DB_EVICTING) {
251 return (db);
253 mutex_exit(&db->db_mtx);
264 dmu_buf_impl_t *db = NULL;
269 db = dn->dn_bonus;
270 mutex_enter(&db->db_mtx);
275 return (db);
285 dbuf_hash_insert(dmu_buf_impl_t *db)
288 objset_t *os = db->db_objset;
289 uint64_t obj = db->db.db_object;
290 int level = db->db_level;
291 uint64_t blkid = db->db_blkid;
308 mutex_enter(&db->db_mtx);
309 db->db_hash_next = h->hash_table[idx];
310 h->hash_table[idx] = db;
321 dbuf_hash_remove(dmu_buf_impl_t *db)
324 uint64_t hv = dbuf_hash(db->db_objset, db->db.db_object,
325 db->db_level, db->db_blkid);
333 ASSERT(refcount_is_zero(&db->db_holds));
334 ASSERT(db->db_state == DB_EVICTING);
335 ASSERT(!MUTEX_HELD(&db->db_mtx));
339 while ((dbf = *dbp) != db) {
343 *dbp = db->db_hash_next;
344 db->db_hash_next = NULL;
355 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
360 if (db->db_user == NULL)
364 ASSERT(db->db_level == 0);
367 ASSERT(db->db.db_data != NULL);
368 ASSERT3U(db->db_state, ==, DB_CACHED);
370 holds = refcount_count(&db->db_holds);
381 ASSERT3U(holds, >=, db->db_dirtycnt);
383 if (db->db_user_immediate_evict == TRUE)
384 ASSERT3U(holds, >=, db->db_dirtycnt);
392 dbuf_evict_user(dmu_buf_impl_t *db)
394 dmu_buf_user_t *dbu = db->db_user;
396 ASSERT(MUTEX_HELD(&db->db_mtx));
401 dbuf_verify_user(db, DBVU_EVICTING);
402 db->db_user = NULL;
431 dbuf_is_metadata(dmu_buf_impl_t *db)
433 if (db->db_level > 0) {
438 DB_DNODE_ENTER(db);
439 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
440 DB_DNODE_EXIT(db);
452 dbuf_include_in_metadata_cache(dmu_buf_impl_t *db)
454 DB_DNODE_ENTER(db);
455 dmu_object_type_t type = DB_DNODE(db)->dn_type;
456 DB_DNODE_EXIT(db);
471 dmu_buf_impl_t *, db);
491 dmu_buf_impl_t *db = obj;
505 return (dbuf_hash(db->db_objset, db->db.db_object,
506 db->db_level, db->db_blkid) %
542 dmu_buf_impl_t *db = multilist_sublist_tail(mls);
543 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) {
544 db = multilist_sublist_prev(mls, db);
547 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db,
550 if (db != NULL) {
551 multilist_sublist_remove(mls, db);
554 db->db.db_size, db);
555 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE);
556 db->db_caching_status = DB_NO_CACHE;
557 dbuf_destroy(db);
732 dbuf_verify(dmu_buf_impl_t *db)
737 ASSERT(MUTEX_HELD(&db->db_mtx));
742 ASSERT(db->db_objset != NULL);
743 DB_DNODE_ENTER(db);
744 dn = DB_DNODE(db);
746 ASSERT(db->db_parent == NULL);
747 ASSERT(db->db_blkptr == NULL);
749 ASSERT3U(db->db.db_object, ==, dn->dn_object);
750 ASSERT3P(db->db_objset, ==, dn->dn_objset);
751 ASSERT3U(db->db_level, <, dn->dn_nlevels);
752 ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
753 db->db_blkid == DMU_SPILL_BLKID ||
756 if (db->db_blkid == DMU_BONUS_BLKID) {
758 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
759 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
760 } else if (db->db_blkid == DMU_SPILL_BLKID) {
762 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
763 ASSERT0(db->db.db_offset);
765 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
768 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next)
769 ASSERT(dr->dr_dbuf == db);
771 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next)
772 ASSERT(dr->dr_dbuf == db);
779 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
780 dr = db->db_data_pending;
785 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
788 /* verify db->db_blkptr */
789 if (db->db_blkptr) {
790 if (db->db_parent == dn->dn_dbuf) {
791 /* db is pointed to by the dnode */
792 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
793 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
794 ASSERT(db->db_parent == NULL);
796 ASSERT(db->db_parent != NULL);
797 if (db->db_blkid != DMU_SPILL_BLKID)
798 ASSERT3P(db->db_blkptr, ==,
799 &dn->dn_phys->dn_blkptr[db->db_blkid]);
801 /* db is pointed to by an indirect block */
802 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
803 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
804 ASSERT3U(db->db_parent->db.db_object, ==,
805 db->db.db_object);
812 ASSERT3P(db->db_blkptr, ==,
813 ((blkptr_t *)db->db_parent->db.db_data +
814 db->db_blkid % epb));
818 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
819 (db->db_buf == NULL || db->db_buf->b_data) &&
820 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
821 db->db_state != DB_FILL && !dn->dn_free_txg) {
833 if (db->db_dirtycnt == 0) {
834 if (db->db_level == 0) {
835 uint64_t *buf = db->db.db_data;
838 for (i = 0; i < db->db.db_size >> 3; i++) {
842 blkptr_t *bps = db->db.db_data;
843 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==,
844 db->db.db_size);
853 i < db->db.db_size / sizeof (blkptr_t);
872 DB_DNODE_EXIT(db);
877 dbuf_clear_data(dmu_buf_impl_t *db)
879 ASSERT(MUTEX_HELD(&db->db_mtx));
880 dbuf_evict_user(db);
881 ASSERT3P(db->db_buf, ==, NULL);
882 db->db.db_data = NULL;
883 if (db->db_state != DB_NOFILL)
884 db->db_state = DB_UNCACHED;
888 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
890 ASSERT(MUTEX_HELD(&db->db_mtx));
893 db->db_buf = buf;
895 db->db.db_data = buf->b_data;
902 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
906 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
907 mutex_enter(&db->db_mtx);
908 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
909 int blksz = db->db.db_size;
910 spa_t *spa = db->db_objset->os_spa;
912 mutex_exit(&db->db_mtx);
914 bcopy(db->db.db_data, abuf->b_data, blksz);
916 abuf = db->db_buf;
917 arc_loan_inuse_buf(abuf, db);
918 db->db_buf = NULL;
919 dbuf_clear_data(db);
920 mutex_exit(&db->db_mtx);
964 dmu_buf_impl_t *db = vdb;
966 mutex_enter(&db->db_mtx);
967 ASSERT3U(db->db_state, ==, DB_READ);
971 ASSERT(refcount_count(&db->db_holds) > 0);
972 ASSERT(db->db_buf == NULL);
973 ASSERT(db->db.db_data == NULL);
977 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
978 ASSERT3P(db->db_buf, ==, NULL);
979 db->db_state = DB_UNCACHED;
980 } else if (db->db_level == 0 && db->db_freed_in_flight) {
984 buf = arc_alloc_buf(db->db_objset->os_spa,
985 db, DBUF_GET_BUFC_TYPE(db), db->db.db_size);
987 arc_release(buf, db);
988 bzero(buf->b_data, db->db.db_size);
990 db->db_freed_in_flight = FALSE;
991 dbuf_set_data(db, buf);
992 db->db_state = DB_CACHED;
996 dbuf_set_data(db, buf);
997 db->db_state = DB_CACHED;
999 cv_broadcast(&db->db_changed);
1000 dbuf_rele_and_unlock(db, NULL, B_FALSE);
1004 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
1010 DB_DNODE_ENTER(db);
1011 dn = DB_DNODE(db);
1012 ASSERT(!refcount_is_zero(&db->db_holds));
1015 ASSERT(MUTEX_HELD(&db->db_mtx));
1016 ASSERT(db->db_state == DB_UNCACHED);
1017 ASSERT(db->db_buf == NULL);
1019 if (db->db_blkid == DMU_BONUS_BLKID) {
1022 ASSERT3U(bonuslen, <=, db->db.db_size);
1023 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
1026 bzero(db->db.db_data, DN_MAX_BONUSLEN);
1028 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
1029 DB_DNODE_EXIT(db);
1030 db->db_state = DB_CACHED;
1031 mutex_exit(&db->db_mtx);
1040 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
1041 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
1042 BP_IS_HOLE(db->db_blkptr)))) {
1043 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1045 dbuf_set_data(db, arc_alloc_buf(db->db_objset->os_spa, db, type,
1046 db->db.db_size));
1047 bzero(db->db.db_data, db->db.db_size);
1049 if (db->db_blkptr != NULL && db->db_level > 0 &&
1050 BP_IS_HOLE(db->db_blkptr) &&
1051 db->db_blkptr->blk_birth != 0) {
1052 blkptr_t *bps = db->db.db_data;
1054 DB_DNODE(db)->dn_indblkshift) / sizeof (blkptr_t));
1057 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
1060 BP_GET_LEVEL(db->db_blkptr) == 1 ?
1062 BP_GET_LSIZE(db->db_blkptr));
1063 BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr));
1065 BP_GET_LEVEL(db->db_blkptr) - 1);
1066 BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0);
1069 DB_DNODE_EXIT(db);
1070 db->db_state = DB_CACHED;
1071 mutex_exit(&db->db_mtx);
1075 DB_DNODE_EXIT(db);
1077 db->db_state = DB_READ;
1078 mutex_exit(&db->db_mtx);
1080 if (DBUF_IS_L2CACHEABLE(db))
1083 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
1084 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
1085 db->db.db_object, db->db_level, db->db_blkid);
1087 dbuf_add_ref(db, NULL);
1089 (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr,
1090 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
1109 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
1111 dbuf_dirty_record_t *dr = db->db_last_dirty;
1113 ASSERT(MUTEX_HELD(&db->db_mtx));
1114 ASSERT(db->db.db_data != NULL);
1115 ASSERT(db->db_level == 0);
1116 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
1120 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
1131 if (db->db_blkid == DMU_BONUS_BLKID) {
1135 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
1136 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
1137 int size = arc_buf_size(db->db_buf);
1138 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1139 spa_t *spa = db->db_objset->os_spa;
1141 arc_get_compression(db->db_buf);
1144 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size);
1147 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db,
1148 size, arc_buf_lsize(db->db_buf), compress_type);
1150 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
1152 db->db_buf = NULL;
1153 dbuf_clear_data(db);
1158 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
1168 ASSERT(!refcount_is_zero(&db->db_holds));
1170 if (db->db_state == DB_NOFILL)
1173 DB_DNODE_ENTER(db);
1174 dn = DB_DNODE(db);
1178 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1180 DBUF_IS_CACHEABLE(db);
1182 mutex_enter(&db->db_mtx);
1183 if (db->db_state == DB_CACHED) {
1189 if (db->db_buf != NULL &&
1190 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF) {
1191 dbuf_fix_old_data(db,
1192 spa_syncing_txg(dmu_objset_spa(db->db_objset)));
1193 err = arc_decompress(db->db_buf);
1194 dbuf_set_data(db, db->db_buf);
1196 mutex_exit(&db->db_mtx);
1198 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE);
1201 DB_DNODE_EXIT(db);
1202 } else if (db->db_state == DB_UNCACHED) {
1207 db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) {
1211 dbuf_read_impl(db, zio, flags);
1216 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE);
1220 DB_DNODE_EXIT(db);
1233 mutex_exit(&db->db_mtx);
1235 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE);
1238 DB_DNODE_EXIT(db);
1241 mutex_enter(&db->db_mtx);
1243 while (db->db_state == DB_READ ||
1244 db->db_state == DB_FILL) {
1245 ASSERT(db->db_state == DB_READ ||
1248 db, zio_t *, zio);
1249 cv_wait(&db->db_changed, &db->db_mtx);
1251 if (db->db_state == DB_UNCACHED)
1254 mutex_exit(&db->db_mtx);
1261 dbuf_noread(dmu_buf_impl_t *db)
1263 ASSERT(!refcount_is_zero(&db->db_holds));
1264 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1265 mutex_enter(&db->db_mtx);
1266 while (db->db_state == DB_READ || db->db_state == DB_FILL)
1267 cv_wait(&db->db_changed, &db->db_mtx);
1268 if (db->db_state == DB_UNCACHED) {
1269 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1270 spa_t *spa = db->db_objset->os_spa;
1272 ASSERT(db->db_buf == NULL);
1273 ASSERT(db->db.db_data == NULL);
1274 dbuf_set_data(db, arc_alloc_buf(spa, db, type, db->db.db_size));
1275 db->db_state = DB_FILL;
1276 } else if (db->db_state == DB_NOFILL) {
1277 dbuf_clear_data(db);
1279 ASSERT3U(db->db_state, ==, DB_CACHED);
1281 mutex_exit(&db->db_mtx);
1287 dmu_buf_impl_t *db = dr->dr_dbuf;
1291 ASSERT(MUTEX_HELD(&db->db_mtx));
1298 ASSERT(db->db_level == 0);
1300 if (db->db_blkid == DMU_BONUS_BLKID ||
1304 ASSERT(db->db_data_pending != dr);
1308 zio_free(db->db_objset->os_spa, txg, bp);
1321 arc_release(dr->dt.dl.dr_data, db);
1334 dmu_buf_impl_t *db, *db_next;
1348 db = avl_find(&dn->dn_dbufs, &db_search, &where);
1349 ASSERT3P(db, ==, NULL);
1351 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
1353 for (; db != NULL; db = db_next) {
1354 db_next = AVL_NEXT(&dn->dn_dbufs, db);
1355 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1357 if (db->db_level != 0 || db->db_blkid > end_blkid) {
1360 ASSERT3U(db->db_blkid, >=, start_blkid);
1363 mutex_enter(&db->db_mtx);
1364 if (dbuf_undirty(db, tx)) {
1369 if (db->db_state == DB_UNCACHED ||
1370 db->db_state == DB_NOFILL ||
1371 db->db_state == DB_EVICTING) {
1372 ASSERT(db->db.db_data == NULL);
1373 mutex_exit(&db->db_mtx);
1376 if (db->db_state == DB_READ || db->db_state == DB_FILL) {
1378 db->db_freed_in_flight = TRUE;
1379 mutex_exit(&db->db_mtx);
1382 if (refcount_count(&db->db_holds) == 0) {
1383 ASSERT(db->db_buf);
1384 dbuf_destroy(db);
1389 if (db->db_last_dirty != NULL) {
1390 dbuf_dirty_record_t *dr = db->db_last_dirty;
1398 if (db->db_blkid != DMU_SPILL_BLKID &&
1399 db->db_blkid > dn->dn_maxblkid)
1400 dn->dn_maxblkid = db->db_blkid;
1409 dbuf_fix_old_data(db, txg);
1413 if (db->db_state == DB_CACHED) {
1414 ASSERT(db->db.db_data != NULL);
1415 arc_release(db->db_buf, db);
1416 bzero(db->db.db_data, db->db.db_size);
1417 arc_buf_freeze(db->db_buf);
1420 mutex_exit(&db->db_mtx);
1426 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
1429 int osize = db->db.db_size;
1430 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1433 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1435 DB_DNODE_ENTER(db);
1436 dn = DB_DNODE(db);
1443 * is OK, because there can be no other references to the db
1451 dmu_buf_will_dirty(&db->db, tx);
1454 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size);
1457 obuf = db->db_buf;
1463 mutex_enter(&db->db_mtx);
1464 dbuf_set_data(db, buf);
1465 arc_buf_destroy(obuf, db);
1466 db->db.db_size = size;
1468 if (db->db_level == 0) {
1469 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
1470 db->db_last_dirty->dt.dl.dr_data = buf;
1472 mutex_exit(&db->db_mtx);
1475 DB_DNODE_EXIT(db);
1479 dbuf_release_bp(dmu_buf_impl_t *db)
1481 objset_t *os = db->db_objset;
1486 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
1488 (void) arc_release(db->db_buf, db);
1498 dmu_buf_impl_t *db = dr->dr_dbuf;
1500 ASSERT(MUTEX_HELD(&db->db_mtx));
1502 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
1508 if (db->db.db_object != DMU_META_DNODE_OBJECT &&
1509 db->db_state != DB_NOFILL) {
1511 ASSERT(arc_released(db->db_buf));
1512 arc_buf_thaw(db->db_buf);
1518 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1527 ASSERT(!refcount_is_zero(&db->db_holds));
1528 DMU_TX_DIRTY_BUF(tx, db);
1530 DB_DNODE_ENTER(db);
1531 dn = DB_DNODE(db);
1558 mutex_enter(&db->db_mtx);
1564 ASSERT(db->db_level != 0 ||
1565 db->db_state == DB_CACHED || db->db_state == DB_FILL ||
1566 db->db_state == DB_NOFILL);
1591 if (db->db_blkid == DMU_SPILL_BLKID)
1597 drp = &db->db_last_dirty;
1599 db->db.db_object == DMU_META_DNODE_OBJECT);
1603 DB_DNODE_EXIT(db);
1606 mutex_exit(&db->db_mtx);
1617 ASSERT3U(dn->dn_nlevels, >, db->db_level);
1636 ASSERT(db->db.db_size != 0);
1638 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1640 if (db->db_blkid != DMU_BONUS_BLKID) {
1641 dmu_objset_willuse_space(os, db->db.db_size, tx);
1650 if (db->db_level == 0) {
1651 void *data_old = db->db_buf;
1653 if (db->db_state != DB_NOFILL) {
1654 if (db->db_blkid == DMU_BONUS_BLKID) {
1655 dbuf_fix_old_data(db, tx->tx_txg);
1656 data_old = db->db.db_data;
1657 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1667 arc_release(db->db_buf, db);
1668 dbuf_fix_old_data(db, tx->tx_txg);
1669 data_old = db->db_buf;
1680 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL)
1681 dr->dr_accounted = db->db.db_size;
1682 dr->dr_dbuf = db;
1692 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1693 db->db_blkid != DMU_SPILL_BLKID) {
1697 db->db_blkid, 1);
1700 db->db_freed_in_flight = FALSE;
1706 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1707 db->db_dirtycnt += 1;
1708 ASSERT3U(db->db_dirtycnt, <=, 3);
1710 mutex_exit(&db->db_mtx);
1712 if (db->db_blkid == DMU_BONUS_BLKID ||
1713 db->db_blkid == DMU_SPILL_BLKID) {
1719 DB_DNODE_EXIT(db);
1738 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
1739 dn->dn_phys->dn_nlevels > db->db_level ||
1740 dn->dn_next_nlevels[txgoff] > db->db_level ||
1741 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
1742 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
1750 ddt_prefetch(os->os_spa, db->db_blkptr);
1752 if (db->db_level == 0) {
1753 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1754 ASSERT(dn->dn_maxblkid >= db->db_blkid);
1757 if (db->db_level+1 < dn->dn_nlevels) {
1758 dmu_buf_impl_t *parent = db->db_parent;
1762 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1765 parent = dbuf_hold_level(dn, db->db_level+1,
1766 db->db_blkid >> epbs, FTAG);
1772 ASSERT3U(db->db_level+1, ==, parent->db_level);
1777 mutex_enter(&db->db_mtx);
1782 if (db->db_last_dirty == dr ||
1791 mutex_exit(&db->db_mtx);
1793 ASSERT(db->db_level+1 == dn->dn_nlevels);
1794 ASSERT(db->db_blkid < dn->dn_nblkptr);
1795 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
1805 DB_DNODE_EXIT(db);
1814 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1828 ASSERT(db->db_objset ==
1829 dmu_objset_pool(db->db_objset)->dp_meta_objset ||
1830 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
1831 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1832 ASSERT0(db->db_level);
1833 ASSERT(MUTEX_HELD(&db->db_mtx));
1838 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1844 ASSERT(dr->dr_dbuf == db);
1846 DB_DNODE_ENTER(db);
1847 dn = DB_DNODE(db);
1849 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1851 ASSERT(db->db.db_size != 0);
1868 } else if (db->db_blkid == DMU_SPILL_BLKID ||
1869 db->db_level + 1 == dn->dn_nlevels) {
1870 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1875 DB_DNODE_EXIT(db);
1877 if (db->db_state != DB_NOFILL) {
1880 ASSERT(db->db_buf != NULL);
1882 if (dr->dt.dl.dr_data != db->db_buf)
1883 arc_buf_destroy(dr->dt.dl.dr_data, db);
1888 ASSERT(db->db_dirtycnt > 0);
1889 db->db_dirtycnt -= 1;
1891 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1892 ASSERT(db->db_state == DB_NOFILL || arc_released(db->db_buf));
1893 dbuf_destroy(db);
1903 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1907 ASSERT(!refcount_is_zero(&db->db_holds));
1915 mutex_enter(&db->db_mtx);
1917 for (dr = db->db_last_dirty;
1924 if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) {
1927 mutex_exit(&db->db_mtx);
1931 mutex_exit(&db->db_mtx);
1933 DB_DNODE_ENTER(db);
1934 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
1936 DB_DNODE_EXIT(db);
1937 (void) dbuf_read(db, NULL, rf);
1938 (void) dbuf_dirty(db, tx);
1944 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1946 db->db_state = DB_NOFILL;
1954 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1956 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1958 ASSERT(db->db_level == 0);
1959 ASSERT(!refcount_is_zero(&db->db_holds));
1961 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1964 dbuf_noread(db);
1965 (void) dbuf_dirty(db, tx);
1971 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1973 mutex_enter(&db->db_mtx);
1974 DBUF_VERIFY(db);
1976 if (db->db_state == DB_FILL) {
1977 if (db->db_level == 0 && db->db_freed_in_flight) {
1978 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1981 bzero(db->db.db_data, db->db.db_size);
1982 db->db_freed_in_flight = FALSE;
1984 db->db_state = DB_CACHED;
1985 cv_broadcast(&db->db_changed);
1987 mutex_exit(&db->db_mtx);
1996 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2001 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset),
2005 DB_DNODE_ENTER(db);
2006 type = DB_DNODE(db)->dn_type;
2007 DB_DNODE_EXIT(db);
2009 ASSERT0(db->db_level);
2010 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2014 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
2015 dl = &db->db_last_dirty->dt.dl;
2024 dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg;
2032 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
2034 ASSERT(!refcount_is_zero(&db->db_holds));
2035 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2036 ASSERT(db->db_level == 0);
2037 ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf));
2039 ASSERT(arc_buf_lsize(buf) == db->db.db_size);
2042 arc_return_buf(buf, db);
2045 mutex_enter(&db->db_mtx);
2047 while (db->db_state == DB_READ || db->db_state == DB_FILL)
2048 cv_wait(&db->db_changed, &db->db_mtx);
2050 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED);
2052 if (db->db_state == DB_CACHED &&
2053 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
2054 mutex_exit(&db->db_mtx);
2055 (void) dbuf_dirty(db, tx);
2056 bcopy(buf->b_data, db->db.db_data, db->db.db_size);
2057 arc_buf_destroy(buf, db);
2063 if (db->db_state == DB_CACHED) {
2064 dbuf_dirty_record_t *dr = db->db_last_dirty;
2066 ASSERT(db->db_buf != NULL);
2068 ASSERT(dr->dt.dl.dr_data == db->db_buf);
2069 if (!arc_released(db->db_buf)) {
2072 arc_release(db->db_buf, db);
2075 arc_buf_destroy(db->db_buf, db);
2076 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
2077 arc_release(db->db_buf, db);
2078 arc_buf_destroy(db->db_buf, db);
2080 db->db_buf = NULL;
2082 ASSERT(db->db_buf == NULL);
2083 dbuf_set_data(db, buf);
2084 db->db_state = DB_FILL;
2085 mutex_exit(&db->db_mtx);
2086 (void) dbuf_dirty(db, tx);
2087 dmu_buf_fill_done(&db->db, tx);
2091 dbuf_destroy(dmu_buf_impl_t *db)
2094 dmu_buf_impl_t *parent = db->db_parent;
2097 ASSERT(MUTEX_HELD(&db->db_mtx));
2098 ASSERT(refcount_is_zero(&db->db_holds));
2100 if (db->db_buf != NULL) {
2101 arc_buf_destroy(db->db_buf, db);
2102 db->db_buf = NULL;
2105 if (db->db_blkid == DMU_BONUS_BLKID) {
2106 ASSERT(db->db.db_data != NULL);
2107 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
2109 db->db_state = DB_UNCACHED;
2112 dbuf_clear_data(db);
2114 if (multilist_link_active(&db->db_cache_link)) {
2115 ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
2116 db->db_caching_status == DB_DBUF_METADATA_CACHE);
2118 multilist_remove(dbuf_caches[db->db_caching_status].cache, db);
2120 &dbuf_caches[db->db_caching_status].size,
2121 db->db.db_size, db);
2123 db->db_caching_status = DB_NO_CACHE;
2126 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
2127 ASSERT(db->db_data_pending == NULL);
2129 db->db_state = DB_EVICTING;
2130 db->db_blkptr = NULL;
2137 mutex_exit(&db->db_mtx);
2139 DB_DNODE_ENTER(db);
2140 dn = DB_DNODE(db);
2142 if (db->db_blkid != DMU_BONUS_BLKID) {
2146 avl_remove(&dn->dn_dbufs, db);
2149 DB_DNODE_EXIT(db);
2161 dnode_rele_and_unlock(dn, db, B_TRUE);
2162 db->db_dnode_handle = NULL;
2164 dbuf_hash_remove(db);
2166 DB_DNODE_EXIT(db);
2169 ASSERT(refcount_is_zero(&db->db_holds));
2171 db->db_parent = NULL;
2173 ASSERT(db->db_buf == NULL);
2174 ASSERT(db->db.db_data == NULL);
2175 ASSERT(db->db_hash_next == NULL);
2176 ASSERT(db->db_blkptr == NULL);
2177 ASSERT(db->db_data_pending == NULL);
2178 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);
2179 ASSERT(!multilist_link_active(&db->db_cache_link));
2181 kmem_cache_free(dbuf_kmem_cache, db);
2190 dbuf_rele_and_unlock(parent, db, B_TRUE);
2266 *bpp = ((blkptr_t *)(*parentp)->db.db_data) +
2290 dmu_buf_impl_t *db, *odb;
2295 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP);
2297 db->db_objset = os;
2298 db->db.db_object = dn->dn_object;
2299 db->db_level = level;
2300 db->db_blkid = blkid;
2301 db->db_last_dirty = NULL;
2302 db->db_dirtycnt = 0;
2303 db->db_dnode_handle = dn->dn_handle;
2304 db->db_parent = parent;
2305 db->db_blkptr = blkptr;
2307 db->db_user = NULL;
2308 db->db_user_immediate_evict = FALSE;
2309 db->db_freed_in_flight = FALSE;
2310 db->db_pending_evict = FALSE;
2314 db->db.db_size = DN_MAX_BONUSLEN -
2316 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
2317 db->db.db_offset = DMU_BONUS_BLKID;
2318 db->db_state = DB_UNCACHED;
2319 db->db_caching_status = DB_NO_CACHE;
2322 return (db);
2324 db->db.db_size = (blkptr != NULL) ?
2326 db->db.db_offset = 0;
2329 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz;
2330 db->db.db_size = blocksize;
2331 db->db.db_offset = db->db_blkid * blocksize;
2342 db->db_state = DB_EVICTING;
2343 if ((odb = dbuf_hash_insert(db)) != NULL) {
2345 kmem_cache_free(dbuf_kmem_cache, db);
2349 avl_add(&dn->dn_dbufs, db);
2351 db->db_state = DB_UNCACHED;
2352 db->db_caching_status = DB_NO_CACHE;
2357 dbuf_add_ref(parent, db);
2361 (void) refcount_add(&dn->dn_holds, db);
2364 dprintf_dbuf(db, "db=%p\n", db);
2366 return (db);
2446 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode,
2448 (void) dbuf_read(db, NULL,
2450 dbuf_rele(db, FTAG);
2527 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object,
2529 if (db != NULL) {
2530 mutex_exit(&db->db_mtx);
2548 dmu_buf_impl_t *db;
2551 FALSE, TRUE, FTAG, &db) == 0) {
2552 blkptr_t *bpp = db->db_buf->b_data;
2554 dbuf_rele(db, FTAG);
2633 dmu_buf_impl_t *db, *parent = NULL;
2642 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid);
2644 if (db == NULL) {
2664 db = dbuf_create(dn, level, blkid, parent, bp);
2667 if (fail_uncached && db->db_state != DB_CACHED) {
2668 mutex_exit(&db->db_mtx);
2672 if (db->db_buf != NULL) {
2673 arc_buf_access(db->db_buf);
2674 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
2677 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
2684 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
2686 db->db_state == DB_CACHED && db->db_data_pending) {
2687 dbuf_dirty_record_t *dr = db->db_data_pending;
2689 if (dr->dt.dl.dr_data == db->db_buf) {
2690 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2692 dbuf_set_data(db,
2693 arc_alloc_buf(dn->dn_objset->os_spa, db, type,
2694 db->db.db_size));
2695 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
2696 db->db.db_size);
2700 if (multilist_link_active(&db->db_cache_link)) {
2701 ASSERT(refcount_is_zero(&db->db_holds));
2702 ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
2703 db->db_caching_status == DB_DBUF_METADATA_CACHE);
2705 multilist_remove(dbuf_caches[db->db_caching_status].cache, db);
2707 &dbuf_caches[db->db_caching_status].size,
2708 db->db.db_size, db);
2710 db->db_caching_status = DB_NO_CACHE;
2712 (void) refcount_add(&db->db_holds, tag);
2713 DBUF_VERIFY(db);
2714 mutex_exit(&db->db_mtx);
2720 ASSERT3P(DB_DNODE(db), ==, dn);
2721 ASSERT3U(db->db_blkid, ==, blkid);
2722 ASSERT3U(db->db_level, ==, level);
2723 *dbp = db;
2737 dmu_buf_impl_t *db;
2738 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db);
2739 return (err ? NULL : db);
2754 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2757 if (db->db_blkid != DMU_SPILL_BLKID)
2761 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset)));
2764 DB_DNODE_ENTER(db);
2765 dn = DB_DNODE(db);
2767 dbuf_new_size(db, blksz, tx);
2769 DB_DNODE_EXIT(db);
2782 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
2784 int64_t holds = refcount_add(&db->db_holds, tag);
2793 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2797 if (db->db_blkid == DMU_BONUS_BLKID)
2803 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) {
2804 (void) refcount_add(&db->db_holds, tag);
2807 mutex_exit(&db->db_mtx);
2820 dbuf_rele(dmu_buf_impl_t *db, void *tag)
2822 mutex_enter(&db->db_mtx);
2823 dbuf_rele_and_unlock(db, tag, B_FALSE);
2827 dmu_buf_rele(dmu_buf_t *db, void *tag)
2829 dbuf_rele((dmu_buf_impl_t *)db, tag);
2846 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag, boolean_t evicting)
2850 ASSERT(MUTEX_HELD(&db->db_mtx));
2851 DBUF_VERIFY(db);
2858 holds = refcount_remove(&db->db_holds, tag);
2865 if (db->db_buf != NULL &&
2866 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) {
2867 arc_buf_freeze(db->db_buf);
2870 if (holds == db->db_dirtycnt &&
2871 db->db_level == 0 && db->db_user_immediate_evict)
2872 dbuf_evict_user(db);
2875 if (db->db_blkid == DMU_BONUS_BLKID) {
2877 boolean_t evict_dbuf = db->db_pending_evict;
2883 DB_DNODE_ENTER(db);
2885 dn = DB_DNODE(db);
2894 DB_DNODE_EXIT(db);
2897 * Do not reference db after its lock is dropped.
2900 mutex_exit(&db->db_mtx);
2905 dnode_rele(dn, db);
2906 } else if (db->db_buf == NULL) {
2911 ASSERT(db->db_state == DB_UNCACHED ||
2912 db->db_state == DB_NOFILL);
2913 dbuf_destroy(db);
2914 } else if (arc_released(db->db_buf)) {
2918 dbuf_destroy(db);
2922 spa_t *spa = dmu_objset_spa(db->db_objset);
2924 if (!DBUF_IS_CACHEABLE(db) &&
2925 db->db_blkptr != NULL &&
2926 !BP_IS_HOLE(db->db_blkptr) &&
2927 !BP_IS_EMBEDDED(db->db_blkptr)) {
2929 bp = *db->db_blkptr;
2932 if (!DBUF_IS_CACHEABLE(db) ||
2933 db->db_pending_evict) {
2934 dbuf_destroy(db);
2935 } else if (!multilist_link_active(&db->db_cache_link)) {
2936 ASSERT3U(db->db_caching_status, ==,
2940 dbuf_include_in_metadata_cache(db) ?
2942 db->db_caching_status = dcs;
2944 multilist_insert(dbuf_caches[dcs].cache, db);
2946 db->db.db_size, db);
2947 mutex_exit(&db->db_mtx);
2949 if (db->db_caching_status == DB_DBUF_CACHE &&
2959 mutex_exit(&db->db_mtx);
2966 dbuf_refcount(dmu_buf_impl_t *db)
2968 return (refcount_count(&db->db_holds));
2975 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2977 mutex_enter(&db->db_mtx);
2978 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2979 if (db->db_user == old_user)
2980 db->db_user = new_user;
2982 old_user = db->db_user;
2983 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2984 mutex_exit(&db->db_mtx);
2998 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
3000 db->db_user_immediate_evict = TRUE;
3013 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
3015 dbuf_verify_user(db, DBVU_NOT_EVICTING);
3016 return (db->db_user);
3026 dmu_buf_get_blkptr(dmu_buf_t *db)
3028 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
3033 dmu_buf_get_objset(dmu_buf_t *db)
3035 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
3040 dmu_buf_dnode_enter(dmu_buf_t *db)
3042 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
3048 dmu_buf_dnode_exit(dmu_buf_t *db)
3050 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
3055 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
3058 ASSERT(MUTEX_HELD(&db->db_mtx));
3060 if (db->db_blkptr != NULL)
3063 if (db->db_blkid == DMU_SPILL_BLKID) {
3064 db->db_blkptr = &dn->dn_phys->dn_spill;
3065 BP_ZERO(db->db_blkptr);
3068 if (db->db_level == dn->dn_phys->dn_nlevels-1) {
3074 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
3075 ASSERT(db->db_parent == NULL);
3076 db->db_parent = dn->dn_dbuf;
3077 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
3078 DBUF_VERIFY(db);
3080 dmu_buf_impl_t *parent = db->db_parent;
3085 mutex_exit(&db->db_mtx);
3087 parent = dbuf_hold_level(dn, db->db_level + 1,
3088 db->db_blkid >> epbs, db);
3090 mutex_enter(&db->db_mtx);
3091 db->db_parent = parent;
3093 db->db_blkptr = (blkptr_t *)parent->db.db_data +
3094 (db->db_blkid & ((1ULL << epbs) - 1));
3095 DBUF_VERIFY(db);
3102 dmu_buf_impl_t *db = dr->dr_dbuf;
3108 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
3110 mutex_enter(&db->db_mtx);
3112 ASSERT(db->db_level > 0);
3113 DBUF_VERIFY(db);
3116 if (db->db_buf == NULL) {
3117 mutex_exit(&db->db_mtx);
3118 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
3119 mutex_enter(&db->db_mtx);
3121 ASSERT3U(db->db_state, ==, DB_CACHED);
3122 ASSERT(db->db_buf != NULL);
3124 DB_DNODE_ENTER(db);
3125 dn = DB_DNODE(db);
3127 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
3128 dbuf_check_blkptr(dn, db);
3129 DB_DNODE_EXIT(db);
3132 db->db_data_pending = dr;
3134 mutex_exit(&db->db_mtx);
3136 dbuf_write(dr, db->db_buf, tx);
3140 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx);
3150 dmu_buf_impl_t *db = dr->dr_dbuf;
3157 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
3159 mutex_enter(&db->db_mtx);
3164 if (db->db_state == DB_UNCACHED) {
3166 ASSERT(db->db.db_data == NULL);
3167 } else if (db->db_state == DB_FILL) {
3169 ASSERT(db->db.db_data != dr->dt.dl.dr_data);
3171 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
3173 DBUF_VERIFY(db);
3175 DB_DNODE_ENTER(db);
3176 dn = DB_DNODE(db);
3178 if (db->db_blkid == DMU_SPILL_BLKID) {
3190 if (db->db_blkid == DMU_BONUS_BLKID) {
3194 ASSERT0(db->db_level);
3197 DB_DNODE_EXIT(db);
3199 if (*datap != db->db.db_data) {
3203 db->db_data_pending = NULL;
3204 drp = &db->db_last_dirty;
3208 ASSERT(dr->dr_dbuf == db);
3215 ASSERT(db->db_dirtycnt > 0);
3216 db->db_dirtycnt -= 1;
3217 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg, B_FALSE);
3229 dbuf_check_blkptr(dn, db);
3237 cv_wait(&db->db_changed, &db->db_mtx);
3241 if (db->db_state != DB_NOFILL &&
3243 refcount_count(&db->db_holds) > 1 &&
3245 *datap == db->db_buf) {
3258 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
3262 *datap = arc_alloc_buf(os->os_spa, db, type, psize);
3266 *datap = arc_alloc_compressed_buf(os->os_spa, db,
3269 bcopy(db->db.db_data, (*datap)->b_data, psize);
3271 db->db_data_pending = dr;
3273 mutex_exit(&db->db_mtx);
3280 DB_DNODE_EXIT(db);
3289 DB_DNODE_EXIT(db);
3308 ASSERT3U(dr->dr_dbuf->db.db_object, ==,
3328 dmu_buf_impl_t *db = vdb;
3337 ASSERT3P(db->db_blkptr, !=, NULL);
3338 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp);
3340 DB_DNODE_ENTER(db);
3341 dn = DB_DNODE(db);
3347 ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
3349 (db->db_blkid == DMU_SPILL_BLKID &&
3352 ASSERT(BP_GET_LEVEL(bp) == db->db_level);
3355 mutex_enter(&db->db_mtx);
3358 if (db->db_blkid == DMU_SPILL_BLKID) {
3361 db->db_blkptr == &dn->dn_phys->dn_spill);
3365 if (db->db_level == 0) {
3367 if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
3368 db->db_blkid != DMU_SPILL_BLKID)
3369 dn->dn_phys->dn_maxblkid = db->db_blkid;
3373 dnode_phys_t *dnp = db->db.db_data;
3374 for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
3387 blkptr_t *ibp = db->db.db_data;
3388 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
3389 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
3395 DB_DNODE_EXIT(db);
3400 mutex_exit(&db->db_mtx);
3403 *db->db_blkptr = *bp;
3418 dmu_buf_impl_t *db = vdb;
3423 ASSERT3U(db->db_level, >, 0);
3424 DB_DNODE_ENTER(db);
3425 dn = DB_DNODE(db);
3430 for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) {
3446 bzero(db->db.db_data, db->db.db_size);
3449 DB_DNODE_EXIT(db);
3465 dmu_buf_impl_t *db = arg;
3466 objset_t *os = db->db_objset;
3471 dr = db->db_data_pending;
3488 dmu_buf_impl_t *db = vdb;
3490 blkptr_t *bp = db->db_blkptr;
3491 objset_t *os = db->db_objset;
3496 ASSERT(db->db_blkptr == bp);
3510 mutex_enter(&db->db_mtx);
3512 DBUF_VERIFY(db);
3514 drp = &db->db_last_dirty;
3515 while ((dr = *drp) != db->db_data_pending)
3518 ASSERT(dr->dr_dbuf == db);
3523 if (db->db_blkid == DMU_SPILL_BLKID) {
3526 DB_DNODE_ENTER(db);
3527 dn = DB_DNODE(db);
3529 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
3530 db->db_blkptr == &dn->dn_phys->dn_spill);
3531 DB_DNODE_EXIT(db);
3535 if (db->db_level == 0) {
3536 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
3538 if (db->db_state != DB_NOFILL) {
3539 if (dr->dt.dl.dr_data != db->db_buf)
3540 arc_buf_destroy(dr->dt.dl.dr_data, db);
3545 DB_DNODE_ENTER(db);
3546 dn = DB_DNODE(db);
3548 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);
3549 if (!BP_IS_HOLE(db->db_blkptr)) {
3552 ASSERT3U(db->db_blkid, <=,
3553 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs));
3554 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
3555 db->db.db_size);
3557 DB_DNODE_EXIT(db);
3563 cv_broadcast(&db->db_changed);
3564 ASSERT(db->db_dirtycnt > 0);
3565 db->db_dirtycnt -= 1;
3566 db->db_data_pending = NULL;
3567 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE);
3586 dmu_buf_impl_t *db = dr->dr_dbuf;
3588 dbuf_write_ready(zio, NULL, db);
3595 dmu_buf_impl_t *db = dr->dr_dbuf;
3598 mutex_enter(&db->db_mtx);
3602 arc_release(dr->dt.dl.dr_data, db);
3604 mutex_exit(&db->db_mtx);
3605 dbuf_write_done(zio, NULL, db);
3667 dbuf_can_remap(const dmu_buf_impl_t *db)
3669 spa_t *spa = dmu_objset_spa(db->db_objset);
3670 blkptr_t *bp = db->db.db_data;
3673 ASSERT3U(db->db_level, >, 0);
3674 ASSERT3S(db->db_state, ==, DB_CACHED);
3679 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) {
3720 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx)
3722 spa_t *spa = dmu_objset_spa(db->db_objset);
3728 if (db->db_level > 0) {
3729 blkptr_t *bp = db->db.db_data;
3730 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) {
3733 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) {
3734 dnode_phys_t *dnp = db->db.db_data;
3735 ASSERT3U(db->db_dnode_handle->dnh_dnode->dn_type, ==,
3737 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT; i++) {
3750 dmu_buf_impl_t *db = dr->dr_dbuf;
3753 dmu_buf_impl_t *parent = db->db_parent;
3762 DB_DNODE_ENTER(db);
3763 dn = DB_DNODE(db);
3766 if (db->db_state != DB_NOFILL) {
3767 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
3774 if (BP_IS_HOLE(db->db_blkptr)) {
3777 dbuf_release_bp(db);
3779 dbuf_remap(dn, db, tx);
3788 ASSERT(db->db_level == parent->db_level-1);
3797 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
3798 db->db_blkid != DMU_SPILL_BLKID) ||
3799 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
3800 if (db->db_blkid != DMU_SPILL_BLKID)
3801 ASSERT3P(db->db_blkptr, ==,
3802 &dn->dn_phys->dn_blkptr[db->db_blkid]);
3806 ASSERT(db->db_level == 0 || data == db->db_buf);
3807 ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
3812 db->db.db_object, db->db_level, db->db_blkid);
3814 if (db->db_blkid == DMU_SPILL_BLKID)
3816 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
3818 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
3819 DB_DNODE_EXIT(db);
3827 dr->dr_bp_copy = *db->db_blkptr;
3829 if (db->db_level == 0 &&
3839 contents, db->db.db_size, db->db.db_size, &zp,
3843 mutex_enter(&db->db_mtx);
3847 mutex_exit(&db->db_mtx);
3848 } else if (db->db_state == DB_NOFILL) {
3852 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp,
3854 dbuf_write_nofill_done, db,
3866 if (db->db_level != 0)
3870 &dr->dr_bp_copy, data, DBUF_IS_L2CACHEABLE(db),
3872 dbuf_write_physdone, dbuf_write_done, db,