Deleted Added
sdiff udiff text old ( 286545 ) new ( 286575 )
full compact
dnode.c (286545) dnode.c (286575)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 7 unchanged lines hidden (view full) ---

16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 7 unchanged lines hidden (view full) ---

16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
24 */
25
26#include <sys/zfs_context.h>
27#include <sys/dbuf.h>
28#include <sys/dnode.h>
29#include <sys/dmu.h>
30#include <sys/dmu_impl.h>
31#include <sys/dmu_tx.h>

--- 368 unchanged lines hidden (view full) ---

400 dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT;
401 dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0;
402}
403
404static dnode_t *
405dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
406 uint64_t object, dnode_handle_t *dnh)
407{
25 */
26
27#include <sys/zfs_context.h>
28#include <sys/dbuf.h>
29#include <sys/dnode.h>
30#include <sys/dmu.h>
31#include <sys/dmu_impl.h>
32#include <sys/dmu_tx.h>

--- 368 unchanged lines hidden (view full) ---

401 dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT;
402 dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0;
403}
404
405static dnode_t *
406dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
407 uint64_t object, dnode_handle_t *dnh)
408{
408 dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
409 dnode_t *dn;
409
410
411 dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
410 ASSERT(!POINTER_IS_VALID(dn->dn_objset));
411 dn->dn_moved = 0;
412
413 /*
414 * Defer setting dn_objset until the dnode is ready to be a candidate
415 * for the dnode_move() callback.
416 */
417 dn->dn_object = object;

--- 20 unchanged lines hidden (view full) ---

438 dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0);
439 dn->dn_id_flags = 0;
440
441 dmu_zfetch_init(&dn->dn_zfetch, dn);
442
443 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
444
445 mutex_enter(&os->os_lock);
412 ASSERT(!POINTER_IS_VALID(dn->dn_objset));
413 dn->dn_moved = 0;
414
415 /*
416 * Defer setting dn_objset until the dnode is ready to be a candidate
417 * for the dnode_move() callback.
418 */
419 dn->dn_object = object;

--- 20 unchanged lines hidden (view full) ---

440 dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0);
441 dn->dn_id_flags = 0;
442
443 dmu_zfetch_init(&dn->dn_zfetch, dn);
444
445 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
446
447 mutex_enter(&os->os_lock);
446 list_insert_head(&os->os_dnodes, dn);
448 if (dnh->dnh_dnode != NULL) {
449 /* Lost the allocation race. */
450 mutex_exit(&os->os_lock);
451 kmem_cache_free(dnode_cache, dn);
452 return (dnh->dnh_dnode);
453 }
454
455 /*
456 * Exclude special dnodes from os_dnodes so an empty os_dnodes
457 * signifies that the special dnodes have no references from
458 * their children (the entries in os_dnodes). This allows
459 * dnode_destroy() to easily determine if the last child has
460 * been removed and then complete eviction of the objset.
461 */
462 if (!DMU_OBJECT_IS_SPECIAL(object))
463 list_insert_head(&os->os_dnodes, dn);
447 membar_producer();
464 membar_producer();
465
448 /*
466 /*
449 * Everything else must be valid before assigning dn_objset makes the
450 * dnode eligible for dnode_move().
467 * Everything else must be valid before assigning dn_objset
468 * makes the dnode eligible for dnode_move().
451 */
452 dn->dn_objset = os;
469 */
470 dn->dn_objset = os;
471
472 dnh->dnh_dnode = dn;
453 mutex_exit(&os->os_lock);
454
455 arc_space_consume(sizeof (dnode_t), ARC_SPACE_OTHER);
456 return (dn);
457}
458
459/*
460 * Caller must be holding the dnode handle, which is released upon return.
461 */
462static void
463dnode_destroy(dnode_t *dn)
464{
465 objset_t *os = dn->dn_objset;
473 mutex_exit(&os->os_lock);
474
475 arc_space_consume(sizeof (dnode_t), ARC_SPACE_OTHER);
476 return (dn);
477}
478
479/*
480 * Caller must be holding the dnode handle, which is released upon return.
481 */
482static void
483dnode_destroy(dnode_t *dn)
484{
485 objset_t *os = dn->dn_objset;
486 boolean_t complete_os_eviction = B_FALSE;
466
467 ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
468
469 mutex_enter(&os->os_lock);
470 POINTER_INVALIDATE(&dn->dn_objset);
487
488 ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
489
490 mutex_enter(&os->os_lock);
491 POINTER_INVALIDATE(&dn->dn_objset);
471 list_remove(&os->os_dnodes, dn);
492 if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
493 list_remove(&os->os_dnodes, dn);
494 complete_os_eviction =
495 list_is_empty(&os->os_dnodes) &&
496 list_link_active(&os->os_evicting_node);
497 }
472 mutex_exit(&os->os_lock);
473
474 /* the dnode can no longer move, so we can release the handle */
475 zrl_remove(&dn->dn_handle->dnh_zrlock);
476
477 dn->dn_allocated_txg = 0;
478 dn->dn_free_txg = 0;
479 dn->dn_assigned_txg = 0;

--- 18 unchanged lines hidden (view full) ---

498 dn->dn_newuid = 0;
499 dn->dn_newgid = 0;
500 dn->dn_id_flags = 0;
501 dn->dn_unlisted_l0_blkid = 0;
502
503 dmu_zfetch_rele(&dn->dn_zfetch);
504 kmem_cache_free(dnode_cache, dn);
505 arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER);
498 mutex_exit(&os->os_lock);
499
500 /* the dnode can no longer move, so we can release the handle */
501 zrl_remove(&dn->dn_handle->dnh_zrlock);
502
503 dn->dn_allocated_txg = 0;
504 dn->dn_free_txg = 0;
505 dn->dn_assigned_txg = 0;

--- 18 unchanged lines hidden (view full) ---

524 dn->dn_newuid = 0;
525 dn->dn_newgid = 0;
526 dn->dn_id_flags = 0;
527 dn->dn_unlisted_l0_blkid = 0;
528
529 dmu_zfetch_rele(&dn->dn_zfetch);
530 kmem_cache_free(dnode_cache, dn);
531 arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER);
532
533 if (complete_os_eviction)
534 dmu_objset_evict_done(os);
506}
507
508void
509dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
510 dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
511{
512 int i;
513

--- 452 unchanged lines hidden (view full) ---

966 /*
967 * Wait for final references to the dnode to clear. This can
968 * only happen if the arc is asyncronously evicting state that
969 * has a hold on this dnode while we are trying to evict this
970 * dnode.
971 */
972 while (refcount_count(&dn->dn_holds) > 0)
973 delay(1);
535}
536
537void
538dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
539 dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
540{
541 int i;
542

--- 452 unchanged lines hidden (view full) ---

995 /*
996 * Wait for final references to the dnode to clear. This can
997 * only happen if the arc is asyncronously evicting state that
998 * has a hold on this dnode while we are trying to evict this
999 * dnode.
1000 */
1001 while (refcount_count(&dn->dn_holds) > 0)
1002 delay(1);
1003 ASSERT(dn->dn_dbuf == NULL ||
1004 dmu_buf_get_user(&dn->dn_dbuf->db) == NULL);
974 zrl_add(&dnh->dnh_zrlock);
975 dnode_destroy(dn); /* implicit zrl_remove() */
976 zrl_destroy(&dnh->dnh_zrlock);
977 dnh->dnh_dnode = NULL;
978}
979
1005 zrl_add(&dnh->dnh_zrlock);
1006 dnode_destroy(dn); /* implicit zrl_remove() */
1007 zrl_destroy(&dnh->dnh_zrlock);
1008 dnh->dnh_dnode = NULL;
1009}
1010
980dnode_t *
1011void
981dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object,
982 dnode_handle_t *dnh)
983{
1012dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object,
1013 dnode_handle_t *dnh)
1014{
984 dnode_t *dn = dnode_create(os, dnp, NULL, object, dnh);
985 dnh->dnh_dnode = dn;
1015 dnode_t *dn;
1016
1017 dn = dnode_create(os, dnp, NULL, object, dnh);
986 zrl_init(&dnh->dnh_zrlock);
987 DNODE_VERIFY(dn);
1018 zrl_init(&dnh->dnh_zrlock);
1019 DNODE_VERIFY(dn);
988 return (dn);
989}
990
991static void
1020}
1021
1022static void
992dnode_buf_pageout(dmu_buf_t *db, void *arg)
1023dnode_buf_pageout(void *dbu)
993{
1024{
994 dnode_children_t *children_dnodes = arg;
1025 dnode_children_t *children_dnodes = dbu;
995 int i;
1026 int i;
996 int epb = db->db_size >> DNODE_SHIFT;
997
1027
998 ASSERT(epb == children_dnodes->dnc_count);
999
1000 for (i = 0; i < epb; i++) {
1028 for (i = 0; i < children_dnodes->dnc_count; i++) {
1001 dnode_handle_t *dnh = &children_dnodes->dnc_children[i];
1002 dnode_t *dn;
1003
1004 /*
1005 * The dnode handle lock guards against the dnode moving to
1006 * another valid address, so there is no need here to guard
1007 * against changes to or from NULL.
1008 */

--- 13 unchanged lines hidden (view full) ---

1022 ASSERT(refcount_is_zero(&dn->dn_holds));
1023 ASSERT(refcount_is_zero(&dn->dn_tx_holds));
1024
1025 dnode_destroy(dn); /* implicit zrl_remove() */
1026 zrl_destroy(&dnh->dnh_zrlock);
1027 dnh->dnh_dnode = NULL;
1028 }
1029 kmem_free(children_dnodes, sizeof (dnode_children_t) +
1029 dnode_handle_t *dnh = &children_dnodes->dnc_children[i];
1030 dnode_t *dn;
1031
1032 /*
1033 * The dnode handle lock guards against the dnode moving to
1034 * another valid address, so there is no need here to guard
1035 * against changes to or from NULL.
1036 */

--- 13 unchanged lines hidden (view full) ---

1050 ASSERT(refcount_is_zero(&dn->dn_holds));
1051 ASSERT(refcount_is_zero(&dn->dn_tx_holds));
1052
1053 dnode_destroy(dn); /* implicit zrl_remove() */
1054 zrl_destroy(&dnh->dnh_zrlock);
1055 dnh->dnh_dnode = NULL;
1056 }
1057 kmem_free(children_dnodes, sizeof (dnode_children_t) +
1030 epb * sizeof (dnode_handle_t));
1058 children_dnodes->dnc_count * sizeof (dnode_handle_t));
1031}
1032
1033/*
1034 * errors:
1035 * EINVAL - invalid object number.
1036 * EIO - i/o error.
1037 * succeeds even for free dnodes.
1038 */

--- 73 unchanged lines hidden (view full) ---

1112 int i;
1113 dnode_children_t *winner;
1114 children_dnodes = kmem_zalloc(sizeof (dnode_children_t) +
1115 epb * sizeof (dnode_handle_t), KM_SLEEP);
1116 children_dnodes->dnc_count = epb;
1117 dnh = &children_dnodes->dnc_children[0];
1118 for (i = 0; i < epb; i++) {
1119 zrl_init(&dnh[i].dnh_zrlock);
1059}
1060
1061/*
1062 * errors:
1063 * EINVAL - invalid object number.
1064 * EIO - i/o error.
1065 * succeeds even for free dnodes.
1066 */

--- 73 unchanged lines hidden (view full) ---

1140 int i;
1141 dnode_children_t *winner;
1142 children_dnodes = kmem_zalloc(sizeof (dnode_children_t) +
1143 epb * sizeof (dnode_handle_t), KM_SLEEP);
1144 children_dnodes->dnc_count = epb;
1145 dnh = &children_dnodes->dnc_children[0];
1146 for (i = 0; i < epb; i++) {
1147 zrl_init(&dnh[i].dnh_zrlock);
1120 dnh[i].dnh_dnode = NULL;
1121 }
1148 }
1122 if (winner = dmu_buf_set_user(&db->db, children_dnodes,
1123 dnode_buf_pageout)) {
1149 dmu_buf_init_user(&children_dnodes->dnc_dbu,
1150 dnode_buf_pageout, NULL);
1151 winner = dmu_buf_set_user(&db->db, &children_dnodes->dnc_dbu);
1152 if (winner != NULL) {
1124
1125 for (i = 0; i < epb; i++) {
1126 zrl_destroy(&dnh[i].dnh_zrlock);
1127 }
1128
1129 kmem_free(children_dnodes, sizeof (dnode_children_t) +
1130 epb * sizeof (dnode_handle_t));
1131 children_dnodes = winner;
1132 }
1133 }
1134 ASSERT(children_dnodes->dnc_count == epb);
1135
1136 dnh = &children_dnodes->dnc_children[idx];
1137 zrl_add(&dnh->dnh_zrlock);
1153
1154 for (i = 0; i < epb; i++) {
1155 zrl_destroy(&dnh[i].dnh_zrlock);
1156 }
1157
1158 kmem_free(children_dnodes, sizeof (dnode_children_t) +
1159 epb * sizeof (dnode_handle_t));
1160 children_dnodes = winner;
1161 }
1162 }
1163 ASSERT(children_dnodes->dnc_count == epb);
1164
1165 dnh = &children_dnodes->dnc_children[idx];
1166 zrl_add(&dnh->dnh_zrlock);
1138 if ((dn = dnh->dnh_dnode) == NULL) {
1167 dn = dnh->dnh_dnode;
1168 if (dn == NULL) {
1139 dnode_phys_t *phys = (dnode_phys_t *)db->db.db_data+idx;
1169 dnode_phys_t *phys = (dnode_phys_t *)db->db.db_data+idx;
1140 dnode_t *winner;
1141
1142 dn = dnode_create(os, phys, db, object, dnh);
1170
1171 dn = dnode_create(os, phys, db, object, dnh);
1143 winner = atomic_cas_ptr(&dnh->dnh_dnode, NULL, dn);
1144 if (winner != NULL) {
1145 zrl_add(&dnh->dnh_zrlock);
1146 dnode_destroy(dn); /* implicit zrl_remove() */
1147 dn = winner;
1148 }
1149 }
1150
1151 mutex_enter(&dn->dn_mtx);
1152 type = dn->dn_type;
1153 if (dn->dn_free_txg ||
1154 ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE) ||
1155 ((flag & DNODE_MUST_BE_FREE) &&
1156 (type != DMU_OT_NONE || !refcount_is_zero(&dn->dn_holds)))) {
1157 mutex_exit(&dn->dn_mtx);
1158 zrl_remove(&dnh->dnh_zrlock);
1159 dbuf_rele(db, FTAG);
1160 return (type == DMU_OT_NONE ? ENOENT : EEXIST);
1161 }
1172 }
1173
1174 mutex_enter(&dn->dn_mtx);
1175 type = dn->dn_type;
1176 if (dn->dn_free_txg ||
1177 ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE) ||
1178 ((flag & DNODE_MUST_BE_FREE) &&
1179 (type != DMU_OT_NONE || !refcount_is_zero(&dn->dn_holds)))) {
1180 mutex_exit(&dn->dn_mtx);
1181 zrl_remove(&dnh->dnh_zrlock);
1182 dbuf_rele(db, FTAG);
1183 return (type == DMU_OT_NONE ? ENOENT : EEXIST);
1184 }
1162 mutex_exit(&dn->dn_mtx);
1163
1164 if (refcount_add(&dn->dn_holds, tag) == 1)
1165 dbuf_add_ref(db, dnh);
1185 if (refcount_add(&dn->dn_holds, tag) == 1)
1186 dbuf_add_ref(db, dnh);
1187 mutex_exit(&dn->dn_mtx);
1188
1166 /* Now we can rely on the hold to prevent the dnode from moving. */
1167 zrl_remove(&dnh->dnh_zrlock);
1168
1169 DNODE_VERIFY(dn);
1170 ASSERT3P(dn->dn_dbuf, ==, db);
1171 ASSERT3U(dn->dn_object, ==, object);
1172 dbuf_rele(db, FTAG);
1173

--- 849 unchanged lines hidden --- 		1189 /* Now we can rely on the hold to prevent the dnode from moving. */
1190 zrl_remove(&dnh->dnh_zrlock);
1191
1192 DNODE_VERIFY(dn);
1193 ASSERT3P(dn->dn_dbuf, ==, db);
1194 ASSERT3U(dn->dn_object, ==, object);
1195 dbuf_rele(db, FTAG);
1196

--- 849 unchanged lines hidden ---