Deleted Added
sdiff udiff text old ( 339034 ) new ( 339105 )
full compact
zio.c (339034) zio.c (339105)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 6 unchanged lines hidden (view full) ---

15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 6 unchanged lines hidden (view full) ---

15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
23 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 Integros [integros.com]
26 */
27
28#include <sys/sysmacros.h>
29#include <sys/zfs_context.h>
30#include <sys/fm/fs/zfs.h>
31#include <sys/spa.h>

--- 7 unchanged lines hidden (view full) ---

39#include <sys/arc.h>
40#include <sys/ddt.h>
41#include <sys/trim_map.h>
42#include <sys/blkptr.h>
43#include <sys/zfeature.h>
44#include <sys/dsl_scan.h>
45#include <sys/metaslab_impl.h>
46#include <sys/abd.h>
24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 Integros [integros.com]
26 */
27
28#include <sys/sysmacros.h>
29#include <sys/zfs_context.h>
30#include <sys/fm/fs/zfs.h>
31#include <sys/spa.h>

--- 7 unchanged lines hidden (view full) ---

39#include <sys/arc.h>
40#include <sys/ddt.h>
41#include <sys/trim_map.h>
42#include <sys/blkptr.h>
43#include <sys/zfeature.h>
44#include <sys/dsl_scan.h>
45#include <sys/metaslab_impl.h>
46#include <sys/abd.h>
47#include <sys/cityhash.h>
47
48SYSCTL_DECL(_vfs_zfs);
49SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO");
50#if defined(__amd64__)
51static int zio_use_uma = 1;
52#else
53static int zio_use_uma = 0;
54#endif

--- 2275 unchanged lines hidden (view full) ---

2330 int error;
2331
2332 int flags = METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER;
2333 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
2334 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
2335 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA));
2336
2337 flags |= METASLAB_ASYNC_ALLOC;
48
49SYSCTL_DECL(_vfs_zfs);
50SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO");
51#if defined(__amd64__)
52static int zio_use_uma = 1;
53#else
54static int zio_use_uma = 0;
55#endif

--- 2275 unchanged lines hidden (view full) ---

2331 int error;
2332
2333 int flags = METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER;
2334 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
2335 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
2336 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA));
2337
2338 flags |= METASLAB_ASYNC_ALLOC;
2338 VERIFY(refcount_held(&mc->mc_alloc_slots, pio));
2339 VERIFY(refcount_held(&mc->mc_alloc_slots[pio->io_allocator],
2340 pio));
2339
2340 /*
2341 * The logical zio has already placed a reservation for
2342 * 'copies' allocation slots but gang blocks may require
2343 * additional copies. These additional copies
2344 * (i.e. gbh_copies - copies) are guaranteed to succeed
2345 * since metaslab_class_throttle_reserve() always allows
2346 * additional reservations for gang blocks.
2347 */
2348 VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies,
2341
2342 /*
2343 * The logical zio has already placed a reservation for
2344 * 'copies' allocation slots but gang blocks may require
2345 * additional copies. These additional copies
2346 * (i.e. gbh_copies - copies) are guaranteed to succeed
2347 * since metaslab_class_throttle_reserve() always allows
2348 * additional reservations for gang blocks.
2349 */
2350 VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies,
2349 pio, flags));
2351 pio->io_allocator, pio, flags));
2350 }
2351
2352 error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE,
2353 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags,
2352 }
2353
2354 error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE,
2355 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags,
2354 &pio->io_alloc_list, pio);
2356 &pio->io_alloc_list, pio, pio->io_allocator);
2355 if (error) {
2356 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
2357 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
2358 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA));
2359
2360 /*
2361 * If we failed to allocate the gang block header then
2362 * we remove any additional allocation reservations that
2363 * we placed here. The original reservation will
2364 * be removed when the logical I/O goes to the ready
2365 * stage.
2366 */
2367 metaslab_class_throttle_unreserve(mc,
2357 if (error) {
2358 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
2359 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
2360 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA));
2361
2362 /*
2363 * If we failed to allocate the gang block header then
2364 * we remove any additional allocation reservations that
2365 * we placed here. The original reservation will
2366 * be removed when the logical I/O goes to the ready
2367 * stage.
2368 */
2369 metaslab_class_throttle_unreserve(mc,
2368 gbh_copies - copies, pio);
2370 gbh_copies - copies, pio->io_allocator, pio);
2369 }
2370 pio->io_error = error;
2371 return (ZIO_PIPELINE_CONTINUE);
2372 }
2373
2374 if (pio == gio) {
2375 gnpp = &gio->io_gang_tree;
2376 } else {

--- 41 unchanged lines hidden (view full) ---

2418 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA));
2419
2420 /*
2421 * Gang children won't throttle but we should
2422 * account for their work, so reserve an allocation
2423 * slot for them here.
2424 */
2425 VERIFY(metaslab_class_throttle_reserve(mc,
2371 }
2372 pio->io_error = error;
2373 return (ZIO_PIPELINE_CONTINUE);
2374 }
2375
2376 if (pio == gio) {
2377 gnpp = &gio->io_gang_tree;
2378 } else {

--- 41 unchanged lines hidden (view full) ---

2420 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA));
2421
2422 /*
2423 * Gang children won't throttle but we should
2424 * account for their work, so reserve an allocation
2425 * slot for them here.
2426 */
2427 VERIFY(metaslab_class_throttle_reserve(mc,
2426 zp.zp_copies, cio, flags));
2428 zp.zp_copies, cio->io_allocator, cio, flags));
2427 }
2428 zio_nowait(cio);
2429 }
2430
2431 /*
2432 * Set pio's pipeline to just wait for zio to finish.
2433 */
2434 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;

--- 473 unchanged lines hidden (view full) ---

2908
2909/*
2910 * ==========================================================================
2911 * Allocate and free blocks
2912 * ==========================================================================
2913 */
2914
2915static zio_t *
2429 }
2430 zio_nowait(cio);
2431 }
2432
2433 /*
2434 * Set pio's pipeline to just wait for zio to finish.
2435 */
2436 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;

--- 473 unchanged lines hidden (view full) ---

2910
2911/*
2912 * ==========================================================================
2913 * Allocate and free blocks
2914 * ==========================================================================
2915 */
2916
2917static zio_t *
2916zio_io_to_allocate(spa_t *spa)
2918zio_io_to_allocate(spa_t *spa, int allocator)
2917{
2918 zio_t *zio;
2919
2919{
2920 zio_t *zio;
2921
2920 ASSERT(MUTEX_HELD(&spa->spa_alloc_lock));
2922 ASSERT(MUTEX_HELD(&spa->spa_alloc_locks[allocator]));
2921
2923
2922 zio = avl_first(&spa->spa_alloc_tree);
2924 zio = avl_first(&spa->spa_alloc_trees[allocator]);
2923 if (zio == NULL)
2924 return (NULL);
2925
2926 ASSERT(IO_IS_ALLOCATING(zio));
2927
2928 /*
2929 * Try to place a reservation for this zio. If we're unable to
2930 * reserve then we throttle.
2931 */
2925 if (zio == NULL)
2926 return (NULL);
2927
2928 ASSERT(IO_IS_ALLOCATING(zio));
2929
2930 /*
2931 * Try to place a reservation for this zio. If we're unable to
2932 * reserve then we throttle.
2933 */
2934 ASSERT3U(zio->io_allocator, ==, allocator);
2932 if (!metaslab_class_throttle_reserve(spa_normal_class(spa),
2935 if (!metaslab_class_throttle_reserve(spa_normal_class(spa),
2933 zio->io_prop.zp_copies, zio, 0)) {
2936 zio->io_prop.zp_copies, zio->io_allocator, zio, 0)) {
2934 return (NULL);
2935 }
2936
2937 return (NULL);
2938 }
2939
2937 avl_remove(&spa->spa_alloc_tree, zio);
2940 avl_remove(&spa->spa_alloc_trees[allocator], zio);
2938 ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE);
2939
2940 return (zio);
2941}
2942
2943static int
2944zio_dva_throttle(zio_t *zio)
2945{

--- 7 unchanged lines hidden (view full) ---

2953 return (ZIO_PIPELINE_CONTINUE);
2954 }
2955
2956 ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
2957
2958 ASSERT3U(zio->io_queued_timestamp, >, 0);
2959 ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE);
2960
2941 ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE);
2942
2943 return (zio);
2944}
2945
2946static int
2947zio_dva_throttle(zio_t *zio)
2948{

--- 7 unchanged lines hidden (view full) ---

2956 return (ZIO_PIPELINE_CONTINUE);
2957 }
2958
2959 ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
2960
2961 ASSERT3U(zio->io_queued_timestamp, >, 0);
2962 ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE);
2963
2961 mutex_enter(&spa->spa_alloc_lock);
2964 zbookmark_phys_t *bm = &zio->io_bookmark;
2965 /*
2966 * We want to try to use as many allocators as possible to help improve
2967 * performance, but we also want logically adjacent IOs to be physically
2968 * adjacent to improve sequential read performance. We chunk each object
2969 * into 2^20 block regions, and then hash based on the objset, object,
2970 * level, and region to accomplish both of these goals.
2971 */
2972 zio->io_allocator = cityhash4(bm->zb_objset, bm->zb_object,
2973 bm->zb_level, bm->zb_blkid >> 20) % spa->spa_alloc_count;
2974 mutex_enter(&spa->spa_alloc_locks[zio->io_allocator]);
2962
2963 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
2975
2976 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
2964 avl_add(&spa->spa_alloc_tree, zio);
2977 avl_add(&spa->spa_alloc_trees[zio->io_allocator], zio);
2965
2978
2966 nio = zio_io_to_allocate(zio->io_spa);
2967 mutex_exit(&spa->spa_alloc_lock);
2979 nio = zio_io_to_allocate(zio->io_spa, zio->io_allocator);
2980 mutex_exit(&spa->spa_alloc_locks[zio->io_allocator]);
2968
2969 if (nio == zio)
2970 return (ZIO_PIPELINE_CONTINUE);
2971
2972 if (nio != NULL) {
2973 ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE);
2974 /*
2975 * We are passing control to a new zio so make sure that

--- 4 unchanged lines hidden (view full) ---

2980 * been waiting.
2981 */
2982 zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE);
2983 }
2984 return (ZIO_PIPELINE_STOP);
2985}
2986
2987void
2981
2982 if (nio == zio)
2983 return (ZIO_PIPELINE_CONTINUE);
2984
2985 if (nio != NULL) {
2986 ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE);
2987 /*
2988 * We are passing control to a new zio so make sure that

--- 4 unchanged lines hidden (view full) ---

2993 * been waiting.
2994 */
2995 zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE);
2996 }
2997 return (ZIO_PIPELINE_STOP);
2998}
2999
3000void
2988zio_allocate_dispatch(spa_t *spa)
3001zio_allocate_dispatch(spa_t *spa, int allocator)
2989{
2990 zio_t *zio;
2991
3002{
3003 zio_t *zio;
3004
2992 mutex_enter(&spa->spa_alloc_lock);
2993 zio = zio_io_to_allocate(spa);
2994 mutex_exit(&spa->spa_alloc_lock);
3005 mutex_enter(&spa->spa_alloc_locks[allocator]);
3006 zio = zio_io_to_allocate(spa, allocator);
3007 mutex_exit(&spa->spa_alloc_locks[allocator]);
2995 if (zio == NULL)
2996 return;
2997
2998 ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE);
2999 ASSERT0(zio->io_error);
3000 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE);
3001}
3002

--- 24 unchanged lines hidden (view full) ---

3027 flags |= METASLAB_GANG_CHILD;
3028 }
3029 if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE) {
3030 flags |= METASLAB_ASYNC_ALLOC;
3031 }
3032
3033 error = metaslab_alloc(spa, mc, zio->io_size, bp,
3034 zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
3008 if (zio == NULL)
3009 return;
3010
3011 ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE);
3012 ASSERT0(zio->io_error);
3013 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE);
3014}
3015

--- 24 unchanged lines hidden (view full) ---

3040 flags |= METASLAB_GANG_CHILD;
3041 }
3042 if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE) {
3043 flags |= METASLAB_ASYNC_ALLOC;
3044 }
3045
3046 error = metaslab_alloc(spa, mc, zio->io_size, bp,
3047 zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
3035 &zio->io_alloc_list, zio);
3048 &zio->io_alloc_list, zio, zio->io_allocator);
3036
3037 if (error != 0) {
3038 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, "
3039 "size %llu, error %d", spa_name(spa), zio, zio->io_size,
3040 error);
3041 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE)
3042 return (zio_write_gang_block(zio));
3043 zio->io_error = error;

--- 43 unchanged lines hidden (view full) ---

3087 }
3088 }
3089}
3090
3091/*
3092 * Try to allocate an intent log block. Return 0 on success, errno on failure.
3093 */
3094int
3049
3050 if (error != 0) {
3051 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, "
3052 "size %llu, error %d", spa_name(spa), zio, zio->io_size,
3053 error);
3054 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE)
3055 return (zio_write_gang_block(zio));
3056 zio->io_error = error;

--- 43 unchanged lines hidden (view full) ---

3100 }
3101 }
3102}
3103
3104/*
3105 * Try to allocate an intent log block. Return 0 on success, errno on failure.
3106 */
3107int
3095zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp,
3096 uint64_t size, boolean_t *slog)
3108zio_alloc_zil(spa_t *spa, uint64_t objset, uint64_t txg, blkptr_t *new_bp,
3109 blkptr_t *old_bp, uint64_t size, boolean_t *slog)
3097{
3098 int error = 1;
3099 zio_alloc_list_t io_alloc_list;
3100
3101 ASSERT(txg > spa_syncing_txg(spa));
3102
3103 metaslab_trace_init(&io_alloc_list);
3110{
3111 int error = 1;
3112 zio_alloc_list_t io_alloc_list;
3113
3114 ASSERT(txg > spa_syncing_txg(spa));
3115
3116 metaslab_trace_init(&io_alloc_list);
3117 /*
3118 * When allocating a zil block, we don't have information about
3119 * the final destination of the block except the objset it's part
3120 * of, so we just hash the objset ID to pick the allocator to get
3121 * some parallelism.
3122 */
3104 error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1,
3123 error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1,
3105 txg, old_bp, METASLAB_HINTBP_AVOID, &io_alloc_list, NULL);
3124 txg, old_bp, METASLAB_HINTBP_AVOID, &io_alloc_list, NULL,
3125 cityhash4(0, 0, 0, objset) % spa->spa_alloc_count);
3106 if (error == 0) {
3107 *slog = TRUE;
3108 } else {
3109 error = metaslab_alloc(spa, spa_normal_class(spa), size,
3110 new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID,
3126 if (error == 0) {
3127 *slog = TRUE;
3128 } else {
3129 error = metaslab_alloc(spa, spa_normal_class(spa), size,
3130 new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID,
3111 &io_alloc_list, NULL);
3131 &io_alloc_list, NULL, cityhash4(0, 0, 0, objset) %
3132 spa->spa_alloc_count);
3112 if (error == 0)
3113 *slog = FALSE;
3114 }
3115 metaslab_trace_fini(&io_alloc_list);
3116
3117 if (error == 0) {
3118 BP_SET_LSIZE(new_bp, size);
3119 BP_SET_PSIZE(new_bp, size);

--- 553 unchanged lines hidden (view full) ---

3673 ASSERT(IO_IS_ALLOCATING(zio));
3674 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
3675 /*
3676 * We were unable to allocate anything, unreserve and
3677 * issue the next I/O to allocate.
3678 */
3679 metaslab_class_throttle_unreserve(
3680 spa_normal_class(zio->io_spa),
3133 if (error == 0)
3134 *slog = FALSE;
3135 }
3136 metaslab_trace_fini(&io_alloc_list);
3137
3138 if (error == 0) {
3139 BP_SET_LSIZE(new_bp, size);
3140 BP_SET_PSIZE(new_bp, size);

--- 553 unchanged lines hidden (view full) ---

3694 ASSERT(IO_IS_ALLOCATING(zio));
3695 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
3696 /*
3697 * We were unable to allocate anything, unreserve and
3698 * issue the next I/O to allocate.
3699 */
3700 metaslab_class_throttle_unreserve(
3701 spa_normal_class(zio->io_spa),
3681 zio->io_prop.zp_copies, zio);
3682 zio_allocate_dispatch(zio->io_spa);
3702 zio->io_prop.zp_copies, zio->io_allocator, zio);
3703 zio_allocate_dispatch(zio->io_spa, zio->io_allocator);
3683 }
3684 }
3685
3686 mutex_enter(&zio->io_lock);
3687 zio->io_state[ZIO_WAIT_READY] = 1;
3688 pio = zio_walk_parents(zio, &zl);
3689 mutex_exit(&zio->io_lock);
3690

--- 66 unchanged lines hidden (view full) ---

3757
3758 ASSERT(IO_IS_ALLOCATING(pio));
3759 ASSERT3P(zio, !=, zio->io_logical);
3760 ASSERT(zio->io_logical != NULL);
3761 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));
3762 ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE);
3763
3764 mutex_enter(&pio->io_lock);
3704 }
3705 }
3706
3707 mutex_enter(&zio->io_lock);
3708 zio->io_state[ZIO_WAIT_READY] = 1;
3709 pio = zio_walk_parents(zio, &zl);
3710 mutex_exit(&zio->io_lock);
3711

--- 66 unchanged lines hidden (view full) ---

3778
3779 ASSERT(IO_IS_ALLOCATING(pio));
3780 ASSERT3P(zio, !=, zio->io_logical);
3781 ASSERT(zio->io_logical != NULL);
3782 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));
3783 ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE);
3784
3785 mutex_enter(&pio->io_lock);
3765 metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags);
3786 metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags,
3787 pio->io_allocator, B_TRUE);
3766 mutex_exit(&pio->io_lock);
3767
3768 metaslab_class_throttle_unreserve(spa_normal_class(zio->io_spa),
3788 mutex_exit(&pio->io_lock);
3789
3790 metaslab_class_throttle_unreserve(spa_normal_class(zio->io_spa),
3769 1, pio);
3791 1, pio->io_allocator, pio);
3770
3771 /*
3772 * Call into the pipeline to see if there is more work that
3773 * needs to be done. If there is work to be done it will be
3774 * dispatched to another taskq thread.
3775 */
3792
3793 /*
3794 * Call into the pipeline to see if there is more work that
3795 * needs to be done. If there is work to be done it will be
3796 * dispatched to another taskq thread.
3797 */
3776 zio_allocate_dispatch(zio->io_spa);
3798 zio_allocate_dispatch(zio->io_spa, pio->io_allocator);
3777}
3778
3779static int
3780zio_done(zio_t *zio)
3781{
3782 spa_t *spa = zio->io_spa;
3783 zio_t *lio = zio->io_logical;
3784 blkptr_t *bp = zio->io_bp;

--- 26 unchanged lines hidden (view full) ---

3811 /*
3812 * If the allocation throttle is enabled, verify that
3813 * we have decremented the refcounts for every I/O that was throttled.
3814 */
3815 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
3816 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
3817 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
3818 ASSERT(bp != NULL);
3799}
3800
3801static int
3802zio_done(zio_t *zio)
3803{
3804 spa_t *spa = zio->io_spa;
3805 zio_t *lio = zio->io_logical;
3806 blkptr_t *bp = zio->io_bp;

--- 26 unchanged lines hidden (view full) ---

3833 /*
3834 * If the allocation throttle is enabled, verify that
3835 * we have decremented the refcounts for every I/O that was throttled.
3836 */
3837 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
3838 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
3839 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
3840 ASSERT(bp != NULL);
3819 metaslab_group_alloc_verify(spa, zio->io_bp, zio);
3820 VERIFY(refcount_not_held(&mc->mc_alloc_slots, zio));
3841 metaslab_group_alloc_verify(spa, zio->io_bp, zio,
3842 zio->io_allocator);
3843 VERIFY(refcount_not_held(&mc->mc_alloc_slots[zio->io_allocator],
3844 zio));
3821 }
3822
3823 for (int c = 0; c < ZIO_CHILD_TYPES; c++)
3824 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
3825 ASSERT(zio->io_children[c][w] == 0);
3826
3827 if (bp != NULL && !BP_IS_EMBEDDED(bp)) {
3828 ASSERT(bp->blk_pad[0] == 0);

--- 418 unchanged lines hidden --- 		3845 }
3846
3847 for (int c = 0; c < ZIO_CHILD_TYPES; c++)
3848 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
3849 ASSERT(zio->io_children[c][w] == 0);
3850
3851 if (bp != NULL && !BP_IS_EMBEDDED(bp)) {
3852 ASSERT(bp->blk_pad[0] == 0);

--- 418 unchanged lines hidden ---