Deleted Added
sdiff udiff text old ( 243503 ) new ( 243524 )
full compact
zio.c (243503) zio.c (243524)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 695 unchanged lines hidden (view full) ---

704
705 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF &&
706 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS &&
707 zp->zp_compress >= ZIO_COMPRESS_OFF &&
708 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS &&
709 DMU_OT_IS_VALID(zp->zp_type) &&
710 zp->zp_level < 32 &&
711 zp->zp_copies > 0 &&
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 695 unchanged lines hidden (view full) ---

704
705 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF &&
706 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS &&
707 zp->zp_compress >= ZIO_COMPRESS_OFF &&
708 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS &&
709 DMU_OT_IS_VALID(zp->zp_type) &&
710 zp->zp_level < 32 &&
711 zp->zp_copies > 0 &&
712 zp->zp_copies <= spa_max_replication(spa) &&
713 zp->zp_dedup <= 1 &&
714 zp->zp_dedup_verify <= 1);
712 zp->zp_copies <= spa_max_replication(spa));
715
716 zio = zio_create(pio, spa, txg, bp, data, size, done, private,
717 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
718 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
719 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE);
720
721 zio->io_ready = ready;
722 zio->io_prop = *zp;

--- 11 unchanged lines hidden (view full) ---

734 zio = zio_create(pio, spa, txg, bp, data, size, done, private,
735 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
736 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE);
737
738 return (zio);
739}
740
741void
713
714 zio = zio_create(pio, spa, txg, bp, data, size, done, private,
715 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
716 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
717 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE);
718
719 zio->io_ready = ready;
720 zio->io_prop = *zp;

--- 11 unchanged lines hidden (view full) ---

732 zio = zio_create(pio, spa, txg, bp, data, size, done, private,
733 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
734 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE);
735
736 return (zio);
737}
738
739void
742zio_write_override(zio_t *zio, blkptr_t *bp, int copies)
740zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite)
743{
744 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
745 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
746 ASSERT(zio->io_stage == ZIO_STAGE_OPEN);
747 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));
748
741{
742 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
743 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
744 ASSERT(zio->io_stage == ZIO_STAGE_OPEN);
745 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));
746
747 /*
748 * We must reset the io_prop to match the values that existed
749 * when the bp was first written by dmu_sync() keeping in mind
750 * that nopwrite and dedup are mutually exclusive.
751 */
752 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup;
753 zio->io_prop.zp_nopwrite = nopwrite;
749 zio->io_prop.zp_copies = copies;
750 zio->io_bp_override = bp;
751}
752
753void
754zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
755{
756 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);

--- 283 unchanged lines hidden (view full) ---

1040
1041 if (zio->io_bp_override) {
1042 ASSERT(bp->blk_birth != zio->io_txg);
1043 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0);
1044
1045 *bp = *zio->io_bp_override;
1046 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1047
754 zio->io_prop.zp_copies = copies;
755 zio->io_bp_override = bp;
756}
757
758void
759zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
760{
761 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);

--- 283 unchanged lines hidden (view full) ---

1045
1046 if (zio->io_bp_override) {
1047 ASSERT(bp->blk_birth != zio->io_txg);
1048 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0);
1049
1050 *bp = *zio->io_bp_override;
1051 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1052
1053 /*
1054 * If we've been overridden and nopwrite is set then
1055 * set the flag accordingly to indicate that a nopwrite
1056 * has already occurred.
1057 */
1058 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) {
1059 ASSERT(!zp->zp_dedup);
1060 zio->io_flags |= ZIO_FLAG_NOPWRITE;
1061 return (ZIO_PIPELINE_CONTINUE);
1062 }
1063
1064 ASSERT(!zp->zp_nopwrite);
1065
1048 if (BP_IS_HOLE(bp) || !zp->zp_dedup)
1049 return (ZIO_PIPELINE_CONTINUE);
1050
1051 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup ||
1052 zp->zp_dedup_verify);
1053
1054 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) {
1055 BP_SET_DEDUP(bp, 1);

--- 71 unchanged lines hidden (view full) ---

1127 BP_SET_LEVEL(bp, zp->zp_level);
1128 BP_SET_DEDUP(bp, zp->zp_dedup);
1129 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
1130 if (zp->zp_dedup) {
1131 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1132 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
1133 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE;
1134 }
1066 if (BP_IS_HOLE(bp) || !zp->zp_dedup)
1067 return (ZIO_PIPELINE_CONTINUE);
1068
1069 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup ||
1070 zp->zp_dedup_verify);
1071
1072 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) {
1073 BP_SET_DEDUP(bp, 1);

--- 71 unchanged lines hidden (view full) ---

1145 BP_SET_LEVEL(bp, zp->zp_level);
1146 BP_SET_DEDUP(bp, zp->zp_dedup);
1147 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
1148 if (zp->zp_dedup) {
1149 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1150 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
1151 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE;
1152 }
1153 if (zp->zp_nopwrite) {
1154 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1155 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
1156 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE;
1157 }
1135 }
1136
1137 return (ZIO_PIPELINE_CONTINUE);
1138}
1139
1140static int
1141zio_free_bp_init(zio_t *zio)
1142{

--- 205 unchanged lines hidden (view full) ---

1348 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);
1349 ASSERT(pio->io_gang_leader == NULL);
1350 ASSERT(pio->io_gang_tree == NULL);
1351
1352 pio->io_flags = pio->io_orig_flags;
1353 pio->io_stage = pio->io_orig_stage;
1354 pio->io_pipeline = pio->io_orig_pipeline;
1355 pio->io_reexecute = 0;
1158 }
1159
1160 return (ZIO_PIPELINE_CONTINUE);
1161}
1162
1163static int
1164zio_free_bp_init(zio_t *zio)
1165{

--- 205 unchanged lines hidden (view full) ---

1371 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);
1372 ASSERT(pio->io_gang_leader == NULL);
1373 ASSERT(pio->io_gang_tree == NULL);
1374
1375 pio->io_flags = pio->io_orig_flags;
1376 pio->io_stage = pio->io_orig_stage;
1377 pio->io_pipeline = pio->io_orig_pipeline;
1378 pio->io_reexecute = 0;
1379 pio->io_flags |= ZIO_FLAG_REEXECUTED;
1356 pio->io_error = 0;
1357 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
1358 pio->io_state[w] = 0;
1359 for (int c = 0; c < ZIO_CHILD_TYPES; c++)
1360 pio->io_child_error[c] = 0;
1361
1362 if (IO_IS_ALLOCATING(pio))
1363 BP_ZERO(pio->io_bp);

--- 460 unchanged lines hidden (view full) ---

1824 SPA_MINBLOCKSIZE);
1825 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid);
1826
1827 zp.zp_checksum = gio->io_prop.zp_checksum;
1828 zp.zp_compress = ZIO_COMPRESS_OFF;
1829 zp.zp_type = DMU_OT_NONE;
1830 zp.zp_level = 0;
1831 zp.zp_copies = gio->io_prop.zp_copies;
1380 pio->io_error = 0;
1381 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
1382 pio->io_state[w] = 0;
1383 for (int c = 0; c < ZIO_CHILD_TYPES; c++)
1384 pio->io_child_error[c] = 0;
1385
1386 if (IO_IS_ALLOCATING(pio))
1387 BP_ZERO(pio->io_bp);

--- 460 unchanged lines hidden (view full) ---

1848 SPA_MINBLOCKSIZE);
1849 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid);
1850
1851 zp.zp_checksum = gio->io_prop.zp_checksum;
1852 zp.zp_compress = ZIO_COMPRESS_OFF;
1853 zp.zp_type = DMU_OT_NONE;
1854 zp.zp_level = 0;
1855 zp.zp_copies = gio->io_prop.zp_copies;
1832 zp.zp_dedup = 0;
1833 zp.zp_dedup_verify = 0;
1856 zp.zp_dedup = B_FALSE;
1857 zp.zp_dedup_verify = B_FALSE;
1858 zp.zp_nopwrite = B_FALSE;
1834
1835 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g],
1836 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp,
1837 zio_write_gang_member_ready, NULL, &gn->gn_child[g],
1838 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
1839 &pio->io_bookmark));
1840 }
1841
1842 /*
1843 * Set pio's pipeline to just wait for zio to finish.
1844 */
1845 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1846
1847 zio_nowait(zio);
1848
1849 return (ZIO_PIPELINE_CONTINUE);
1850}
1851
1852/*
1859
1860 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g],
1861 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp,
1862 zio_write_gang_member_ready, NULL, &gn->gn_child[g],
1863 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
1864 &pio->io_bookmark));
1865 }
1866
1867 /*
1868 * Set pio's pipeline to just wait for zio to finish.
1869 */
1870 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1871
1872 zio_nowait(zio);
1873
1874 return (ZIO_PIPELINE_CONTINUE);
1875}
1876
1877/*
1878 * The zio_nop_write stage in the pipeline determines if allocating
1879 * a new bp is necessary. By leveraging a cryptographically secure checksum,
1880 * such as SHA256, we can compare the checksums of the new data and the old
1881 * to determine if allocating a new block is required. The nopwrite
1882 * feature can handle writes in either syncing or open context (i.e. zil
1883 * writes) and as a result is mutually exclusive with dedup.
1884 */
1885static int
1886zio_nop_write(zio_t *zio)
1887{
1888 blkptr_t *bp = zio->io_bp;
1889 blkptr_t *bp_orig = &zio->io_bp_orig;
1890 zio_prop_t *zp = &zio->io_prop;
1891
1892 ASSERT(BP_GET_LEVEL(bp) == 0);
1893 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
1894 ASSERT(zp->zp_nopwrite);
1895 ASSERT(!zp->zp_dedup);
1896 ASSERT(zio->io_bp_override == NULL);
1897 ASSERT(IO_IS_ALLOCATING(zio));
1898
1899 /*
1900 * Check to see if the original bp and the new bp have matching
1901 * characteristics (i.e. same checksum, compression algorithms, etc).
1902 * If they don't then just continue with the pipeline which will
1903 * allocate a new bp.
1904 */
1905 if (BP_IS_HOLE(bp_orig) ||
1906 !zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_dedup ||
1907 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) ||
1908 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) ||
1909 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) ||
1910 zp->zp_copies != BP_GET_NDVAS(bp_orig))
1911 return (ZIO_PIPELINE_CONTINUE);
1912
1913 /*
1914 * If the checksums match then reset the pipeline so that we
1915 * avoid allocating a new bp and issuing any I/O.
1916 */
1917 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) {
1918 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup);
1919 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig));
1920 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig));
1921 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF);
1922 ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop,
1923 sizeof (uint64_t)) == 0);
1924
1925 *bp = *bp_orig;
1926 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1927 zio->io_flags |= ZIO_FLAG_NOPWRITE;
1928 }
1929
1930 return (ZIO_PIPELINE_CONTINUE);
1931}
1932
1933/*
1853 * ==========================================================================
1854 * Dedup
1855 * ==========================================================================
1856 */
1857static void
1858zio_ddt_child_read_done(zio_t *zio)
1859{
1860 blkptr_t *bp = zio->io_bp;

--- 255 unchanged lines hidden (view full) ---

2116 * (And automatically e-mail a paper to Nature?)
2117 */
2118 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) {
2119 zp->zp_checksum = spa_dedup_checksum(spa);
2120 zio_pop_transforms(zio);
2121 zio->io_stage = ZIO_STAGE_OPEN;
2122 BP_ZERO(bp);
2123 } else {
1934 * ==========================================================================
1935 * Dedup
1936 * ==========================================================================
1937 */
1938static void
1939zio_ddt_child_read_done(zio_t *zio)
1940{
1941 blkptr_t *bp = zio->io_bp;

--- 255 unchanged lines hidden (view full) ---

2197 * (And automatically e-mail a paper to Nature?)
2198 */
2199 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) {
2200 zp->zp_checksum = spa_dedup_checksum(spa);
2201 zio_pop_transforms(zio);
2202 zio->io_stage = ZIO_STAGE_OPEN;
2203 BP_ZERO(bp);
2204 } else {
2124 zp->zp_dedup = 0;
2205 zp->zp_dedup = B_FALSE;
2125 }
2126 zio->io_pipeline = ZIO_WRITE_PIPELINE;
2127 ddt_exit(ddt);
2128 return (ZIO_PIPELINE_CONTINUE);
2129 }
2130
2131 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp);
2132 ASSERT(ditto_copies < SPA_DVAS_PER_BP);

--- 640 unchanged lines hidden (view full) ---

2773 zio_t *pio, *pio_next;
2774
2775 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) ||
2776 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY))
2777 return (ZIO_PIPELINE_STOP);
2778
2779 if (zio->io_ready) {
2780 ASSERT(IO_IS_ALLOCATING(zio));
2206 }
2207 zio->io_pipeline = ZIO_WRITE_PIPELINE;
2208 ddt_exit(ddt);
2209 return (ZIO_PIPELINE_CONTINUE);
2210 }
2211
2212 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp);
2213 ASSERT(ditto_copies < SPA_DVAS_PER_BP);

--- 640 unchanged lines hidden (view full) ---

2854 zio_t *pio, *pio_next;
2855
2856 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) ||
2857 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY))
2858 return (ZIO_PIPELINE_STOP);
2859
2860 if (zio->io_ready) {
2861 ASSERT(IO_IS_ALLOCATING(zio));
2781 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp));
2862 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) ||
2863 (zio->io_flags & ZIO_FLAG_NOPWRITE));
2782 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);
2783
2784 zio->io_ready(zio);
2785 }
2786
2787 if (bp != NULL && bp != &zio->io_bp_copy)
2788 zio->io_bp_copy = *bp;
2789

--- 65 unchanged lines hidden (view full) ---

2855 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) &&
2856 zio->io_bp_override == NULL &&
2857 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {
2858 ASSERT(!BP_SHOULD_BYTESWAP(bp));
2859 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp));
2860 ASSERT(BP_COUNT_GANG(bp) == 0 ||
2861 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp)));
2862 }
2864 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);
2865
2866 zio->io_ready(zio);
2867 }
2868
2869 if (bp != NULL && bp != &zio->io_bp_copy)
2870 zio->io_bp_copy = *bp;
2871

--- 65 unchanged lines hidden (view full) ---

2937 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) &&
2938 zio->io_bp_override == NULL &&
2939 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {
2940 ASSERT(!BP_SHOULD_BYTESWAP(bp));
2941 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp));
2942 ASSERT(BP_COUNT_GANG(bp) == 0 ||
2943 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp)));
2944 }
2945 if (zio->io_flags & ZIO_FLAG_NOPWRITE)
2946 VERIFY(BP_EQUAL(bp, &zio->io_bp_orig));
2863 }
2864
2865 /*
2866 * If there were child vdev/gang/ddt errors, they apply to us now.
2867 */
2868 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV);
2869 zio_inherit_child_errors(zio, ZIO_CHILD_GANG);
2870 zio_inherit_child_errors(zio, ZIO_CHILD_DDT);

--- 93 unchanged lines hidden (view full) ---

2964 * We defer this until now to avoid conflating logical child
2965 * errors with errors that happened to the zio itself when
2966 * updating vdev stats and reporting FMA events above.
2967 */
2968 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL);
2969
2970 if ((zio->io_error || zio->io_reexecute) &&
2971 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio &&
2947 }
2948
2949 /*
2950 * If there were child vdev/gang/ddt errors, they apply to us now.
2951 */
2952 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV);
2953 zio_inherit_child_errors(zio, ZIO_CHILD_GANG);
2954 zio_inherit_child_errors(zio, ZIO_CHILD_DDT);

--- 93 unchanged lines hidden (view full) ---

3048 * We defer this until now to avoid conflating logical child
3049 * errors with errors that happened to the zio itself when
3050 * updating vdev stats and reporting FMA events above.
3051 */
3052 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL);
3053
3054 if ((zio->io_error || zio->io_reexecute) &&
3055 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio &&
2972 !(zio->io_flags & ZIO_FLAG_IO_REWRITE))
3056 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)))
2973 zio_dva_unallocate(zio, zio->io_gang_tree, bp);
2974
2975 zio_gang_tree_free(&zio->io_gang_tree);
2976
2977 /*
2978 * Godfather I/Os should never suspend.
2979 */
2980 if ((zio->io_flags & ZIO_FLAG_GODFATHER) &&

--- 127 unchanged lines hidden (view full) ---

3108 */
3109static zio_pipe_stage_t *zio_pipeline[] = {
3110 NULL,
3111 zio_read_bp_init,
3112 zio_free_bp_init,
3113 zio_issue_async,
3114 zio_write_bp_init,
3115 zio_checksum_generate,
3057 zio_dva_unallocate(zio, zio->io_gang_tree, bp);
3058
3059 zio_gang_tree_free(&zio->io_gang_tree);
3060
3061 /*
3062 * Godfather I/Os should never suspend.
3063 */
3064 if ((zio->io_flags & ZIO_FLAG_GODFATHER) &&

--- 127 unchanged lines hidden (view full) ---

3192 */
3193static zio_pipe_stage_t *zio_pipeline[] = {
3194 NULL,
3195 zio_read_bp_init,
3196 zio_free_bp_init,
3197 zio_issue_async,
3198 zio_write_bp_init,
3199 zio_checksum_generate,
3200 zio_nop_write,
3116 zio_ddt_read_start,
3117 zio_ddt_read_done,
3118 zio_ddt_write,
3119 zio_ddt_free,
3120 zio_gang_assemble,
3121 zio_gang_issue,
3122 zio_dva_allocate,
3123 zio_dva_free,

--- 50 unchanged lines hidden --- 		3201 zio_ddt_read_start,
3202 zio_ddt_read_done,
3203 zio_ddt_write,
3204 zio_ddt_free,
3205 zio_gang_assemble,
3206 zio_gang_issue,
3207 zio_dva_allocate,
3208 zio_dva_free,

--- 50 unchanged lines hidden ---