1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 23 unchanged lines hidden (view full) ---

32#include <sys/vdev_impl.h>
33#include <sys/zio_impl.h>
34#include <sys/zio_compress.h>
35#include <sys/zio_checksum.h>
36#include <sys/dmu_objset.h>
37#include <sys/arc.h>
38#include <sys/ddt.h>
39#include <sys/trim_map.h>
40#include <sys/zfeature.h>
41
42SYSCTL_DECL(_vfs_zfs);
43SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO");
44#if defined(__amd64__)
45static int zio_use_uma = 1;
46#else
47static int zio_use_uma = 0;

--- 210 unchanged lines hidden (view full) ---

258 * excess / transient data in-core during a crashdump.
259 */
260void *
261zio_buf_alloc(size_t size)
262{
263 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
264 int flags = zio_exclude_metadata ? KM_NODEBUG : 0;
265
266 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
267
268 if (zio_use_uma)
269 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE));
270 else
271 return (kmem_alloc(size, KM_SLEEP|flags));
272}
273
274/*

--- 418 unchanged lines hidden (view full) ---

693 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
694 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
695 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE);
696
697 zio->io_ready = ready;
698 zio->io_physdone = physdone;
699 zio->io_prop = *zp;
700
701 return (zio);
702}
703
704zio_t *
705zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data,
706 uint64_t size, zio_done_func_t *done, void *private,
707 zio_priority_t priority, enum zio_flag flags, zbookmark_t *zb)
708{

--- 23 unchanged lines hidden (view full) ---

732 zio->io_prop.zp_nopwrite = nopwrite;
733 zio->io_prop.zp_copies = copies;
734 zio->io_bp_override = bp;
735}
736
737void
738zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
739{
740 metaslab_check_free(spa, bp);
741
742 /*
743 * Frees that are for the currently-syncing txg, are not going to be
744 * deferred, and which will not need to do a read (i.e. not GANG or
745 * DEDUP), can be processed immediately. Otherwise, put them on the
746 * in-memory list for later processing.
747 */

--- 9 unchanged lines hidden (view full) ---

757
758zio_t *
759zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
760 uint64_t size, enum zio_flag flags)
761{
762 zio_t *zio;
763 enum zio_stage stage = ZIO_FREE_PIPELINE;
764
765 dprintf_bp(bp, "freeing in txg %llu, pass %u",
766 (longlong_t)txg, spa->spa_sync_pass);
767
768 ASSERT(!BP_IS_HOLE(bp));
769 ASSERT(spa_syncing_txg(spa) == txg);
770 ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free);
771
772 metaslab_check_free(spa, bp);
773 arc_freed(spa, bp);
774
775 if (zfs_trim_enabled)
776 stage |= ZIO_STAGE_ISSUE_ASYNC | ZIO_STAGE_VDEV_IO_START |
777 ZIO_STAGE_VDEV_IO_ASSESS;
778 /*
779 * GANG and DEDUP blocks can induce a read (for the gang block header,

--- 13 unchanged lines hidden (view full) ---

793}
794
795zio_t *
796zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
797 zio_done_func_t *done, void *private, enum zio_flag flags)
798{
799 zio_t *zio;
800
801 /*
802 * A claim is an allocation of a specific block. Claims are needed
803 * to support immediate writes in the intent log. The issue is that
804 * immediate writes contain committed data, but in a txg that was
805 * *not* committed. Upon opening the pool after an unclean shutdown,
806 * the intent log claims all blocks that contain immediate write data
807 * so that the SPA knows they're in use.
808 *

--- 208 unchanged lines hidden (view full) ---

1017zio_read_bp_init(zio_t **ziop)
1018{
1019 zio_t *zio = *ziop;
1020 blkptr_t *bp = zio->io_bp;
1021
1022 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF &&
1023 zio->io_child_type == ZIO_CHILD_LOGICAL &&
1024 !(zio->io_flags & ZIO_FLAG_RAW)) {
1025 uint64_t psize = BP_GET_PSIZE(bp);
1026 void *cbuf = zio_buf_alloc(psize);
1027
1028 zio_push_transform(zio, cbuf, psize, psize, zio_decompress);
1029 }
1030
1031 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0)
1032 zio->io_flags |= ZIO_FLAG_DONT_CACHE;
1033
1034 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP)
1035 zio->io_flags |= ZIO_FLAG_DONT_CACHE;
1036
1037 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
1038 zio->io_pipeline = ZIO_DDT_READ_PIPELINE;

--- 28 unchanged lines hidden (view full) ---

1067
1068 if (zio->io_bp_override) {
1069 ASSERT(bp->blk_birth != zio->io_txg);
1070 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0);
1071
1072 *bp = *zio->io_bp_override;
1073 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1074
1075 /*
1076 * If we've been overridden and nopwrite is set then
1077 * set the flag accordingly to indicate that a nopwrite
1078 * has already occurred.
1079 */
1080 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) {
1081 ASSERT(!zp->zp_dedup);
1082 zio->io_flags |= ZIO_FLAG_NOPWRITE;

--- 32 unchanged lines hidden (view full) ---

1115 ASSERT(zio->io_txg == spa_syncing_txg(spa));
1116 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1117 ASSERT(!BP_GET_DEDUP(bp));
1118
1119 if (pass >= zfs_sync_pass_dont_compress)
1120 compress = ZIO_COMPRESS_OFF;
1121
1122 /* Make sure someone doesn't change their mind on overwrites */
1123 ASSERT(MIN(zp->zp_copies + BP_IS_GANG(bp),
1124 spa_max_replication(spa)) == BP_GET_NDVAS(bp));
1125 }
1126
1127 if (compress != ZIO_COMPRESS_OFF) {
1128 metaslab_class_t *mc = spa_normal_class(spa);
1129 void *cbuf = zio_buf_alloc(lsize);
1130 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize,
1131 (size_t)metaslab_class_get_minblocksize(mc));
1132 if (psize == 0 || psize == lsize) {
1133 compress = ZIO_COMPRESS_OFF;
1134 zio_buf_free(cbuf, lsize);
1135 } else {
1136 ASSERT(psize < lsize);
1137 zio_push_transform(zio, cbuf, psize, lsize, NULL);
1138 }
1139 }
1140
1141 /*
1142 * The final pass of spa_sync() must be all rewrites, but the first
1143 * few passes offer a trade-off: allocating blocks defers convergence,
1144 * but newly allocated blocks are sequential, so they can be written
1145 * to disk faster. Therefore, we allow the first few passes of

--- 1757 unchanged lines hidden (view full) ---

2903zio_checksum_verified(zio_t *zio)
2904{
2905 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
2906}
2907
2908/*
2909 * ==========================================================================
2910 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other.
2911 * An error of 0 indictes success. ENXIO indicates whole-device failure,
2912 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO
2913 * indicate errors that are specific to one I/O, and most likely permanent.
2914 * Any other error is presumed to be worse because we weren't expecting it.
2915 * ==========================================================================
2916 */
2917int
2918zio_worst_error(int e1, int e2)
2919{

--- 95 unchanged lines hidden (view full) ---

3015 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) ||
3016 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE))
3017 return (ZIO_PIPELINE_STOP);
3018
3019 for (int c = 0; c < ZIO_CHILD_TYPES; c++)
3020 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
3021 ASSERT(zio->io_children[c][w] == 0);
3022
3023 if (bp != NULL) {
3024 ASSERT(bp->blk_pad[0] == 0);
3025 ASSERT(bp->blk_pad[1] == 0);
3026 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 ||
3027 (bp == zio_unique_parent(zio)->io_bp));
3028 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) &&
3029 zio->io_bp_override == NULL &&
3030 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {
3031 ASSERT(!BP_SHOULD_BYTESWAP(bp));

--- 316 unchanged lines hidden ---