1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 27 unchanged lines hidden (view full) ---

36#include <sys/zio_compress.h>
37#include <sys/zio_checksum.h>
38#include <sys/dmu_objset.h>
39#include <sys/arc.h>
40#include <sys/ddt.h>
41#include <sys/trim_map.h>
42#include <sys/blkptr.h>
43#include <sys/zfeature.h>
44#include <sys/metaslab_impl.h>
45#include <sys/abd.h>
46
47SYSCTL_DECL(_vfs_zfs);
48SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO");
49#if defined(__amd64__)
50static int zio_use_uma = 1;
51#else

--- 381 unchanged lines hidden (view full) ---

433 return ((*zl)->zl_parent);
434}
435
436zio_t *
437zio_walk_children(zio_t *pio, zio_link_t **zl)
438{
439 list_t *cl = &pio->io_child_list;
440
441 *zl = (*zl == NULL) ? list_head(cl) : list_next(cl, *zl);
442 if (*zl == NULL)
443 return (NULL);
444
445 ASSERT((*zl)->zl_parent == pio);
446 return ((*zl)->zl_child);
447}
448

--- 18 unchanged lines hidden (view full) ---

467 * Vdev I/Os can only have vdev children.
468 * The following ASSERT captures all of these constraints.
469 */
470 ASSERT3S(cio->io_child_type, <=, pio->io_child_type);
471
472 zl->zl_parent = pio;
473 zl->zl_child = cio;
474
475 mutex_enter(&cio->io_lock);
476 mutex_enter(&pio->io_lock);
477
478 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);
479
480 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
481 pio->io_children[cio->io_child_type][w] += !cio->io_state[w];
482
483 list_insert_head(&pio->io_child_list, zl);
484 list_insert_head(&cio->io_parent_list, zl);
485
486 pio->io_child_count++;
487 cio->io_parent_count++;
488
489 mutex_exit(&pio->io_lock);
490 mutex_exit(&cio->io_lock);
491}
492
493static void
494zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl)
495{
496 ASSERT(zl->zl_parent == pio);
497 ASSERT(zl->zl_child == cio);
498
499 mutex_enter(&cio->io_lock);
500 mutex_enter(&pio->io_lock);
501
502 list_remove(&pio->io_child_list, zl);
503 list_remove(&cio->io_parent_list, zl);
504
505 pio->io_child_count--;
506 cio->io_parent_count--;
507
508 mutex_exit(&pio->io_lock);
509 mutex_exit(&cio->io_lock);
510
511 kmem_cache_free(zio_link_cache, zl);
512}
513
514static boolean_t
515zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
516{
517 boolean_t waiting = B_FALSE;
518

--- 464 unchanged lines hidden (view full) ---

983 ASSERT(spa_syncing_txg(spa) == txg);
984 ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free);
985
986 if (BP_IS_EMBEDDED(bp))
987 return (zio_null(pio, spa, NULL, NULL, NULL, 0));
988
989 metaslab_check_free(spa, bp);
990 arc_freed(spa, bp);
991
992 if (zfs_trim_enabled)
993 stage |= ZIO_STAGE_ISSUE_ASYNC | ZIO_STAGE_VDEV_IO_START |
994 ZIO_STAGE_VDEV_IO_ASSESS;
995 /*
996 * GANG and DEDUP blocks can induce a read (for the gang block header,
997 * or the DDT), so issue them asynchronously so that this thread is
998 * not tied up.

--- 861 unchanged lines hidden (view full) ---

1860 /*
1861 * As we reexecute pio's children, new children could be created.
1862 * New children go to the head of pio's io_child_list, however,
1863 * so we will (correctly) not reexecute them. The key is that
1864 * the remainder of pio's io_child_list, from 'cio_next' onward,
1865 * cannot be affected by any side effects of reexecuting 'cio'.
1866 */
1867 zio_link_t *zl = NULL;
1868 for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
1869 cio_next = zio_walk_children(pio, &zl);
1870 mutex_enter(&pio->io_lock);
1871 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
1872 pio->io_children[cio->io_child_type][w]++;
1873 mutex_exit(&pio->io_lock);
1874 zio_reexecute(cio);
1875 }
1876
1877 /*
1878 * Now that all children have been reexecuted, execute the parent.
1879 * We don't reexecute "The Godfather" I/O here as it's the
1880 * responsibility of the caller to wait on it.
1881 */
1882 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) {
1883 pio->io_queued_timestamp = gethrtime();

--- 1295 unchanged lines hidden (view full) ---

3179
3180 if (zio->io_vd->vdev_removing) {
3181 ASSERT(zio->io_flags &
3182 (ZIO_FLAG_PHYSICAL | ZIO_FLAG_SELF_HEAL |
3183 ZIO_FLAG_INDUCE_DAMAGE));
3184 }
3185 }
3186
3187 /*
3188 * We keep track of time-sensitive I/Os so that the scan thread
3189 * can quickly react to certain workloads. In particular, we care
3190 * about non-scrubbing, top-level reads and writes with the following
3191 * characteristics:
3192 * - synchronous writes of user data to non-slog devices
3193 * - any reads of user data
3194 * When these conditions are met, adjust the timestamp of spa_last_io
3195 * which allows the scan thread to adjust its workload accordingly.
3196 */
3197 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL &&
3198 vd == vd->vdev_top && !vd->vdev_islog &&
3199 zio->io_bookmark.zb_objset != DMU_META_OBJSET &&
3200 zio->io_txg != spa_syncing_txg(spa)) {
3201 uint64_t old = spa->spa_last_io;
3202 uint64_t new = ddi_get_lbolt64();
3203 if (old != new)
3204 (void) atomic_cas_64(&spa->spa_last_io, old, new);
3205 }
3206
3207 align = 1ULL << vd->vdev_top->vdev_ashift;
3208
3209 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) &&
3210 P2PHASE(zio->io_size, align) != 0) {
3211 /* Transform logical writes to be a full physical block size. */
3212 uint64_t asize = P2ROUNDUP(zio->io_size, align);
3213 abd_t *abuf = NULL;
3214 if (zio->io_type == ZIO_TYPE_READ ||

--- 133 unchanged lines hidden (view full) ---

3348
3349 if (unexpected_error)
3350 VERIFY(vdev_probe(vd, zio) == NULL);
3351
3352 return (ZIO_PIPELINE_CONTINUE);
3353}
3354
3355/*
3356 * For non-raidz ZIOs, we can just copy aside the bad data read from the
3357 * disk, and use that to finish the checksum ereport later.
3358 */
3359static void
3360zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr,
3361 const void *good_buf)
3362{
3363 /* no processing needed */

--- 850 unchanged lines hidden ---