xref: /freebsd-current/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fault/auto_spare_multiple.ksh

#!/bin/ksh -p

#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#

#
# Copyright (c) 2017 by Intel Corporation. All rights reserved.
# Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/fault/fault.cfg

#
# DESCRIPTION:
# Testing Fault Management Agent ZED Logic - Automated Auto-Spare Test when
# multiple drives are faulted.
#
# STRATEGY:
# 1. Create a pool with two hot spares
# 2. Inject IO ERRORS with a zinject error handler on the first device
# 3. Start a scrub
# 4. Verify the ZED kicks in a hot spare and expected pool/device status
# 5. Inject IO ERRORS on a second device
# 6. Start a scrub
# 7. Verify the ZED kicks in a second hot spare
# 8. Clear the fault on both devices
# 9. Verify the hot spares are available and expected pool/device status
# 10. Rinse and repeat, this time faulting both devices at the same time
#

verify_runnable "both"

function cleanup
{
	log_must zinject -c all
	destroy_pool $TESTPOOL
	rm -f $DATA_DEVS $SPARE_DEVS
}

log_assert "ZED should be able to handle multiple faulted devices"
log_onexit cleanup

# Events not supported on FreeBSD
if ! is_freebsd; then
	# Clear events from previous runs
	zed_events_drain
fi

FAULT_DEV1="$TEST_BASE_DIR/fault-dev1"
FAULT_DEV2="$TEST_BASE_DIR/fault-dev2"
SAFE_DEV1="$TEST_BASE_DIR/safe-dev1"
SAFE_DEV2="$TEST_BASE_DIR/safe-dev2"
SAFE_DEV3="$TEST_BASE_DIR/safe-dev3"
SAFE_DEV4="$TEST_BASE_DIR/safe-dev4"
DATA_DEVS="$FAULT_DEV1 $FAULT_DEV2 $SAFE_DEV1 $SAFE_DEV2 $SAFE_DEV3 $SAFE_DEV4"
SPARE_DEV1="$TEST_BASE_DIR/spare-dev1"
SPARE_DEV2="$TEST_BASE_DIR/spare-dev2"
SPARE_DEVS="$SPARE_DEV1 $SPARE_DEV2"

for type in "mirror" "raidz" "raidz2" "raidz3" "draid2:1s"; do
	if [ "$type" = "draid2:1s" ]; then
		# 1. Create a dRAID pool with a distributed and traditional
		# hot spare to provide test coverage for both configurations.
		#
		# Corruption is injected in the third and fourth vdevs
		# since the dRAID permutation at these offsets maps to
		# distributed spare space and not data devices.
		#
		truncate -s $MINVDEVSIZE $DATA_DEVS $SPARE_DEV1
		log_must zpool create -f $TESTPOOL $type $SAFE_DEV1 \
		    $SAFE_DEV2 $FAULT_DEV1 $FAULT_DEV2 $SAFE_DEV3 $SAFE_DEV4 \
		    spare $SPARE_DEV1
		SPARE1=$SPARE_DEV1
		SPARE2="draid2-0-0"
	elif [ "$type" = "mirror" ]; then
		# 1. Create a 3-way mirror pool with two hot spares
		truncate -s $MINVDEVSIZE $DATA_DEVS $SPARE_DEVS
		log_must zpool create -f $TESTPOOL $type \
		    $FAULT_DEV1 $FAULT_DEV2 $SAFE_DEV1 spare $SPARE_DEVS
		SPARE1=$SPARE_DEV1
		SPARE2=$SPARE_DEV2
	else
		# 1. Create a raidz pool with two hot spares
		truncate -s $MINVDEVSIZE $DATA_DEVS $SPARE_DEVS
		log_must zpool create -f $TESTPOOL $type $DATA_DEVS \
		    spare $SPARE_DEVS
		SPARE1=$SPARE_DEV1
		SPARE2=$SPARE_DEV2
	fi

	# 2. Inject IO ERRORS with a zinject error handler on the first device
	log_must zinject -d $FAULT_DEV1 -e io -T all -f 100 $TESTPOOL

	# 3. Start a scrub
	log_must zpool scrub $TESTPOOL

	# 4. Verify the ZED kicks in a hot spare and the pool/device status
	log_note "Wait for ZED to auto-spare"
	log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "FAULTED" 60
	log_must wait_vdev_state $TESTPOOL $SPARE1 "ONLINE" 60
	log_must wait_hotspare_state $TESTPOOL $SPARE1 "INUSE"
	log_must check_state $TESTPOOL "" "DEGRADED"

	# 5. Inject IO ERRORS on a second device
	log_must zinject -d $FAULT_DEV2 -e io -T all -f 100 $TESTPOOL

	# 6. Start a scrub
	while is_pool_scrubbing $TESTPOOL || is_pool_resilvering $TESTPOOL; do
		sleep 1
	done
	log_must zpool scrub $TESTPOOL

	# 7. Verify the ZED kicks in a second hot spare
	log_note "Wait for ZED to auto-spare"
	log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "FAULTED" 60
	log_must wait_vdev_state $TESTPOOL $SPARE2 "ONLINE" 60
	log_must wait_hotspare_state $TESTPOOL $SPARE2 "INUSE"
	log_must check_state $TESTPOOL "" "DEGRADED"

	while is_pool_scrubbing $TESTPOOL || is_pool_resilvering $TESTPOOL; do
		sleep 1
	done

	# 8. Clear the fault on both devices
	log_must zinject -c all
	log_must zpool clear $TESTPOOL $FAULT_DEV1
	log_must zpool clear $TESTPOOL $FAULT_DEV2

	# 9. Verify the hot spares are available and expected pool/device status
	log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "ONLINE" 60
	log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "ONLINE" 60
	log_must wait_hotspare_state $TESTPOOL $SPARE1 "AVAIL"
	log_must wait_hotspare_state $TESTPOOL $SPARE2 "AVAIL"
	log_must check_state $TESTPOOL "" "ONLINE"

	# Cleanup
	cleanup
done

# Rinse and repeat, this time faulting both devices at the same time
# NOTE: "raidz" is excluded since it cannot survive 2 faulted devices
# NOTE: "mirror" is a 3-way mirror here and should survive this test
for type in "mirror" "raidz2" "raidz3" "draid2:1s"; do
	if [ "$type" = "draid2:1s" ]; then
		# 1. Create a dRAID pool with a distributed and traditional
		# hot spare to provide test coverage for both configurations.
		#
		# Corruption is injected in the third and fourth vdevs
		# since the dRAID permutation at these offsets maps to
		# distributed spare space and not data devices.
		#
		truncate -s $MINVDEVSIZE $DATA_DEVS $SPARE_DEV1
		log_must zpool create -f $TESTPOOL $type $SAFE_DEV1 \
		    $SAFE_DEV2 $FAULT_DEV1 $FAULT_DEV2 $SAFE_DEV3 $SAFE_DEV4 \
		    spare $SPARE_DEV1
		SPARE1=$SPARE_DEV1
		SPARE2="draid2-0-0"
	elif [ "$type" = "mirror" ]; then
		# 1. Create a 3-way mirror pool with two hot spares
		truncate -s $MINVDEVSIZE $DATA_DEVS $SPARE_DEVS
		log_must zpool create -f $TESTPOOL $type \
		    $FAULT_DEV1 $FAULT_DEV2 $SAFE_DEV1 spare $SPARE_DEVS
		SPARE1=$SPARE_DEV1
		SPARE2=$SPARE_DEV2
	else
		# 1. Create a raidz pool with two hot spares
		truncate -s $MINVDEVSIZE $DATA_DEVS $SPARE_DEVS
		log_must zpool create -f $TESTPOOL $type $DATA_DEVS \
		    spare $SPARE_DEVS
		SPARE1=$SPARE_DEV1
		SPARE2=$SPARE_DEV2
	fi

	# 2. Inject IO ERRORS with a zinject error handler on two devices
	log_must eval "zinject -d $FAULT_DEV1 -e io -T all -f 100 $TESTPOOL &"
	log_must eval "zinject -d $FAULT_DEV2 -e io -T all -f 100 $TESTPOOL &"

	# 3. Start a scrub
	log_must zpool scrub $TESTPOOL

	# 4. Verify the ZED kicks in two hot spares and the pool/device status
	log_note "Wait for ZED to auto-spare"
	log_must wait_vdev_state $TESTPOOL $FAULT_DEV1 "FAULTED" 60
	log_must wait_vdev_state $TESTPOOL $FAULT_DEV2 "FAULTED" 60
	log_must wait_vdev_state $TESTPOOL $SPARE1 "ONLINE" 60
	log_must wait_vdev_state $TESTPOOL $SPARE2 "ONLINE" 60
	log_must wait_hotspare_state $TESTPOOL $SPARE1 "INUSE"
	log_must wait_hotspare_state $TESTPOOL $SPARE2 "INUSE"
	log_must check_state $TESTPOOL "" "DEGRADED"

	# 5. Clear the fault on both devices
	log_must zinject -c all
	log_must zpool clear $TESTPOOL $FAULT_DEV1
	log_must zpool clear $TESTPOOL $FAULT_DEV2

	# Cleanup
	cleanup
done

log_pass "ZED successfully handles multiple faulted devices"