1#!/usr/local/bin/ksh93 -p 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 25# Copyright 2014 Spectra Logic Corp. All rights reserved. 26# Use is subject to license terms. 27# 28. $STF_SUITE/tests/hotspare/hotspare.kshlib 29 30################################################################################ 31# 32# __stc_assertion_start 33# 34# ID: zfsd_hotspare_001_pos 35# 36# DESCRIPTION: 37# If an active spare fails, it will be replaced by an available spare. 38# 39# STRATEGY: 40# 1. Create a storage pool with two hot spares 41# 2. Fail one vdev 42# 3. Verify that a spare gets activated 43# 4. Fail the spare 44# 5. Verify the failed spare was replaced by the other spare. 45# 46# TESTABILITY: explicit 47# 48# TEST_AUTOMATION_LEVEL: automated 49# 50# CODING STATUS: COMPLETED (2014-05-13) 51# 52# __stc_assertion_end 53# 54############################################################################### 55 56verify_runnable "global" 57 58function cleanup 59{ 60 if poolexists $TESTPOOL ; then 61 destroy_pool $TESTPOOL 62 fi 63 64 partition_cleanup 65} 66 67 68log_onexit cleanup 69 70function verify_assertion # type 71{ 72 typeset pool_type=$1 73 74 typeset err_dev=${devarray[3]} 75 typeset raidz2_dev="${devarray[4]}" 76 typeset mntp=$(get_prop mountpoint $TESTPOOL) 77 78 # fail a basic vdev 79 $ZINJECT -d $err_dev -A fault $TESTPOOL 80 81 # ZFSD can take up to 60 seconds to replace a failed device 82 # (though it's usually faster). 83 for ((timeout=0; $timeout<10; timeout=$timeout+1)); do 84 check_state $TESTPOOL "$fail_spare" "INUSE" 85 spare_inuse=$? 86 if [[ $spare_inuse == 0 ]]; then 87 break 88 fi 89 $SLEEP 6 90 done 91 log_must $ZPOOL status $TESTPOOL 92 log_must check_state $TESTPOOL "$fail_spare" "INUSE" 93 94 # The zpool history should log when a spare device becomes active 95 log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \ 96 $GREP "spare in vdev=$fail_spare for vdev=$err_dev" > /dev/null 97 98 ###################################################################### 99 # Now fail the active hotspare, and check that the second comes online 100 ###################################################################### 101 102 # fail the spare vdev 103 $ZINJECT -d $fail_spare -A fault $TESTPOOL 104 105 # ZFSD can take up to 60 seconds to replace a failed device 106 # (though it's usually faster). 107 for ((timeout=0; $timeout<10; timeout=$timeout+1)); do 108 check_state $TESTPOOL "$standby_spare" "INUSE" 109 spare_inuse=$? 110 if [[ $spare_inuse == 0 ]]; then 111 break 112 fi 113 $SLEEP 6 114 done 115 log_must $ZPOOL status $TESTPOOL 116 117 # The standby spare should be in use, while the original spare should 118 # be faulted. 119 log_must check_state $TESTPOOL $standby_spare "online" 120 log_must check_state $TESTPOOL $standby_spare "INUSE" 121 log_mustnot check_state $TESTPOOL $fail_spare "online" 122 123 # The zpool history should log when a spare device becomes active 124 log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \ 125 $GREP "spare in vdev=$standby_spare for vdev=$fail_spare" > /dev/null 126 127 # do cleanup 128 destroy_pool $TESTPOOL 129} 130 131log_onexit cleanup 132 133log_assert "An active damaged spare will be replaced by an available spare" 134 135ensure_zfsd_running 136set_devs 137 138typeset fail_spare="${devarray[0]}" 139typeset standby_spare="${devarray[1]}" 140typeset spares="$fail_spare $standby_spare" 141 142set -A my_keywords "mirror" "raidz1" "raidz2" 143 144for keyword in "${my_keywords[@]}"; do 145 setup_hotspares "$keyword" 146 verify_assertion "$keyword" 147done 148 149log_pass "If one of the spare fail, the other available spare will be in use" 150