1#!/usr/local/bin/ksh93 -p
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23#
24# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
25# Copyright 2014 Spectra Logic Corp.  All rights reserved.
26# Use is subject to license terms.
27#
28. $STF_SUITE/tests/hotspare/hotspare.kshlib
29
30################################################################################
31#
32# __stc_assertion_start
33#
34# ID: zfsd_hotspare_001_pos
35#
36# DESCRIPTION: 
37#	If an active spare fails, it will be replaced by an available spare.
38#
39# STRATEGY:
40#	1. Create a storage pool with two hot spares
41#	2. Fail one vdev
42#	3. Verify that a spare gets activated
43#	4. Fail the spare
44#	5. Verify the failed spare was replaced by the other spare.
45#
46# TESTABILITY: explicit
47#
48# TEST_AUTOMATION_LEVEL: automated
49#
50# CODING STATUS: COMPLETED (2014-05-13)
51#
52# __stc_assertion_end
53#
54###############################################################################
55
56verify_runnable "global"
57
58function cleanup
59{
60	if poolexists $TESTPOOL ; then 
61		destroy_pool $TESTPOOL
62	fi
63
64	partition_cleanup
65}
66
67
68log_onexit cleanup
69
70function verify_assertion # type
71{
72	typeset pool_type=$1
73
74	typeset err_dev=${devarray[3]}
75	typeset raidz2_dev="${devarray[4]}"
76	typeset mntp=$(get_prop mountpoint $TESTPOOL)
77 
78	# fail a basic vdev
79	$ZINJECT -d $err_dev -A fault $TESTPOOL
80
81	# ZFSD can take up to 60 seconds to replace a failed device
82	# (though it's usually faster).  
83	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
84		check_state $TESTPOOL "$fail_spare" "INUSE"
85		spare_inuse=$?
86		if [[ $spare_inuse == 0 ]]; then
87			break
88		fi
89		$SLEEP 6
90	done
91	log_must $ZPOOL status $TESTPOOL
92	log_must check_state $TESTPOOL "$fail_spare" "INUSE"
93
94	# The zpool history should log when a spare device becomes active
95	log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \
96	$GREP "spare in vdev=$fail_spare for vdev=$err_dev" > /dev/null
97
98	######################################################################
99	# Now fail the active hotspare, and check that the second comes online
100	######################################################################
101
102	# fail the spare vdev
103	$ZINJECT -d $fail_spare -A fault $TESTPOOL
104
105	# ZFSD can take up to 60 seconds to replace a failed device
106	# (though it's usually faster).  
107	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
108		check_state $TESTPOOL "$standby_spare" "INUSE"
109		spare_inuse=$?
110		if [[ $spare_inuse == 0 ]]; then
111			break
112		fi
113		$SLEEP 6
114	done
115	log_must $ZPOOL status $TESTPOOL
116
117	# The standby spare should be in use, while the original spare should
118	# be faulted.
119	log_must check_state $TESTPOOL $standby_spare "online"
120	log_must check_state $TESTPOOL $standby_spare "INUSE"
121	log_mustnot check_state $TESTPOOL $fail_spare "online"
122
123	# The zpool history should log when a spare device becomes active
124	log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \
125		$GREP "spare in vdev=$standby_spare for vdev=$fail_spare" > /dev/null
126
127	# do cleanup
128	destroy_pool $TESTPOOL
129}
130
131log_onexit cleanup
132
133log_assert "An active damaged spare will be replaced by an available spare"
134
135ensure_zfsd_running
136set_devs
137
138typeset  fail_spare="${devarray[0]}"
139typeset  standby_spare="${devarray[1]}"
140typeset  spares="$fail_spare $standby_spare"
141
142set -A my_keywords "mirror" "raidz1" "raidz2"
143
144for keyword in "${my_keywords[@]}"; do
145	setup_hotspares "$keyword"
146	verify_assertion "$keyword"
147done
148
149log_pass "If one of the spare fail, the other available spare will be in use"
150