1#!/bin/ksh -p 2# 3# CDDL HEADER START 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# http://www.illumos.org/license/CDDL. 13# 14# CDDL HEADER END 15# 16 17# 18# Copyright (c) 2019 Datto, Inc. All rights reserved. 19# Copyright (c) 2022 Axcient. 20# 21 22. $STF_SUITE/include/libtest.shlib 23 24# 25# DESCRIPTION: 26# OpenZFS should be able to heal data using corrective recv when the send file 27# was generated with the --compressed flag 28# 29# STRATEGY: 30# 0. Create a file, checksum the file to be corrupted then compare it's checksum 31# with the one obtained after healing under different testing scenarios: 32# 1. Test healing (aka corrective) recv from a full send file 33# 2. Test healing recv (aka heal recv) from an incremental send file 34# 3. Test healing recv when compression on-disk is off but source was compressed 35# 4. Test heal recv when compression on-disk is on but source was uncompressed 36# 5. Test heal recv when compression doesn't match between send file and on-disk 37# 6. Test healing recv of an encrypted dataset using an unencrypted send file 38# 7. Test healing recv (on an encrypted dataset) using a raw send file 39# 8. Test healing when specifying destination filesystem only (no snapshot) 40# 9. Test incremental recv aftear healing recv 41# 42 43verify_runnable "both" 44 45DISK=${DISKS%% *} 46 47backup=$TEST_BASE_DIR/backup 48raw_backup=$TEST_BASE_DIR/raw_backup 49ibackup=$TEST_BASE_DIR/ibackup 50unc_backup=$TEST_BASE_DIR/unc_backup 51 52function cleanup 53{ 54 log_must rm -f $backup $raw_backup $ibackup $unc_backup 55 56 poolexists $TESTPOOL && destroy_pool $TESTPOOL 57 log_must zpool create -f $TESTPOOL $DISK 58} 59 60function test_corrective_recv 61{ 62 log_must zpool scrub -w $TESTPOOL 63 log_must zpool status -v $TESTPOOL 64 log_must eval "zpool status -v $TESTPOOL | \ 65 grep \"Permanent errors have been detected\"" 66 67 # make sure we will read the corruption from disk by flushing the ARC 68 log_must zinject -a 69 70 log_must eval "zfs recv -c $1 < $2" 71 72 log_must zpool scrub -w $TESTPOOL 73 log_must zpool status -v $TESTPOOL 74 log_mustnot eval "zpool status -v $TESTPOOL | \ 75 grep \"Permanent errors have been detected\"" 76 typeset cksum=$(md5digest $file) 77 [[ "$cksum" == "$checksum" ]] || \ 78 log_fail "Checksums differ ($cksum != $checksum)" 79} 80 81log_onexit cleanup 82 83log_assert "ZFS corrective receive should be able to heal data corruption" 84 85typeset passphrase="password" 86typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0" 87 88log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL" 89log_must zpool create -f -o feature@head_errlog=disabled $TESTPOOL $DISK 90 91log_must eval "echo $passphrase > /$TESTPOOL/pwd" 92 93log_must zfs create -o primarycache=none \ 94 -o atime=off -o compression=lz4 $TESTPOOL/$TESTFS1 95 96log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync 97log_must eval "echo 'aaaaaaaa' >> "$file 98typeset checksum=$(md5digest $file) 99 100log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1 101 102# create full send file 103log_must eval "zfs send --compressed $TESTPOOL/$TESTFS1@snap1 > $backup" 104 105log_must dd if=/dev/urandom of=$file"1" bs=1024 count=1024 oflag=sync 106log_must eval "echo 'bbbbbbbb' >> "$file"1" 107log_must zfs snapshot $TESTPOOL/$TESTFS1@snap2 108# create incremental send file 109log_must eval "zfs send -c -i $TESTPOOL/$TESTFS1@snap1 \ 110 $TESTPOOL/$TESTFS1@snap2 > $ibackup" 111 112corrupt_blocks_at_level $file 0 113# test healing recv from a full send file 114test_corrective_recv $TESTPOOL/$TESTFS1@snap1 $backup 115 116corrupt_blocks_at_level $file"1" 0 117# test healing recv from an incremental send file 118test_corrective_recv $TESTPOOL/$TESTFS1@snap2 $ibackup 119 120# create new uncompressed dataset using our send file 121log_must eval "zfs recv -o compression=off -o primarycache=none \ 122 $TESTPOOL/$TESTFS2 < $backup" 123typeset compr=$(get_prop compression $TESTPOOL/$TESTFS2) 124[[ "$compr" == "off" ]] || \ 125 log_fail "Unexpected compression $compr in recved dataset" 126corrupt_blocks_at_level "/$TESTPOOL/$TESTFS2/$TESTFILE0" 0 127# test healing recv when compression on-disk is off but source was compressed 128test_corrective_recv "$TESTPOOL/$TESTFS2@snap1" $backup 129 130# create a full sendfile from an uncompressed source 131log_must eval "zfs send --compressed $TESTPOOL/$TESTFS2@snap1 > $unc_backup" 132log_must eval "zfs recv -o compression=gzip -o primarycache=none \ 133 $TESTPOOL/testfs3 < $unc_backup" 134typeset compr=$(get_prop compression $TESTPOOL/testfs3) 135[[ "$compr" == "gzip" ]] || \ 136 log_fail "Unexpected compression $compr in recved dataset" 137corrupt_blocks_at_level "/$TESTPOOL/testfs3/$TESTFILE0" 0 138# test healing recv when compression on-disk is on but source was uncompressed 139test_corrective_recv "$TESTPOOL/testfs3@snap1" $unc_backup 140 141# create new compressed dataset using our send file 142log_must eval "zfs recv -o compression=gzip -o primarycache=none \ 143 $TESTPOOL/testfs4 < $backup" 144typeset compr=$(get_prop compression $TESTPOOL/testfs4) 145[[ "$compr" == "gzip" ]] || \ 146 log_fail "Unexpected compression $compr in recved dataset" 147corrupt_blocks_at_level "/$TESTPOOL/testfs4/$TESTFILE0" 0 148# test healing recv when compression doesn't match between send file and on-disk 149test_corrective_recv "$TESTPOOL/testfs4@snap1" $backup 150 151# create new encrypted (and compressed) dataset using our send file 152log_must eval "zfs recv -o encryption=aes-256-ccm -o keyformat=passphrase \ 153 -o keylocation=file:///$TESTPOOL/pwd -o primarycache=none \ 154 $TESTPOOL/testfs5 < $backup" 155typeset encr=$(get_prop encryption $TESTPOOL/testfs5) 156[[ "$encr" == "aes-256-ccm" ]] || \ 157 log_fail "Unexpected encryption $encr in recved dataset" 158log_must eval "zfs send --raw $TESTPOOL/testfs5@snap1 > $raw_backup" 159log_must eval "zfs send --compressed $TESTPOOL/testfs5@snap1 > $backup" 160corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 161# test healing recv of an encrypted dataset using an unencrypted send file 162test_corrective_recv "$TESTPOOL/testfs5@snap1" $backup 163corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 164log_must zfs unmount $TESTPOOL/testfs5 165log_must zfs unload-key $TESTPOOL/testfs5 166# test healing recv (on an encrypted dataset) using a raw send file 167test_corrective_recv "$TESTPOOL/testfs5@snap1" $raw_backup 168# non raw send file healing an encrypted dataset with an unloaded key will fail 169log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap1 < $backup" 170 171log_must zfs rollback -r $TESTPOOL/$TESTFS1@snap1 172corrupt_blocks_at_level $file 0 173# test healing when specifying destination filesystem only (no snapshot) 174test_corrective_recv $TESTPOOL/$TESTFS1 $backup 175# test incremental recv aftear healing recv 176log_must eval "zfs recv $TESTPOOL/$TESTFS1 < $ibackup" 177 178# test that healing recv can not be combined with incompatible recv options 179log_mustnot eval "zfs recv -h -c $TESTPOOL/$TESTFS1@snap1 < $backup" 180log_mustnot eval "zfs recv -F -c $TESTPOOL/$TESTFS1@snap1 < $backup" 181log_mustnot eval "zfs recv -s -c $TESTPOOL/$TESTFS1@snap1 < $backup" 182log_mustnot eval "zfs recv -u -c $TESTPOOL/$TESTFS1@snap1 < $backup" 183log_mustnot eval "zfs recv -d -c $TESTPOOL/$TESTFS1@snap1 < $backup" 184log_mustnot eval "zfs recv -e -c $TESTPOOL/$TESTFS1@snap1 < $backup" 185 186# ensure healing recv doesn't work when snap GUIDS don't match 187log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap2 < $backup" 188log_mustnot eval "zfs recv -c $TESTPOOL/testfs5 < $backup" 189 190# test that healing recv doesn't work on non-existing snapshots 191log_mustnot eval "zfs recv -c $TESTPOOL/$TESTFS1@missing < $backup" 192 193log_pass "OpenZFS corrective recv works for data healing" 194