zfs/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib

#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/redundancy/redundancy.cfg

function cleanup
{
	if poolexists $TESTPOOL; then
		destroy_pool $TESTPOOL
	fi
	typeset dir
	for dir in $TESTDIR $BASEDIR; do
		if [[ -d $dir ]]; then
			log_must rm -rf $dir
		fi
	done
}

#
# Get random number between min and max number.
#
# $1 Minimal value
# $2 Maximal value
#
function random
{
	typeset -i min=$1
	typeset -i max=$2
	typeset -i value

	while true; do
		((value = RANDOM % (max + 1)))
		if ((value >= min)); then
			break
		fi
	done

	echo $value
}

#
# Get the number of checksum errors for the pool.
#
# $1 Pool
#
function cksum_pool
{
	typeset -i cksum=$(zpool status $1 | awk '
	    !NF { isvdev = 0 }
	    isvdev { errors += $NF }
	    /CKSUM$/ { isvdev = 1 }
	    END { print errors }
	')

	echo $cksum
}

#
# Record the directories construction and checksum all the files which reside
# within the specified pool
#
# $1 The specified pool
# $2 The file which save the record.
#
function record_data
{
	typeset pool=$1
	typeset recordfile=$2

	[[ -z $pool ]] && log_fail "No specified pool."
	[[ -f $recordfile ]] && log_must rm -f $recordfile

	sync_pool $pool
	typeset mntpnt
	mntpnt=$(get_prop mountpoint $pool)
	log_must eval "du -a $mntpnt > $recordfile 2>&1"
	#
	# When the data was damaged, checksum is failing and return 1
	# So, will not use log_must
	#
	find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1
}

#
# Create test pool and fill with files and directories.
#
# $1 pool name
# $2 pool type
# $3 virtual devices number
#
function setup_test_env
{
	typeset pool=$1
	typeset keyword=$2
	typeset -i vdev_cnt=$3
	typeset vdevs

	typeset -i i=0
	while (( i < vdev_cnt )); do
		vdevs="$vdevs $BASEDIR/vdev$i"
		((i += 1))
	done

	if [[ ! -d $BASEDIR ]]; then
		log_must mkdir $BASEDIR
	fi

	if poolexists $pool ; then
		destroy_pool $pool
	fi

	log_must truncate -s $MINVDEVSIZE $vdevs

	log_must zpool create -f -m $TESTDIR $pool $keyword $vdevs

	log_note "Filling up the filesystem ..."
	typeset -i ret=0
	typeset -i i=0
	typeset file=$TESTDIR/file
	typeset -i limit
	(( limit = $(get_prop available $pool) / 2 ))

	while true ; do
		[[ $(get_prop available $pool) -lt $limit ]] && break
		file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES
		ret=$?
		(( $ret != 0 )) && break
		(( i = i + 1 ))
	done

	record_data $TESTPOOL $PRE_RECORD_FILE
}

function refill_test_env
{
	log_note "Re-filling the filesystem ..."
	typeset pool=$1
	typeset -i ret=0
	typeset -i i=0
	typeset mntpnt
	mntpnt=$(get_prop mountpoint $pool)
	typeset file=$mntpnt/file
	while [[ -e $file.$i ]]; do
		log_must rm -f $file.$i
		file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES
		ret=$?
		(( $ret != 0 )) && break
		(( i = i + 1 ))
	done

	record_data $TESTPOOL $PRE_RECORD_FILE
}

#
# Check pool status is healthy
#
# $1 pool
#
function is_healthy
{
	typeset pool=$1

	typeset healthy_output="pool '$pool' is healthy"
	typeset real_output=$(zpool status -x $pool)

	if [[ "$real_output" == "$healthy_output" ]]; then
		return 0
	else
		typeset -i ret
		zpool status -x $pool | grep "state:" | \
			grep "FAULTED" >/dev/null 2>&1
		ret=$?
		(( $ret == 0 )) && return 1
		typeset l_scan
		typeset errnum
		l_scan=$(zpool status -x $pool | grep "scan:")
		l_scan=${l_scan##*"with"}
		errnum=$(echo $l_scan | awk '{print $1}')

		return $errnum
	fi
}

#
# Check pool data is valid
#
# $1 pool
#
function is_data_valid
{
	typeset pool=$1

	log_must zpool scrub -w $pool

	record_data $pool $PST_RECORD_FILE
	if ! cmp $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null; then
		log_must cat $PRE_RECORD_FILE
		log_must cat $PST_RECORD_FILE
		diff -u $PRE_RECORD_FILE $PST_RECORD_FILE
		return 1
	fi

	return 0
}

#
# Get the specified count devices name
#
# $1 pool name
# $2 devices count
#
function get_vdevs #pool cnt
{
	typeset pool=$1
	typeset -i cnt=$2

	typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \
		grep -vEe "^pool$|^capacity$|^mirror\-[0-9]$|^raidz[1-3]\-[0-9]$|^draid[1-3].*\-[0-9]$|---" | \
		grep -vEe "/old$|^$pool$")
	typeset -i i=0
	typeset vdevs
	while ((i < cnt)); do
		typeset dev=$(echo $all_devs | awk '{print $1}')
		eval all_devs=\${all_devs##*$dev}

		vdevs="$dev $vdevs"
		((i += 1))
	done

	echo "$vdevs"
}

#
# Create and replace the same name virtual device files
#
# $1 pool name
# $2-n virtual device files
#
function replace_missing_devs
{
	typeset pool=$1
	shift

	typeset vdev
	for vdev in $@; do
		log_must dd if=/dev/zero of=$vdev \
		    bs=1024k count=$((MINVDEVSIZE / (1024 * 1024))) \
		    conv=fdatasync
		log_must zpool replace -wf $pool $vdev $vdev
	done
}

#
# Damage the pool's virtual device files.
#
# $1 pool name
# $2 Failing devices count
# $3 damage vdevs method, if not null, we keep
#    the label for the vdevs
#
function damage_devs
{
	typeset pool=$1
	typeset -i cnt=$2
	typeset label="$3"
	typeset vdevs
	typeset -i bs_count=$(((MINVDEVSIZE / 1024) - 4096))

	vdevs=$(get_vdevs $pool $cnt)
	typeset dev
	if [[ -n $label ]]; then
		for dev in $vdevs; do
			log_must dd if=/dev/zero of=$dev seek=512 bs=1024 \
			    count=$bs_count conv=notrunc >/dev/null 2>&1
		done
	else
		for dev in $vdevs; do
			log_must dd if=/dev/zero of=$dev bs=1024 \
			    count=$bs_count conv=notrunc >/dev/null 2>&1
		done
	fi

	sync_pool $pool
}

#
# Clear errors in the pool caused by data corruptions
#
# $1 pool name
#
function clear_errors
{
	typeset pool=$1

	log_must zpool clear $pool

	if ! is_healthy $pool ; then
		log_note "$pool should be healthy."
		return 1
	fi
	if ! is_data_valid $pool ; then
		log_note "Data should be valid in $pool."
		return 1
	fi

	return 0
}

#
# Remove the specified pool's virtual device files
#
# $1 Pool name
# $2 Missing devices count
#
function remove_devs
{
	typeset pool=$1
	typeset -i cnt=$2
	typeset vdevs

	vdevs=$(get_vdevs $pool $cnt)
	log_must rm -f $vdevs

	sync_pool $pool
}

#
# Recover the bad or missing device files in the pool
#
# $1 Pool name
# $2 Missing devices count
#
function recover_bad_missing_devs
{
	typeset pool=$1
	typeset -i cnt=$2
	typeset vdevs

	vdevs=$(get_vdevs $pool $cnt)
	replace_missing_devs $pool $vdevs

	if ! is_healthy $pool ; then
		log_note "$pool should be healthy."
		return 1
	fi
	if ! is_data_valid $pool ; then
		log_note "Data should be valid in $pool."
		return 1
	fi

	return 0
}