xref: /freebsd-10.2-release/usr.bin/bmake/tests/common.sh

#!/bin/sh
#
# Common code used run regression tests for usr.bin/make.
#
# $FreeBSD: head/tools/regression/usr.bin/make/common.sh 201526 2010-01-04 18:57:22Z obrien $

#
# Output a message and exit with an error.
#
fatal()
{
	echo "fatal: $*" >/dev/stderr
	exit 1
}

#
# Check whether the working directory exists - it must.
#
ensure_workdir()
{
	if [ ! -d ${WORK_DIR} ] ; then
		fatal "working directory ${WORK_DIR} does not exist."
	fi
}

#
# Make sure all tests have been run
#
ensure_run()
{
	if [ -z "${TEST_N}" ] ; then
		TEST_N=1
	fi

	FAIL=
	N=1
	while [ ${N} -le ${TEST_N} ] ; do
		if ! skip_test ${N} ; then
			if [ ! -f ${OUTPUT_DIR}/status.${N} -o \
			     ! -f ${OUTPUT_DIR}/stdout.${N} -o \
			     ! -f ${OUTPUT_DIR}/stderr.${N} ] ; then
				echo "Test ${SUBDIR}/${N} no yet run"
				FAIL=yes
			fi
		fi
		N=$((N + 1))
	done

	if [ ! -z "${FAIL}" ] ; then
		exit 1
	fi
}

#
# Output usage messsage.
#
print_usage()
{
	echo "Usage: sh -v -m <path> -w <dir> $0 command(s)"
	echo " setup	- setup working directory"
	echo " run	- run the tests"
	echo " show	- show test results"
	echo " compare	- compare actual and expected results"
	echo " diff	- diff actual and expected results"
	echo " reset	- reset the test to its initial state"
	echo " clean	- delete working and output directory"
	echo " test	- setup + run + compare"
	echo " prove	- setup + run + compare + clean"
	echo " desc	- print short description"
	echo " update	- update the expected results with the current results"
	echo " help	- show this information"
}

#
# Return 0 if we should skip the test. 1 otherwise
#
skip_test()
{
	eval skip=\${TEST_${1}_SKIP}
	if [ -z "${skip}" ] ; then
		return 1
	else
		return 0
	fi
}

#
# Common function for setup and reset.
#
common_setup()
{
	#
	# If a Makefile exists in the source directory - copy it over
	#
	if [ -e Makefile -a ! -e ${WORK_DIR}/Makefile ] ; then
		cp Makefile ${WORK_DIR}/Makefile
	fi

	#
	# If the TEST_MAKE_DIRS variable is set, create those directories
	#
	set -- ${TEST_MAKE_DIRS}
	while [ $# -ne 0 ] ; do
		if [ ! -d ${WORK_DIR}/${1} ] ; then
			mkdir -p -m ${2} ${WORK_DIR}/${1}
		else
			chmod ${2} ${WORK_DIR}/${1}
		fi
		shift ; shift
	done

	#
	# If the TEST_COPY_FILES variable is set, copy those files over to
	# the working directory. The value is assumed to be pairs of
	# filenames and modes.
	#
	set -- ${TEST_COPY_FILES}
	while [ $# -ne 0 ] ; do
		if [ ! -e ${WORK_DIR}/${1} ] ; then
			cp ${1} ${WORK_DIR}/${1}
		fi
		chmod ${2} ${WORK_DIR}/${1}
		shift ; shift
	done

	#
	# If the TEST_TOUCH variable is set, it is taken to be a list
	# of pairs of filenames and arguments to touch(1). The arguments
	# to touch must be surrounded by single quotes if there are more
	# than one argument.
	#
	eval set -- ${TEST_TOUCH}
	while [ $# -ne 0 ] ; do
		eval touch ${2} ${WORK_DIR}/${1}
		shift ; shift
	done

	#
	# Now create links
	#
	eval set -- ${TEST_LINKS}
	while [ $# -ne 0 ] ; do
		eval ln ${WORK_DIR}/${1} ${WORK_DIR}/${2}
		shift ; shift
	done
}

#
# Setup the test. This creates the working and output directories and
# populates it with files. If there is a setup_test() function - call it.
#
eval_setup()
{
	#
	# Check whether the working directory exists. If it does exit
	# fatally so that we don't clobber a test the user is working on.
	#
	if [ -d ${WORK_DIR} ] ; then
		fatal "working directory ${WORK_DIR} already exists."
	fi

	#
	# Now create it and the output directory
	#
	mkdir -p ${WORK_DIR}
	rm -rf ${OUTPUT_DIR}
	mkdir -p ${OUTPUT_DIR}

	#
	# Common stuff
	#
	common_setup

	#
	# Now after all execute the user's setup function if it exists.
	#
	setup_test
}

#
# Default setup_test function does nothing. This may be overriden by
# the test.
#
setup_test()
{
}

#
# Reset the test. Here we need to rely on information from the test.
# We executed the same steps as in the setup, by try not to clobber existing
# files.
# All files and directories that are listed on the TEST_CLEAN_FILES
# variable are removed. Then the TEST_TOUCH list is executed and finally
# the reset_test() function called if it exists.
#
eval_reset()
{
	ensure_workdir

	#
	# Clean the output directory
	#
	rm -rf ${OUTPUT_DIR}/*

	#
	# Common stuff
	#
	common_setup

	#
	# Remove files.
	#
	for f in ${TEST_CLEAN_FILES} ; do
		rm -rf ${WORK_DIR}/${f}
	done

	#
	# Execute test's function
	#
	reset_test
}

#
# Default reset_test function does nothing. This may be overriden by
# the test.
#
reset_test()
{
}

#
# Clean the test. This simply removes the working and output directories.
#
eval_clean()
{
	#
	# If you have special cleaning needs, provide a 'cleanup' shell script.
	#
	if [ -n "${TEST_CLEANUP}" ] ; then
		. ${SRC_DIR}/cleanup
	fi
	rm -rf ${WORK_DIR}
	rm -rf ${OUTPUT_DIR}
}

#
# Run the test.
#
eval_run()
{
	ensure_workdir

	if [ -z "${TEST_N}" ] ; then
		TEST_N=1
	fi

	N=1
	while [ ${N} -le ${TEST_N} ] ; do
		if ! skip_test ${N} ; then
			( cd ${WORK_DIR} ;
			  exec 1>${OUTPUT_DIR}/stdout.${N} 2>${OUTPUT_DIR}/stderr.${N}
			  run_test ${N}
			  echo $? >${OUTPUT_DIR}/status.${N}
			)
		fi
		N=$((N + 1))
	done
}

#
# Default run_test() function.  It can be replaced by the
# user specified regression test. The argument to this function is
# the test number.
#
run_test()
{
	eval args=\${TEST_${1}-test${1}}
        ${MAKE_PROG} $args
}

#
# Show test results.
#
eval_show()
{
	ensure_workdir

	if [ -z "${TEST_N}" ] ; then
		TEST_N=1
	fi

	N=1
	while [ ${N} -le ${TEST_N} ] ; do
		if ! skip_test ${N} ; then
			echo "=== Test ${N} Status =================="
			cat ${OUTPUT_DIR}/status.${N}
			echo ".......... Stdout .................."
			cat ${OUTPUT_DIR}/stdout.${N}
			echo ".......... Stderr .................."
			cat ${OUTPUT_DIR}/stderr.${N}
		fi
		N=$((N + 1))
	done
}

#
# Compare results with expected results
#
eval_compare()
{
	ensure_workdir
	ensure_run

	if [ -z "${TEST_N}" ] ; then
		TEST_N=1
	fi

	echo "1..${TEST_N}"
	N=1
	while [ ${N} -le ${TEST_N} ] ; do
		fail=
		todo=
		if ! skip_test ${N} ; then
			do_compare stdout ${N} || fail="${fail}stdout "
			do_compare stderr ${N} || fail="${fail}stderr "
			do_compare status ${N} || fail="${fail}status "
			eval todo=\${TEST_${N}_TODO}
		fi
		if [ ! -z "$fail" ]; then
			echo -n "not "
		fi
		echo -n "ok ${N} ${SUBDIR}/${N}"
		if [ ! -z "$fail" -o ! -z "$todo" ]; then
			echo -n " # "
		fi
		if [ ! -z "$todo" ] ; then
			echo -n "TODO $todo; "
		fi
		if [ ! -z "$fail" ] ; then
			echo "reason: ${fail}"
		fi
		echo
		N=$((N + 1))
	done
}

#
# Check if the test result is the same as the expected result.
#
# $1	Input file
# $2	Test number
#
do_compare()
{
	local EXPECTED RESULT
	EXPECTED="expected.$1.$2"
	RESULT="${OUTPUT_DIR}/$1.$2"

	if [ -f $EXPECTED ]; then
		diff -q $EXPECTED $RESULT 1>/dev/null 2>/dev/null
		return $?
	else
		return 1	# FAIL
	fi
}

#
# Diff current and expected results
#
eval_diff()
{
	ensure_workdir
	ensure_run

	if [ -z "${TEST_N}" ] ; then
		TEST_N=1
	fi

	N=1
	while [ ${N} -le ${TEST_N} ] ; do
		if ! skip_test ${N} ; then
			FAIL=
			do_diff stdout ${N}
			do_diff stderr ${N}
			do_diff status ${N}
		fi
		N=$((N + 1))
	done
}

#
# Check if the test result is the same as the expected result.
#
# $1	Input file
# $2	Test number
#
do_diff()
{
	local EXPECTED RESULT
	EXPECTED="expected.$1.$2"
	RESULT="${OUTPUT_DIR}/$1.$2"

	echo diff -u $EXPECTED $RESULT
	if [ -f $EXPECTED ]; then
		diff -u $EXPECTED $RESULT
	else
		echo "${EXPECTED} does not exist"
	fi
}

#
# Update expected results
#
eval_update()
{
	ensure_workdir
	ensure_run

	if [ -z "${TEST_N}" ] ; then
		TEST_N=1
	fi

	FAIL=
	N=1
	while [ ${N} -le ${TEST_N} ] ; do
		if ! skip_test ${N} ; then
			cp ${OUTPUT_DIR}/stdout.${N} expected.stdout.${N}
			cp ${OUTPUT_DIR}/stderr.${N} expected.stderr.${N}
			cp ${OUTPUT_DIR}/status.${N} expected.status.${N}
		fi
		N=$((N + 1))
	done
}

#
# Print description
#
eval_desc()
{
	echo "${SUBDIR}: ${DESC}"
}

#
# Run the test
#
eval_test()
{
	eval_setup
	eval_run
	eval_compare
}

#
# Run the test for prove(1)
#
eval_prove()
{
	eval_setup
	eval_run
	eval_compare
	eval_clean
}

#
# Main function. Execute the command(s) on the command line.
#
eval_cmd()
{
	if [ $# -eq 0 ] ; then
		# if no arguments given default to 'prove'
		set -- prove
	fi

	for i
	do
		case $i in

		setup | run | compare | diff | clean | reset | show | \
		test | prove | desc | update)
			eval eval_$i
			;;
		* | help)
			print_usage
			;;
		esac
	done
}

##############################################################################
#
# Main code
#

#
# Parse command line arguments.
#
args=`getopt m:w:v $*`
if [ $? != 0 ]; then
	echo 'Usage: ...'
	exit 2
fi
set -- $args
for i; do
	case "$i" in
	-m)
		MAKE_PROG="$2"
		shift
		shift
		;;
	-w)
		WORK_BASE="$2"
		shift
		shift
		;;
	-v)
		VERBOSE=1
		shift
		;;
	--)
		shift
		break
		;;
	esac
done

#
# Determine our sub-directory. Argh.
#
SRC_DIR=`pwd`
SRC_BASE=`while [ ! -f common.sh ] ; do cd .. ; done ; pwd`
SUBDIR=`echo ${SRC_DIR} | sed "s@${SRC_BASE}/@@"`

#
# Construct working directory
#
WORK_BASE=${WORK_BASE:-"/tmp/$USER.make.test"}
WORK_DIR=${WORK_BASE}/${SUBDIR}
OUTPUT_DIR=${WORK_DIR}.OUTPUT

#
# Make to use
#
MAKE_PROG=${MAKE_PROG:-/usr/bin/make}