Run Tests

This script is used in a Jenkins/Kitchen test suite.

Original Version

#!/usr/bin/env bash

KITCHEN=$(which kitchen)
VAGRANT=$(which vagrant)
VBOX=$(which VBoxManage)

function run_tests_cleanup_vms {
  vm_ids=""

  for machine in $($KITCHEN list -b "$1"); do
    vm_ids="$vm_ids $(vagrant global-status | grep $PWD | grep $machine | cut -f 1 -d ' ')"
  done
  $KITCHEN destroy "$1" > /dev/null

  # We then run destroy over any stray VMs
  # That is, VMs that might have failed during create, so were not destroyed
  # by kitchen destroy
  for vm_id in $vm_ids; do
    vagrant destroy --force $vm_id > /dev/null 2>&1
  done
}

if [ -z $KITCHEN ]; then
  echo "Cannot find kitchen binary. Please install to run tests."
  echo "The easiest way to do this is probably by installing ChefDK."
  echo "See the testing-utils README for more information."
  exit 1
fi

if [ -z $VAGRANT ]; then
  echo "Cannot find Vagrant binary. Please install to run tests."
  echo "See the testing-utils README for more information."
  exit 1
fi

if [ -z $VBOX ]; then
  echo "VirtualBox does not seem to be installed. Please install to run tests."
  echo "See the testing-utils README for more information."
  exit 1
fi

if [ -e "$PWD/test/testing_utils_plugin.sh" ]; then
  . "$PWD/test/testing_utils_plugin.sh"
  HAS_PLUGIN=TRUE
else
  HAS_PLUGIN=FALSE
fi

POSITIONAL=()
PID=${$}
logfile=/tmp/run_tests.${PID}.log
exit_value=0

echo "PARAMETERS: $@" > $logfile

while [[ $# -gt 0 ]]; do
  key="$1"

  case $key in
    -e|--use-emerge)
    echo "INFO: Using emerge repos"
    export USE_EMERGE=TRUE
    shift
    ;;
    -h|--help)
    echo "Usage: run_tests [options] [instance pattern]"
    echo "Options:"
    echo "  --use-emerge          Use the emerge repositories"
    echo "  --update [box]        Update the Vagrant boxes and exit. Update single box if specified."
    echo "  --cleanup [instance]  Destroy VMs matching instance pattern. All if not specified."
    echo "                        The instance pattern is passed to kitchen to match the instances upon"
    echo "                        which to run the tests. This can be a string or regex."
    echo "  --with-cleanup        Destroy all instances after testing, even if it has errors. By default,"
    echo "                        if a instance has errors it is not destroyed."

    if [ "$HAS_PLUGIN" == 'TRUE' ]; then
      echo ''
      echo "Options for $(plugin_name) tests"
      show_plugin_help
    fi

    rm -f $logfile
    exit 0
    ;;
    --update)
    echo "Pruning old Vagrant boxes, updating boxes and exiting"
    vagrant box prune
    if [ -z "$2" ]; then
      echo "Checking all boxes"
      for box in $($VAGRANT box outdated --global | grep outdated | cut -f 2 -d "'" | sort ); do
        $VAGRANT box update --box $box
      done
    else
      echo "Updating specified box: $2"
      $VAGRANT box update --box "$2"
    fi
    rm $logfile
    exit 0
    ;;
    --with-cleanup)
    export WITH_CLEANUP=TRUE
    shift
    ;;
    --cleanup)
    echo "Destroying VMs and exiting"
    run_tests_cleanup_vms "$2"
    exit 0
    ;;
    *)
    POSITIONAL+=("$1") # save it in an array for later
    shift
    ;;
  esac
done

set -- "${POSITIONAL[@]}" # restore positional parameters

if [ "$HAS_PLUGIN" == 'TRUE' ]; then
  parse_plugin_command_line "$@"
  set -- "${PLUGIN_POSITIONAL[@]}"
fi

filter_list=$1

run_tests_cleanup_vms "$filter_list"

for machine in $($KITCHEN list -b "${filter_list}"); do
  if $KITCHEN test $machine; then
    echo "PASSED: $machine" >> $logfile
  else
    echo "FAILED: $machine" >> $logfile
    exit_value=1
  fi
done

echo "Testing done"
cat $logfile
rm $logfile

unset USE_EMERGE

if [ "$HAS_PLUGIN" == 'TRUE' ]; then
  plugin_cleanup
fi

if [ "$WITH_CLEANUP" == 'TRUE' ]; then
  unset WITH_CLEANUP
  run_tests_cleanup_vms ${filter_list}
fi

exit $exit_value

Biggest Issues

Spaghetti logic (option parsing from start to end)
Options provide limited value (mostly for modifying personal directory)
Haphazard option parsing and heavy use of environment variables
Lots of bugs that require pre-/post-execution tasks

New Version

Uses functions to organize logic
Fully backward-compatible
Supports short and long arguments (long was considered a hard requirement)
Lots more built-in documentation
Exposes helpful options
Lots more logging

#!/usr/bin/env bash
##
# Wrapper script for running integration tests and working with kitchen.
##


##
# Main / Main Operations
##

# Primary entry point for script execution
main() {
	check_packages || die 'Package requisites not met.'
	case "$OPERATION" in
		test) main_test || die 'Some (or all) tests failed.';;
		clean) main_clean || die 'Issue encountered while cleaning up test artifacts.';;
		update) main_update || die 'Issue encountered while updating vagrant boxes.';;
	esac
}

# Update vagrant boxes
main_update() {
	local err=0
	msg "Pruning old Vagrant boxes, updating boxes and exiting."
	vagrant box prune || die 'Issue encountered pruning vagrant boxes.'
	if [[ "$TARGET" == '' ]]; then
		msg 'Updating all boxes.'
		boxes=("$(vagrant box outdated --global | awk -F"'" '/outdated/ { print $2 }')")
		if [[ "${boxes[0]}" != '' ]]; then
			for box in "${boxes[@]}"; do
				vagrant box update --box "$box" || err=1
			done
		fi
	else
		msg 'Updating specified box.'
		vagrant box update --box "$TARGET" || err=1
	fi

	return $err
}

# Clean up test artifacts
main_clean() {
	cleanup || return 1
	return 0
}

# Run integration tests and clean up
main_test() {
	local err=0
	cleanup

	log "PARAMETERS: ${PARAMS[*]}"
	log "PLUGIN ARGS: ${PLUGIN_ARGS[*]}"

	msg 'Testing: start'
	for machine in $(kitchen list -b "$TARGET"); do
		msg "Running tests on: $machine"
		[[ "$KEEPLOGS" ]] && lf="&>'$LOGFILE.$machine'"
		if eval kitchen test "$machine" $lf; then
			log "PASSED: $machine"
		else
			log "FAILED: $machine"
			err=1
		fi
	done
	msg 'Testing: end'


	printf '#- Results: start\n%s\n#- Results: end\n' "$(< "$LOGFILE")"

	if [[ "$KEEPLOGS" ]]; then
		msg "Test logs retained at $LOGFILE and $LOGFILE.{machine}."
	else
		rm "$LOGFILE"*
	fi

	[[ "$CLEANUP" ]] && cleanup

	return $err
}

# Clean up test machines and artifacts
cleanup() {
	local err=0
	msg 'Running clean up tasks.'

	[[ "$HAS_PLUGIN" ]] && plugin_cleanup
	kitchen destroy "$TARGET" &>/dev/null || err=1

	IFS=' ' read -r -a machines <<< "$(kitchen list -b "$TARGET")"
	IFS=$'\n' read -r -a boxes <<< "$(vagrant global-status | grep "$PWD")"

	for machine in "${machines[@]}"; do
		local nl=()
		for box in "${boxes[@]}"; do
			vid="$(awk -v tgt="$machine" '/tgt/ {print $1}' <<<"$box")"
			if [[ "$vid" ]]; then
				vagrant destroy --force "$vid" &> /dev/null || err=1
			else
				nl+=("$box")
			fi
		done
		boxes=("${nl[@]}")
	done

	return $err
}


##
# Helper Functions
##

# Parse options passed to script and load into environment
load_opts() {
	PARAMS=("$@")

	# Defaults
	LOGFILE="/tmp/run_tests.${$}.log"
	OPERATION='test'
	PLUGIN_ARGS=()
	unset HAS_PLUGIN
	unset KEEPLOGS
	unset CLEANUP
	unset TARGET
	unset QUIET


	local OPTIND OPTARG opt
	while getopts ':xukceqdh-:' opt; do
		case "$opt" in
			x) OPERATION='clean';;
			u) OPERATION='update';;
			k) KEEPLOGS='y';;
			c) CLEANUP='y';;
			e) export USE_EMERGE='TRUE';;
			q) QUIET='y';;
			d) set -x;;
			h) show_usage; exit 0;;
			-)
				case "$OPTARG" in
					clean)	OPERATION='clean';;
					update) OPERATION='update';;
					keeplogs) KEEPLOGS='y';;
					cleanup) CLEANUP='y';;
					emerge) export USE_EMERGE='TRUE';;
					quiet) QUIET='y';;
					debug) set -x;;
					help) show_usage; exit 0;;
					# legacy opts
					use-emerge) export USE_EMERGE='TRUE';;
					with-cleanup) CLEANUP='y';;
					*)
						PLUGIN_ARGS+=("--$OPTARG")
						n="${PARAMS[$((OPTIND-1))]}"
						if [[
						    ! "$n" =~ '-' &&
						    ! "$OPTARG" =~ '='
						    ]]; then
							PLUGIN_ARGS+=("$n")
							let 'OPTIND += 1'
						fi
						;;
				esac
				;;
		esac
	done
	TARGET="${PARAMS[$((OPTIND-1))]}"

	if [[ -e "$PWD/test/testing_utils_plugin.sh" ]]; then
		. "$PWD/test/testing_utils_plugin.sh"
		parse_plugin_command_line ${PLUGIN_ARGS[@]}
		HAS_PLUGIN='y'
	fi
}

# Print usage/help information
show_usage() {
	cat <<-EOF
	$0 [options] <target>

	Positional Options:
	  <target>		Restrict execution to instances matching <target> (default: all available)

	Execution Options:
	  <none>		Run integration tests
	  -u	--update	Update vagrant boxes
	  -x	--clean		Clean up test artifacts and exit
	  -h	--help		Show this usage text and exit.

	Runtime Options:
	  -d	--debug		Enable shell debug
	  -q	--quiet		Don't print informational messages

	Test Options:
	  -e	--emerge	Use $Client's Emerge repositories
	  -c	--cleanup	Clean up machines after testing (default: keep failed)
	  -k	--keeplogs	Write logs to per-test files (default: stdout)
	EOF

	if [[ "$HAS_PLUGIN" ]]; then
		printf '\n%s\n' "Options for test plugin:"
		show_plugin_help
	fi

	cat <<-EOF

	Examples:

	    Run all available tests on all supported distros:
	        run_tests.sh

	    Run tests, removing all machines after testng:
	        run_tests.sh -c

	    Run tests on Ubuntu machines:
	        run_tests.sh ubuntu

	    Clean up images from testing; no testing:
	        run_tests.sh -x

	    Typical usage:
	        run_tests.sh -eck ubuntu-1604
	EOF
}

# Verify required packages are installed
check_packages() {
	local err=0
	for pkg in 'kitchen' 'vagrant' 'VBoxManage'; do
		if ! command -v "$pkg" >/dev/null; then
			printf 'Not found in path: %s\n' "$pkg"
			err=1
		fi
	done
	return $err
}

# Write message to log file
log() {
	[[ "$LOGFILE" && "$*" ]] | return 1
	printf '%s\n' "$*" >> "$LOGFILE"
}

# Print a message (if tty/non-quiet)
msg() {
	[[ -t 1 ]] || return 0
	[[ "$QUIET" ]] || printf 'INFO: %s\n' "$*"
}

# Print a message and exit
die() {
	[[ "$*" ]] && printf '*** %s ***\n' "$*"
	exit 2
}


##
# Script Kickoff
##

load_opts "$@"
main