]> git.tue.mpg.de Git - paraslash.git/commitdiff
First draft of a test-suite.
authorAndre Noll <maan@systemlinux.org>
Thu, 10 Feb 2011 22:57:55 +0000 (23:57 +0100)
committerAndre Noll <maan@systemlinux.org>
Mon, 28 Feb 2011 00:42:32 +0000 (01:42 +0100)
Idea and parts of the implementation taken from test-lib.sh of the
git project. At the moment there are only three basic tests with more
to follow.

Makefile.in
t/.gitignore [new file with mode: 0644]
t/audio_files/short-44100-2.ogg [new file with mode: 0644]
t/makefile.test [new file with mode: 0644]
t/t0000-help-output.sh [new file with mode: 0755]
t/t0001-oggdec-correctness.sh [new file with mode: 0755]
t/t0002-oggdec-performance.sh [new file with mode: 0755]
t/test-lib.sh [new file with mode: 0644]
web/manual.m4

index 3fbbce21ab5f3fe91d674750a4d3f7804c346188..a246f4c161832b7f182ea5a457cdc6d435768c4b 100644 (file)
@@ -9,6 +9,7 @@ PACKAGE_VERSION := @PACKAGE_VERSION@
 PACKAGE_STRING := @PACKAGE_STRING@
 install_sh := @install_sh@
 cmdline_dir := @cmdline_dir@
+executables := @executables@
 
 build_date := $(shell date)
 uname_s := $(shell uname -s 2>/dev/null || echo "UNKNOWN_OS")
@@ -67,6 +68,7 @@ man_pages_in := $(patsubst %, web/%.man.in.html, @executables@)
 ggo_dir := ggo
 object_dir := objects
 man_dir := man/man1
+test_dir := t
 
 m4_ggos := afh audioc audiod client filter gui recv server write
 all_ggos := $(m4_ggos) dccp_recv alsa_write oss_write fade http_recv \
@@ -93,10 +95,10 @@ endif
 ifndef BUILD_VERBOSE
        BUILD_VERBOSE = 0
 endif
-ifeq ($(BUILD_VERBOSE),1)
-       Q =
-else
+ifeq ($(BUILD_VERBOSE),0)
        Q = @
+else
+       Q =
 endif
 
 .PHONY: all clean distclean maintainer-clean install man tarball\
@@ -277,7 +279,7 @@ clean2: clean
        $(Q) rm -rf man $(object_dir)
        $(Q) rm -f *_command_list.*
 
-distclean: clean2
+distclean: clean2 test-clean
        @[ -z "$(Q)" ] || echo 'DISTCLEAN'
        $(Q) rm -f Makefile autoscan.log config.status config.log
        $(Q) rm -rf autom4te.cache aclocal.m4
@@ -313,3 +315,4 @@ $(tarball): $(cmdline_generated)
 %.pdf: %.ps
        ps2pdf - - < $< > $@
 
+include $(test_dir)/makefile.test
diff --git a/t/.gitignore b/t/.gitignore
new file mode 100644 (file)
index 0000000..e61ae9c
--- /dev/null
@@ -0,0 +1,2 @@
+/trashes
+/test-results
diff --git a/t/audio_files/short-44100-2.ogg b/t/audio_files/short-44100-2.ogg
new file mode 100644 (file)
index 0000000..91981cd
Binary files /dev/null and b/t/audio_files/short-44100-2.ogg differ
diff --git a/t/makefile.test b/t/makefile.test
new file mode 100644 (file)
index 0000000..d4aedf8
--- /dev/null
@@ -0,0 +1,32 @@
+RM ?= rm -f
+
+results_dir := $(test_dir)/test-results
+trash_dir := $(test_dir)/trashes
+
+test_options := --executables-dir $(shell pwd)
+test_options += --results-dir $(results_dir)
+test_options += --trash-dir $(trash_dir)
+test_options += --executables "$(executables)"
+test_options += --objects "$(basename $(notdir $(all_objs)))"
+
+ifdef V
+       ifeq ("$(origin V)", "command line")
+               test_options += --verbose=$(V)
+       endif
+endif
+
+tests := $(wildcard $(test_dir)/t[0-9][0-9][0-9][0-9]-*.sh)
+
+test: $(tests)
+
+$(tests): all
+       $(Q) $@ $(test_options)
+
+test-help:
+       $(Q) for t in $(tests); do $$t $(test_options) -h; done
+
+test-clean:
+       $(RM) -r $(results_dir)
+       $(RM) -r $(trash_dir)
+
+.PHONY: $(tests) test-help
diff --git a/t/t0000-help-output.sh b/t/t0000-help-output.sh
new file mode 100755 (executable)
index 0000000..fcc2c78
--- /dev/null
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+test_description='Parse help output of all executables.
+
+Each paraslash executable supports the -h switch which instructs
+the program to print out all available options and to exit. This test
+checks whether this help output looks as expected.'
+
+. ${0%/*}/test-lib.sh
+
+grep_output()
+{
+       local regex='^  -h, --help'
+       $1 -h | grep "$regex"
+}
+
+for exe in $o_executables; do
+       test_expect_success "$exe" "grep_output $o_executables_dir/$exe"
+done
+test_done
diff --git a/t/t0001-oggdec-correctness.sh b/t/t0001-oggdec-correctness.sh
new file mode 100755 (executable)
index 0000000..01260e5
--- /dev/null
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+test_description='Check correctness of oggdec output.
+
+Executes para_filter -f oggdec on the test files provided by the test
+suite and compares the output against the output of the reference
+implementation.'
+
+. ${0%/*}/test-lib.sh
+
+test_require_objects "oggdec_filter"
+missing_objects="$result"
+
+test_require_executables "oggdec"
+missing_executables="$result"
+
+get_audio_file_paths ogg
+oggs="$result"
+
+for ogg in $oggs; do
+       if [[ -n "$missing_objects" ]]; then
+               test_skip "${ogg##*/}" "missing object(s): $missing_objects"
+               continue
+       fi
+       if [[ -n "$missing_executables" ]]; then
+               test_skip "${ogg##*/}" \
+                       "missing executables(s): $missing_executables"
+               continue
+       fi
+       test_expect_success "${ogg##*/}" "
+               $PARA_FILTER -f oggdec < $ogg | sha1sum > filter.sha1 &&
+               oggdec --quiet --raw --output - -  < $ogg | sha1sum > oggdec.sha1 &&
+               diff -u filter.sha1 oggdec.sha1
+       "
+done
+test_done
diff --git a/t/t0002-oggdec-performance.sh b/t/t0002-oggdec-performance.sh
new file mode 100755 (executable)
index 0000000..d496a3e
--- /dev/null
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+test_description='Measure time to decode ogg/vorbis files.
+
+Executes para_filter -f oggdec on the test files provided by the
+test suite and fails if it takes much longer than the reference
+implementation.'
+
+. ${0%/*}/test-lib.sh
+
+test_require_objects "oggdec_filter"
+missing_objects="$result"
+
+get_audio_file_paths ogg
+oggs="$result"
+
+test_require_executables "oggdec"
+missing_executables="$result"
+
+for ogg in $oggs; do
+       if [[ -n "$missing_objects" ]]; then
+               test_skip "${ogg##*/}" "missing object(s): $missing_objects"
+               continue
+       fi
+       if [[ -n "$missing_executables" ]]; then
+               test_skip "${ogg##*/}" \
+                       "missing executables(s): $missing_executables"
+               continue
+       fi
+       test_expect_success "${ogg##*/}" '
+               test_duration oggdec --quiet --raw --output - - < $ogg &&
+               t1=$result &&
+               test_duration $PARA_FILTER -f oggdec < $ogg &&
+               t2=$result &&
+               echo "oggdec: $t1, para_filter: $t2"
+               (($t2 <= $t1 * 3 / 2 + 100))
+       '
+done
+test_done
diff --git a/t/test-lib.sh b/t/test-lib.sh
new file mode 100644 (file)
index 0000000..e444e57
--- /dev/null
@@ -0,0 +1,292 @@
+#!/bin/bash
+
+# paraslash test suite helper functions
+# Licensed under the GPL v2. For licencing details see COPYING.
+# uses ideas and code from git's test-lib.sh, Copyright (c) 2005 Junio C Hamano
+
+
+get_audio_file_paths()
+{
+       local suffix="$1"
+
+       if (($# == 0)); then
+               result=$(find "$test_audio_file_dir" -type f)
+       else
+               result=$(find "$test_audio_file_dir" -type f -name "*.$suffix")
+       fi
+}
+
+say_color()
+{
+       if [[ "$o_nocolor" != "true" && -n "$1" ]]; then
+               export TERM=$ORIGINAL_TERM
+               case "$1" in
+                       error) tput bold; tput setaf 1;;
+                       skip)  tput setaf 5;;
+                       ok)
+                               (($o_verbose == 0)) && return
+                               tput setaf 2;;
+                       pass)  tput bold; tput setaf 2;;
+                       info)  tput setaf 3;;
+                       run)
+                               (($o_verbose == 0)) && return
+                               tput setaf 6;;
+               esac
+       fi
+       shift
+       printf "%s\n" "$*"
+       if [[ "$o_nocolor" != "true" && -n "$1" ]]; then
+               tput sgr0
+               export TERM=dumb
+       fi
+}
+
+die()
+{
+       local code=$?
+       [[ "$exit_ok" == "true" ]] && exit $code
+       say_color error "FATAL: Unexpected exit with code $code"
+       exit 1
+}
+
+error()
+{
+       say_color error "error: $*"
+       exit_ok="true"
+       exit 1
+}
+
+say()
+{
+       say_color info "$*"
+}
+
+_test_run()
+{
+       local f
+
+       let test_count++
+       eval >&3 2>&4 "$2"
+       if (($? == 0)); then
+               let test_success++
+               say_color ok "ok $test_count - $1"
+               return
+       fi
+       let test_failure++
+       say_color error "not ok - $test_count $1"
+       f="$o_results_dir/${0##*/}-$$.out"
+       if [[ -s "$f" ]]; then
+               sed -e 's/^/#   /' < "$f"
+       else
+               sed -e 's/^/#   /' <<< "$2"
+       fi
+       [[ "$o_immediate" != "true" ]] && return
+       exit_ok="true"
+       exit 1
+}
+
+test_skip()
+{
+       (($# != 2)) && error "bug: not 2 parameters to test_skip()"
+       let test_count++
+       let test_skipped++
+       say_color skip >&3 "skipping test $this_test.$test_count ($1): $2"
+       say_color skip "ok $test_count - $1 # skipped ($2)"
+}
+
+test_require_objects()
+{
+       local o1 o2 found
+
+       result=
+       # if no objects were given, we assume this test is run manually
+       # rather than via "make test". We won't check anything in this case
+       [[ -z "$o_objects" ]] && return
+
+       for o1 in $1; do
+               found=
+               for o2 in $o_objects; do
+                       [[ "$o1" != "$o2" ]] && continue
+                       found="true"
+                       break
+               done
+               [[ "$found" == "true" ]] && continue
+               [[ -n "$result" ]] && result+=" "
+               result+="$o1"
+       done
+       [[ -z "$result" ]]
+}
+
+test_require_executables()
+{
+       local i
+
+       result=
+       for i in "$@"; do
+               [[ -n "$(builtin type -t "$i")" ]] && continue
+               [[ -n "$result" ]] && result+=" "
+               result+="$i"
+       done
+       [[ -z "$result" ]]
+}
+
+test_duration()
+{
+       local t=$(exec 2>&1 1>/dev/null; time -p "$@")
+       result=$(awk '{print $2 * 1000}' <<< $t)
+}
+
+test_expect_success()
+{
+       (($# != 2)) && error "bug: not 2 parameters to test_expect_success()"
+       say >&3 "expecting success: $2"
+       _test_run "$1" "$2"
+       echo >&3 ""
+}
+
+test_done()
+{
+       test_results_path="$o_results_dir/${0##*/}-$$.counts"
+       {
+               echo "total $test_count"
+               echo "success $test_success"
+               echo "failed $test_failure"
+               echo "skipped $test_skipped"
+               echo
+       } > $test_results_path
+
+       exit_ok="true"
+       msg="$test_count test(s) ($test_skipped test(s) skipped)"
+       if (($test_failure == 0)); then
+               say_color pass "# ${0##*/}: passed all $msg"
+               exit 0
+       else
+               say_color error "# ${0##*/}: failed $test_failure among $msg"
+               exit 1
+       fi
+}
+
+sanitize_environment()
+{
+       export LANG=C
+       export LC_ALL=C
+       export PAGER=cat
+       export TZ=UTC
+       export TERM=dumb
+       export EDITOR=:
+       export HOME=$(pwd)
+
+       unset VISUAL
+       unset EMAIL
+       unset CDPATH
+       unset GREP_OPTIONS
+}
+
+can_use_colors()
+{
+       result="false"
+       [[ "$TERM" == "dumb" ]] && return
+       [[ -t 1 ]] || return
+       tput bold >/dev/null 2>&1 || return
+       tput setaf 1 >/dev/null 2>&1 || return
+       tput sgr0 >/dev/null 2>&1 || return
+       result="true"
+}
+
+parse_options()
+{
+       while (($# > 0)); do
+               case "$1" in
+               -i|--immediate) o_immediate="true"; shift;;
+               -l|--long) export o_long="true"; shift;;
+               -h|--help) o_help="true"; shift;;
+               -v=0|--verbose=0) o_verbose="0"; shift;;
+               -v=1|--verbose=1) o_verbose="1"; shift;;
+               -v|--verbose|-v=2|--verbose=2) o_verbose="2"; shift;;
+               --no-color) o_nocolor="true"; shift;;
+               --results-dir) o_results_dir="$2"; shift; shift;;
+               --trash-dir) o_trash_dir="$2"; shift; shift;;
+               --executables-dir) export o_executables_dir="$2"; shift; shift;;
+               --executables) export o_executables="$2"; shift; shift;;
+               --objects) export o_objects="$2"; shift; shift;;
+               *) echo "error: unknown test option '$1'" >&2; exit 1;;
+               esac
+       done
+       [[ -z "$o_verbose" ]] && o_verbose=1
+}
+
+create_trash_dir_and_cd()
+{
+       local trash="$o_trash_dir/trash-dir.${0##*/}"
+
+       rm -rf "$trash" || error "could not remove trash dir"
+       mkdir -p "$trash" || error "could not make trash dir"
+       cd "$trash" || error "could not change to trash dir"
+}
+
+fixup_dirs()
+{
+       local wd=$(pwd)
+
+       test_dir="$wd/${0%/*}"
+       test_audio_file_dir="$test_dir/audio_files"
+
+       [[ -z "$o_results_dir" ]] && o_results_dir="$test_dir/test-results"
+       [[ -z "$o_executables_dir" ]] && o_executables_dir="$test_dir/.."
+       [[ -z "$o_trash_dir" ]] && o_trash_dir="$test_dir/trashes"
+
+       # we want alsolute paths because relative paths become invalid
+       # after changing to the trash dir
+       [[ -n "${o_results_dir##/*}" ]] && o_results_dir="$wd/$o_results_dir"
+       [[ -n "${o_executables_dir##/*}" ]] && o_executables_dir="$wd/$o_results_dir"
+       [[ -n "${o_trash_dir##/*}" ]] && o_trash_dir="$wd/$o_trash_dir"
+
+       mkdir -p "$o_results_dir"
+}
+
+parse_options "$@"
+if [[ "$o_nocolor" != "true" ]]; then
+       can_use_colors
+       [[ "$result" != "true" ]] && o_nocolor="true"
+fi
+
+# Each test must set test_description
+[[ -z "${test_description}" ]] && error "${0##*/} did not set test_description"
+if [[ "$o_help" == "true" ]]; then
+       printf "${0##*/}: "
+       sed -e '1!d' <<< "$test_description"
+       if (($o_verbose >= 2)); then
+               echo
+               sed -e '1,2d' -e 's/^/  /g' <<<"$test_description"
+               echo
+       fi
+       exit 0
+fi
+fixup_dirs
+
+[[ -z "$o_executables" ]] && o_executables="para_afh para_audioc para_audiod
+       para_client para_fade para_filter para_gui para_recv para_server
+       para_write"
+for exe in $o_executables; do
+       export $(tr 'a-z' 'A-Z' <<< $exe)="$o_executables_dir/$exe"
+done
+
+test_failure=0
+test_count=0
+test_success=0
+test_skipped=0
+
+ORIGINAL_TERM=$TERM
+sanitize_environment
+create_trash_dir_and_cd
+
+if (($o_verbose >= 2)); then
+       exec 4>&2 3>&1
+else
+       exec 4>$o_results_dir/${0##*/}-$$.out 3>&4
+fi
+
+exit_ok=
+trap 'die' EXIT
+
+say_color run "# running ${0##*/}"
index 946b9d0953621b23a3a6e30e62c211674d0c6ea5..5926b55536c736c0bd8f031bd596d51e31587fbb 100644 (file)
@@ -280,7 +280,11 @@ might need to tell the configure script where to find them. Try
        ./configure --help
 
 to see a list of options. If the paraslash package was compiled
-successfully, execute as root,
+successfully, execute (optionally)
+
+       make test
+
+to run the paraslash test suite. If all tests pass, execute as root
 
        make install