From 915de2adb584acea89f3f654a6c9b329f682100f Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Mon, 29 Sep 2014 12:02:11 +0000 Subject: ftracetest: Add POSIX.3 standard and XFAIL result codes Add XFAIL and POSIX 1003.3 standard codes (UNRESOLVED/ UNTESTED/UNSUPPORTED) as result codes. These are used for the results that test case is expected to fail or unsupported feature (by config). To return these result code, this introduces exit_unresolved, exit_untested, exit_unsupported and exit_xfail functions, which use real-time signals to notify the result code to ftracetest. This also set "errexit" option for the testcases, so that the tests don't need to exit explicitly. Note that if the test returns UNRESOLVED/UNSUPPORTED/FAIL, its test log including executed commands is shown on console and main logfile as below. ------ # ./ftracetest samples/ === Ftrace unit tests === [1] failure-case example [FAIL] execute: /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/fail.tc + . /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/fail.tc ++ cat non-exist-file cat: non-exist-file: No such file or directory [2] pass-case example [PASS] [3] unresolved-case example [UNRESOLVED] execute: /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unresolved.tc + . /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unresolved.tc ++ trap exit_unresolved INT ++ kill -INT 29324 +++ exit_unresolved +++ kill -s 38 29265 +++ exit 0 [4] unsupported-case example [UNSUPPORTED] execute: /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unsupported.tc + . /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unsupported.tc ++ exit_unsupported ++ kill -s 40 29265 ++ exit 0 [5] untested-case example [UNTESTED] [6] xfail-case example [XFAIL] # of passed: 1 # of failed: 1 # of unresolved: 1 # of untested: 1 # of unsupported: 1 # of xfailed: 1 # of undefined(test bug): 0 ------ Link: http://lkml.kernel.org/p/20140929120211.30203.99510.stgit@kbuild-f20.novalocal Acked-by: Namhyung Kim Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt --- tools/testing/selftests/ftrace/ftracetest | 124 ++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 15 deletions(-) (limited to 'tools/testing/selftests/ftrace/ftracetest') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index 4c6c2fad2cda..a8f81c782856 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -114,22 +114,106 @@ prlog "=== Ftrace unit tests ===" # Testcase management +# Test result codes - Dejagnu extended code +PASS=0 # The test succeeded. +FAIL=1 # The test failed, but was expected to succeed. +UNRESOLVED=2 # The test produced indeterminate results. (e.g. interrupted) +UNTESTED=3 # The test was not run, currently just a placeholder. +UNSUPPORTED=4 # The test failed because of lack of feature. +XFAIL=5 # The test failed, and was expected to fail. + +# Accumulations PASSED_CASES= FAILED_CASES= +UNRESOLVED_CASES= +UNTESTED_CASES= +UNSUPPORTED_CASES= +XFAILED_CASES= +UNDEFINED_CASES= +TOTAL_RESULT=0 + CASENO=0 testcase() { # testfile CASENO=$((CASENO+1)) prlog -n "[$CASENO]"`grep "^#[ \t]*description:" $1 | cut -f2 -d:` } -failed() { - prlog " [FAIL]" - FAILED_CASES="$FAILED_CASES $CASENO" + +eval_result() { # retval sigval + local retval=$2 + if [ $2 -eq 0 ]; then + test $1 -ne 0 && retval=$FAIL + fi + case $retval in + $PASS) + prlog " [PASS]" + PASSED_CASES="$PASSED_CASES $CASENO" + return 0 + ;; + $FAIL) + prlog " [FAIL]" + FAILED_CASES="$FAILED_CASES $CASENO" + return 1 # this is a bug. + ;; + $UNRESOLVED) + prlog " [UNRESOLVED]" + UNRESOLVED_CASES="$UNRESOLVED_CASES $CASENO" + return 1 # this is a kind of bug.. something happened. + ;; + $UNTESTED) + prlog " [UNTESTED]" + UNTESTED_CASES="$UNTESTED_CASES $CASENO" + return 0 + ;; + $UNSUPPORTED) + prlog " [UNSUPPORTED]" + UNSUPPORTED_CASES="$UNSUPPORTED_CASES $CASENO" + return 1 # this is not a bug, but the result should be reported. + ;; + $XFAIL) + prlog " [XFAIL]" + XFAILED_CASES="$XFAILED_CASES $CASENO" + return 0 + ;; + *) + prlog " [UNDEFINED]" + UNDEFINED_CASES="$UNDEFINED_CASES $CASENO" + return 1 # this must be a test bug + ;; + esac +} + +# Signal handling for result codes +SIG_RESULT= +SIG_BASE=36 # Use realtime signals +SIG_PID=$$ + +SIG_UNRESOLVED=$((SIG_BASE + UNRESOLVED)) +exit_unresolved () { + kill -s $SIG_UNRESOLVED $SIG_PID + exit 0 +} +trap 'SIG_RESULT=$UNRESOLVED' $SIG_UNRESOLVED + +SIG_UNTESTED=$((SIG_BASE + UNTESTED)) +exit_untested () { + kill -s $SIG_UNTESTED $SIG_PID + exit 0 } -passed() { - prlog " [PASS]" - PASSED_CASES="$PASSED_CASES $CASENO" +trap 'SIG_RESULT=$UNTESTED' $SIG_UNTESTED + +SIG_UNSUPPORTED=$((SIG_BASE + UNSUPPORTED)) +exit_unsupported () { + kill -s $SIG_UNSUPPORTED $SIG_PID + exit 0 } +trap 'SIG_RESULT=$UNSUPPORTED' $SIG_UNSUPPORTED +SIG_XFAIL=$((SIG_BASE + XFAIL)) +exit_xfail () { + kill -s $SIG_XFAIL $SIG_PID + exit 0 +} +trap 'SIG_RESULT=$XFAIL' $SIG_XFAIL # Run one test case run_test() { # testfile @@ -137,14 +221,17 @@ run_test() { # testfile local testlog=`mktemp --tmpdir=$LOG_DIR ${testname}-XXXXXX.log` testcase $1 echo "execute: "$1 > $testlog - (cd $TRACING_DIR; set -x ; . $1) >> $testlog 2>&1 - ret=$? - if [ $ret -ne 0 ]; then - failed - catlog $testlog - else - passed + SIG_RESULT=0 + # setup PID and PPID, $$ is not updated. + (cd $TRACING_DIR; read PID _ < /proc/self/stat ; + set -e; set -x; . $1) >> $testlog 2>&1 + eval_result $? $SIG_RESULT + if [ $? -eq 0 ]; then + # Remove test log if the test was done as it was expected. [ $KEEP_LOG -eq 0 ] && rm $testlog + else + catlog $testlog + TOTAL_RESULT=1 fi } @@ -152,8 +239,15 @@ run_test() { # testfile for t in $TEST_CASES; do run_test $t done + prlog "" prlog "# of passed: " `echo $PASSED_CASES | wc -w` prlog "# of failed: " `echo $FAILED_CASES | wc -w` - -test -z "$FAILED_CASES" # if no error, return 0 +prlog "# of unresolved: " `echo $UNRESOLVED_CASES | wc -w` +prlog "# of untested: " `echo $UNTESTED_CASES | wc -w` +prlog "# of unsupported: " `echo $UNSUPPORTED_CASES | wc -w` +prlog "# of xfailed: " `echo $XFAILED_CASES | wc -w` +prlog "# of undefined(test bug): " `echo $UNDEFINED_CASES | wc -w` + +# if no error, return 0 +exit $TOTAL_RESULT -- cgit v1.2.3-70-g09d2