From: David Disseldorp <ddiss@suse.de>
To: fstests@vger.kernel.org, tytso@mit.edu
Cc: David Disseldorp <ddiss@suse.de>
Subject: [RFC PATCH v2 5/6] check: add -L <n> parameter to rerun failed tests
Date: Tue, 28 Jun 2022 00:22:55 +0200 [thread overview]
Message-ID: <20220627222256.14175-6-ddiss@suse.de> (raw)
In-Reply-To: <20220627222256.14175-1-ddiss@suse.de>
If check is run with -L <n>, then a failed test will be rerun <n> times
before proceeding to the next test. Following completion of the rerun
loop, aggregate pass/fail statistics are printed.
Rerun tests will be tracked as a single failure in overall pass/fail
metrics (via @try and @bad), with .out.bad, .dmesg and .full saved using
a .rerun# suffix.
Suggested-by: Theodore Ts'o <tytso@mit.edu>
Link: https://lwn.net/Articles/897061/
Signed-off-by: David Disseldorp <ddiss@suse.de>
---
check | 94 ++++++++++++++++++++++++++++++++++++++++++++-------
common/report | 8 +++--
2 files changed, 88 insertions(+), 14 deletions(-)
diff --git a/check b/check
index aa7dac2f..726c83d9 100755
--- a/check
+++ b/check
@@ -26,6 +26,7 @@ do_report=false
DUMP_OUTPUT=false
iterations=1
istop=false
+loop_on_fail=0
# This is a global variable used to pass test failure text to reporting gunk
_err_msg=""
@@ -75,6 +76,7 @@ check options
--large-fs optimise scratch device for large filesystems
-s section run only specified section from config file
-S section exclude the specified section from the config file
+ -L <n> loop tests <n> times following a failure, measuring aggregate pass/fail metrics
testlist options
-g group[,group...] include tests from these groups
@@ -333,6 +335,9 @@ while [ $# -gt 0 ]; do
;;
--large-fs) export LARGE_SCRATCH_DEV=yes ;;
--extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
+ -L) [[ $2 =~ ^[0-9]+$ ]] || usage
+ loop_on_fail=$2; shift
+ ;;
-*) usage ;;
*) # not an argument, we've got tests now.
@@ -555,6 +560,18 @@ _expunge_test()
_stash_test_status() {
local test_seq="$1"
local test_status="$2"
+ local test_time="$3"
+ local loop_num="$4"
+ local report_msg="$5"
+
+ if $do_report && [[ ! $test_status =~ ^(init|expunge)$ ]]; then
+ _make_testcase_report "$section" "$test_seq" \
+ "$test_status" "$test_time" \
+ "$report_msg"
+ fi
+
+ # only stash result for first failure (triggering loop)
+ ((loop_num > 1)) && return
case "$test_status" in
fail)
@@ -610,6 +627,38 @@ _run_seq() {
fi
}
+# Check whether the last test should be rerun according to loop-on-error state
+# and return "0" if so, otherwise return "1".
+_ix_inc() {
+ local test_status="$1"
+ local loop_len="$2"
+
+ if ((!loop_on_fail)); then
+ echo 1
+ return
+ fi
+
+ if [ "$test_status" == "fail" ] && ((!loop_len)); then
+ echo 0 # initial failure of this test, start loop-on-fail
+ elif ((loop_len > 0)) && ((loop_len < loop_on_fail)); then
+ echo 0 # continue loop following initial failure
+ else
+ echo 1 # completed or not currently in a failure loop
+ fi
+}
+
+_failure_loop_dump_stats() {
+ awk "BEGIN {
+ n=split(\"$*\", arr);"'
+ for (i = 1; i <= n; i++)
+ stats[arr[i]]++;
+ printf("aggregate results across %d runs: ", n);
+ for (x in stats)
+ printf("%s=%d (%.1f%%)", (i-- > n ? x : ", " x),
+ stats[x], 100 * stats[x] / n);
+ }'
+}
+
_detect_kmemleak
_prepare_test_list
@@ -750,14 +799,29 @@ function run_section()
seqres="$check"
_check_test_fs
- local tc_status="init"
+ local tc_status="init" ix agg_msg
prev_seq=""
- for seq in $list ; do
+ local -a _list=( $list ) loop_status=()
+ for ((ix = 0; ix < ${#_list[*]};
+ ix += $(_ix_inc "$tc_status" "${#loop_status[*]}"))); do
+ seq="${_list[$ix]}"
+
+ if [ "$seq" == "$prev_seq" ]; then
+ loop_status+=("$tc_status")
+ elif ((${#loop_status[*]})); then
+ # leaving rerun-on-failure loop
+ loop_status+=("$tc_status")
+ agg_msg=$(_failure_loop_dump_stats "${loop_status[@]}")
+ echo "$seqnum $agg_msg"
+ fi
+
# Run report for previous test!
- _stash_test_status "$seqnum" "$tc_status"
- if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
- _make_testcase_report "$section" "$seqnum" \
- "$tc_status" "$((stop - start))"
+ _stash_test_status "$seqnum" "$tc_status" "$((stop - start))" \
+ "${#loop_status[*]}" "$agg_msg"
+
+ if [ -n "$agg_msg" ]; then
+ loop_status=()
+ agg_msg=""
fi
prev_seq="$seq"
@@ -827,7 +891,9 @@ function run_section()
fi
# record that we really tried to run this test.
- try+=("$seqnum")
+ if ((!${#loop_status[*]})); then
+ try+=("$seqnum")
+ fi
awk 'BEGIN {lasttime=" "} \
$1 == "'$seqnum'" {lasttime=" " $2 "s ... "; exit} \
@@ -954,13 +1020,17 @@ function run_section()
fi
done
- # make sure we record the status of the last test we ran.
- _stash_test_status "$seqnum" "$tc_status"
- if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
- _make_testcase_report "$section" "$seqnum" "$tc_status" \
- "$((stop - start))"
+ if ((${#loop_status[*]})); then
+ # leaving rerun-on-failure loop
+ loop_status+=("$tc_status")
+ agg_msg=$(_failure_loop_dump_stats "${loop_status[@]}")
+ echo "$seqnum $agg_msg"
fi
+ # Run report for previous test!
+ _stash_test_status "$seqnum" "$tc_status" "$((stop - start))" \
+ "${#loop_status[*]}" "$agg_msg"
+
sect_stop=`_wallclock`
interrupt=false
_wrapup
diff --git a/common/report b/common/report
index 5ca41bc4..cede4987 100644
--- a/common/report
+++ b/common/report
@@ -71,6 +71,7 @@ _xunit_make_testcase_report()
local test_name="$2"
local test_status="$3"
local test_time="$4"
+ local test_md="$5"
# TODO: other places may also win if no-section mode will be named like 'default/global'
if [ $sect_name == '-no-sections-' ]; then
@@ -79,7 +80,8 @@ _xunit_make_testcase_report()
fi
local report=$tmp.report.xunit.$sect_name.xml
- echo -e "\t<testcase classname=\"xfstests.$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report
+ [ -n "$test_md" ] && test_md=" status=\"$(echo "$test_md"|encode_xml)\""
+ echo -e "\t<testcase classname=\"xfstests.$sect_name\" name=\"$test_name\" time=\"$test_time\"${test_md}>" >> $report
case $test_status in
"pass")
;;
@@ -162,11 +164,13 @@ _make_testcase_report()
local test_seq="$2"
local test_status="$3"
local test_time="$4"
+ local test_md="$5"
for report in $REPORT_LIST; do
case "$report" in
"xunit")
_xunit_make_testcase_report "$sect_name" "$test_seq" \
- "$test_status" "$test_time"
+ "$test_status" \
+ "$test_time" "$test_md"
;;
*)
_dump_err "report format '$report' is not supported"
--
2.35.3
next prev parent reply other threads:[~2022-06-27 22:23 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-27 22:22 [RFC PATCH v2 0/6] check: add option to rerun failed tests David Disseldorp
2022-06-27 22:22 ` [RFC PATCH v2 1/6] report: use array for REPORT_ENV_LIST David Disseldorp
2022-06-28 14:54 ` Darrick J. Wong
2022-06-27 22:22 ` [RFC PATCH v2 2/6] report: pass through most details as function parameters David Disseldorp
2022-06-28 14:55 ` Darrick J. Wong
2022-06-27 22:22 ` [RFC PATCH v2 3/6] check: make a few variables local David Disseldorp
2022-06-28 14:56 ` Darrick J. Wong
2022-06-27 22:22 ` [RFC PATCH v2 4/6] check: append bad / notrun arrays in helper function David Disseldorp
2022-06-28 15:00 ` Darrick J. Wong
2022-06-27 22:22 ` David Disseldorp [this message]
2022-06-28 15:15 ` [RFC PATCH v2 5/6] check: add -L <n> parameter to rerun failed tests Darrick J. Wong
2022-06-28 22:34 ` David Disseldorp
2022-06-27 22:22 ` [RFC PATCH v2 6/6] check: stash full/dmesg/out.bad files on rerun David Disseldorp
2022-06-28 15:16 ` Darrick J. Wong
2022-06-28 22:36 ` David Disseldorp
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220627222256.14175-6-ddiss@suse.de \
--to=ddiss@suse.de \
--cc=fstests@vger.kernel.org \
--cc=tytso@mit.edu \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.