* [PATCH 0/2][RESEND] Add report generators support @ 2017-01-31 17:43 Dmitry Monakhov 2017-01-31 17:43 ` [PATCH 1/2] check: prepare test report generator infrastructure Dmitry Monakhov 2017-01-31 17:43 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 0 siblings, 2 replies; 14+ messages in thread From: Dmitry Monakhov @ 2017-01-31 17:43 UTC (permalink / raw) To: fstests; +Cc: Dmitry Monakhov xfstests has no stable logging API which is suitable for machine parsing. This makes it difficult to integrate with external atomation tools (Jenkins.io, k8s jobs) In fact thare are alot of well known parsable report formats TAP [1], Junit [2], yaml. Let's add basic infrastructure so people may add report formats they do care about. Usecase example with Jenkins: - Split huge xfstest job to smaller peaces. - Submit it as separate jobs and run it in parallel on different VMs - Fetch and merge results to single job report. This approach allow me to speedup xfstests x50 times, now full run of '-g auto' takes only 3minutes TOC - check: prepare test report generator infrastructure - report:Add xunit format report generator Footnotes: [1] https://testanything.org [2] http://help.catchsoftware.com/display/ET/JUnit+Format ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 1/2] check: prepare test report generator infrastructure 2017-01-31 17:43 [PATCH 0/2][RESEND] Add report generators support Dmitry Monakhov @ 2017-01-31 17:43 ` Dmitry Monakhov 2017-02-16 9:18 ` Eryu Guan 2017-01-31 17:43 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 1 sibling, 1 reply; 14+ messages in thread From: Dmitry Monakhov @ 2017-01-31 17:43 UTC (permalink / raw) To: fstests; +Cc: Dmitry Monakhov Save testcase data which later may be used by report generators - Save failure reason to $err_msg variable - Save number of notrun tests to $n_notrun counter, similar to $n_try,$n_bad Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> --- check | 32 ++++++++++++++++++++++---------- common/btrfs | 12 ++++++------ common/rc | 20 +++++++++++++------- common/xfs | 33 +++++++++++++++++---------------- 4 files changed, 58 insertions(+), 39 deletions(-) diff --git a/check b/check index 5a93c94..edc0899 100755 --- a/check +++ b/check @@ -28,6 +28,7 @@ try="" n_bad=0 sum_bad=0 bad="" +n_notrun=0 notrun="" interrupt=true diff="diff -u" @@ -37,6 +38,7 @@ randomize=false export here=`pwd` xfile="" brief_test_summary=false +err_msg="" DUMP_OUTPUT=false @@ -368,6 +370,7 @@ _wipe_counters() { n_try="0" n_bad="0" + n_notrun="0" unset try notrun bad } @@ -596,6 +599,7 @@ for section in $HOST_OPTIONS_SECTIONS; do for seq in $list do err=false + err_msg="" if [ ! -f $seq ]; then # Try to get full name in case the user supplied only seq id # and the test has a name. A bit of hassle to find really @@ -629,14 +633,17 @@ for section in $HOST_OPTIONS_SECTIONS; do echo -n "$seqnum" - if $showme; then - echo - continue - fi + if $showme; then + echo + start=0 + stop=0 + n_notrun=`expr $n_notrun + 1` + continue + fi - if [ ! -f $seq ]; then - echo " - no such test?" - else + if [ ! -f $seq ]; then + echo " - no such test?" + else # really going to try and run this one # rm -f $seqres.out.bad @@ -684,7 +691,8 @@ for section in $HOST_OPTIONS_SECTIONS; do if [ -f core ] then - echo -n " [dumped core]" + err_msg="[dumped core]" + echo -n " $err_msg" mv core $RESULT_BASE/$seqnum.core err=true fi @@ -695,15 +703,18 @@ for section in $HOST_OPTIONS_SECTIONS; do $timestamp && echo " [not run]" && echo -n " $seqnum -- " cat $seqres.notrun notrun="$notrun $seqnum" + n_notrun=`expr $n_notrun + 1` else if [ $sts -ne 0 ] then - echo -n " [failed, exit status $sts]" + err_msg="[failed, exit status $sts]" + echo -n " $err_msg" err=true fi if [ ! -f $seq.out ] then - echo " - no qualified output" + err_msg="no qualified output" + echo "- $err_msg" err=true else @@ -733,6 +744,7 @@ for section in $HOST_OPTIONS_SECTIONS; do " to see the entire diff)" fi; } | \ sed -e 's/^\(.\)/ \1/' + err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)" err=true fi fi diff --git a/common/btrfs b/common/btrfs index 96c3635..64c40bd 100644 --- a/common/btrfs +++ b/common/btrfs @@ -105,24 +105,24 @@ _check_btrfs_filesystem() if [ -f ${RESULT_DIR}/require_scratch.require_qgroup_report ]; then $BTRFS_UTIL_PROG check $device --qgroup-report > $tmp.qgroup_report 2>&1 if grep -qE "Counts for qgroup.*are different" $tmp.qgroup_report ; then - echo "_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers (see $seqres.full)" - echo "_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers" \ - >> $seqres.full + msg="_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers" + echo "$msg" >> $seqres.full echo "*** qgroup_report.$FSTYP output ***" >>$seqres.full cat $tmp.qgroup_report >>$seqres.full echo "*** qgroup_report.$FSTYP output ***" >>$seqres.full + err_msg="$msg (see $seqres.full)" fi rm -f $tmp.qgroup_report fi $BTRFS_UTIL_PROG check $device >$tmp.fsck 2>&1 if [ $? -ne 0 ]; then - echo "_check_btrfs_filesystem: filesystem on $device is inconsistent (see $seqres.full)" - - echo "_check_btrfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full + msg="_check_btrfs_filesystem: filesystem on $device is inconsistent" + echo "$msg" >>$seqres.full echo "*** fsck.$FSTYP output ***" >>$seqres.full cat $tmp.fsck >>$seqres.full echo "*** end fsck.$FSTYP output" >>$seqres.full + err_msg="$msg (see $seqres.full)" ok=0 fi diff --git a/common/rc b/common/rc index 7e2eaee..f97b4d2 100644 --- a/common/rc +++ b/common/rc @@ -1017,7 +1017,10 @@ _repair_scratch_fs() _scratch_xfs_repair "$@" 2>&1 res=$? fi - test $res -ne 0 && >&2 echo "xfs_repair failed, err=$res" + if [ test $res -ne 0]; then + err_msg="xfs_repair failed, err=$res" + >&2 echo "$err_msg" + fi return $res ;; *) @@ -1029,7 +1032,8 @@ _repair_scratch_fs() res=0 ;; *) - >&2 echo "fsck.$FSTYP failed, err=$res" + err_msg="fsck.$FSTYP failed, err=$res" + >&2 echo "$err_msg" ;; esac return $res @@ -2130,7 +2134,8 @@ _mount_or_remount_rw() _overlay_mount $device $mountpoint fi if [ $? -ne 0 ]; then - echo "!!! failed to remount $device on $mountpoint" + err_msg="!!! failed to remount $device on $mountpoint" + echo "$err_msg" return 0 # ok=0 fi else @@ -2164,12 +2169,12 @@ _check_generic_filesystem() fsck -t $FSTYP $FSCK_OPTIONS $device >$tmp.fsck 2>&1 if [ $? -ne 0 ] then - echo "_check_generic_filesystem: filesystem on $device is inconsistent (see $seqres.full)" - - echo "_check_generic filesystem: filesystem on $device is inconsistent" >>$seqres.full + msg="_check_generic_filesystem: filesystem on $device is inconsistent" + echo "$msg" >>$seqres.full echo "*** fsck.$FSTYP output ***" >>$seqres.full cat $tmp.fsck >>$seqres.full echo "*** end fsck.$FSTYP output" >>$seqres.full + err_msg="$msg (see $seqres.full)" ok=0 fi @@ -3023,7 +3028,8 @@ _check_dmesg() -e "general protection fault:" \ $seqres.dmesg if [ $? -eq 0 ]; then - echo "_check_dmesg: something found in dmesg (see $seqres.dmesg)" + err_msg="_check_dmesg: something found in dmesg (see $seqres.dmesg)" + echo "$err_msg" return 1 else rm -f $seqres.dmesg diff --git a/common/xfs b/common/xfs index 767a481..ad4c505 100644 --- a/common/xfs +++ b/common/xfs @@ -333,7 +333,8 @@ _check_xfs_filesystem() if [ -n "$TEST_XFS_SCRUB" ] && [ -x "$XFS_SCRUB_PROG" ]; then "$XFS_SCRUB_PROG" $scrubflag -vd $device >>$seqres.full if [ $? -ne 0 ]; then - echo "filesystem on $device failed scrub (see $seqres.full)" + err_msg="filesystem on $device failed scrub (see $seqres.full)" + echo "$err_msg" ok=0 fi fi @@ -344,12 +345,12 @@ _check_xfs_filesystem() $XFS_LOGPRINT_PROG -t $extra_log_options $device 2>&1 \ | tee $tmp.logprint | grep -q "<CLEAN>" if [ $? -ne 0 -a "$HOSTOS" = "Linux" ]; then - echo "_check_xfs_filesystem: filesystem on $device has dirty log (see $seqres.full)" - - echo "_check_xfs_filesystem: filesystem on $device has dirty log" >>$seqres.full + msg="_check_xfs_filesystem: filesystem on $device has dirty log" + echo "$msg" >>$seqres.full echo "*** xfs_logprint -t output ***" >>$seqres.full cat $tmp.logprint >>$seqres.full echo "*** end xfs_logprint output" >>$seqres.full + err_msg="$msg (see $seqres.full)" ok=0 fi @@ -362,24 +363,24 @@ _check_xfs_filesystem() _fix_malloc >$tmp.fs_check fi if [ -s $tmp.fs_check ]; then - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (c) (see $seqres.full)" - - echo "_check_xfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (c)" + echo "$msg" >>$seqres.full echo "*** xfs_check output ***" >>$seqres.full cat $tmp.fs_check >>$seqres.full echo "*** end xfs_check output" >>$seqres.full + err_msg="$msg (see $seqres.full)" ok=0 fi $XFS_REPAIR_PROG -n $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 if [ $? -ne 0 ]; then - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (r) (see $seqres.full)" - - echo "_check_xfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (r)" + echo "$msg" >>$seqres.full echo "*** xfs_repair -n output ***" >>$seqres.full cat $tmp.repair | _fix_malloc >>$seqres.full echo "*** end xfs_repair output" >>$seqres.full + err_msg="$msg (see $seqres.full)" ok=0 fi @@ -389,12 +390,12 @@ _check_xfs_filesystem() if [ -n "$TEST_XFS_REPAIR_REBUILD" ]; then $XFS_REPAIR_PROG $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 if [ $? -ne 0 ]; then - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild) (see $seqres.full)" - - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild)" >>$seqres.full + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild)" + echo "$msg" >>$seqres.full echo "*** xfs_repair output ***" >>$seqres.full cat $tmp.repair | _fix_malloc >>$seqres.full echo "*** end xfs_repair output" >>$seqres.full + err_msg="$msg (see $seqres.full)" ok=0 fi @@ -402,12 +403,12 @@ _check_xfs_filesystem() $XFS_REPAIR_PROG -n $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 if [ $? -ne 0 ]; then - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify) (see $seqres.full)" - - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify)" >>$seqres.full + err_msg="_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify)" + echo "$msg" >>$seqres.full echo "*** xfs_repair -n output ***" >>$seqres.full cat $tmp.repair | _fix_malloc >>$seqres.full echo "*** end xfs_repair output" >>$seqres.full + err_msg="$msg (see $seqres.full)" ok=0 fi -- 2.9.3 ^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: [PATCH 1/2] check: prepare test report generator infrastructure 2017-01-31 17:43 ` [PATCH 1/2] check: prepare test report generator infrastructure Dmitry Monakhov @ 2017-02-16 9:18 ` Eryu Guan 2017-02-16 9:38 ` Dmitry Monakhov 0 siblings, 1 reply; 14+ messages in thread From: Eryu Guan @ 2017-02-16 9:18 UTC (permalink / raw) To: Dmitry Monakhov; +Cc: fstests, Dmitry Monakhov On Tue, Jan 31, 2017 at 09:43:20PM +0400, Dmitry Monakhov wrote: > Save testcase data which later may be used by report generators > - Save failure reason to $err_msg variable > - Save number of notrun tests to $n_notrun counter, similar to $n_try,$n_bad Sorry for the late review. > > Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> > --- > check | 32 ++++++++++++++++++++++---------- > common/btrfs | 12 ++++++------ > common/rc | 20 +++++++++++++------- > common/xfs | 33 +++++++++++++++++---------------- > 4 files changed, 58 insertions(+), 39 deletions(-) > > diff --git a/check b/check > index 5a93c94..edc0899 100755 > --- a/check > +++ b/check > @@ -28,6 +28,7 @@ try="" > n_bad=0 > sum_bad=0 > bad="" > +n_notrun=0 > notrun="" > interrupt=true > diff="diff -u" > @@ -37,6 +38,7 @@ randomize=false > export here=`pwd` > xfile="" > brief_test_summary=false > +err_msg="" > > DUMP_OUTPUT=false > > @@ -368,6 +370,7 @@ _wipe_counters() > { > n_try="0" > n_bad="0" > + n_notrun="0" > unset try notrun bad > } > > @@ -596,6 +599,7 @@ for section in $HOST_OPTIONS_SECTIONS; do > for seq in $list > do > err=false > + err_msg="" > if [ ! -f $seq ]; then > # Try to get full name in case the user supplied only seq id > # and the test has a name. A bit of hassle to find really > @@ -629,14 +633,17 @@ for section in $HOST_OPTIONS_SECTIONS; do > > echo -n "$seqnum" > > - if $showme; then > - echo > - continue > - fi > + if $showme; then > + echo > + start=0 > + stop=0 > + n_notrun=`expr $n_notrun + 1` > + continue > + fi > > - if [ ! -f $seq ]; then > - echo " - no such test?" > - else > + if [ ! -f $seq ]; then > + echo " - no such test?" > + else > # really going to try and run this one > # > rm -f $seqres.out.bad > @@ -684,7 +691,8 @@ for section in $HOST_OPTIONS_SECTIONS; do > > if [ -f core ] > then > - echo -n " [dumped core]" > + err_msg="[dumped core]" > + echo -n " $err_msg" > mv core $RESULT_BASE/$seqnum.core > err=true > fi > @@ -695,15 +703,18 @@ for section in $HOST_OPTIONS_SECTIONS; do > $timestamp && echo " [not run]" && echo -n " $seqnum -- " > cat $seqres.notrun > notrun="$notrun $seqnum" > + n_notrun=`expr $n_notrun + 1` > else > if [ $sts -ne 0 ] > then > - echo -n " [failed, exit status $sts]" > + err_msg="[failed, exit status $sts]" > + echo -n " $err_msg" > err=true > fi > if [ ! -f $seq.out ] > then > - echo " - no qualified output" > + err_msg="no qualified output" > + echo "- $err_msg" > err=true > else > > @@ -733,6 +744,7 @@ for section in $HOST_OPTIONS_SECTIONS; do > " to see the entire diff)" > fi; } | \ > sed -e 's/^\(.\)/ \1/' > + err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)" > err=true > fi > fi > diff --git a/common/btrfs b/common/btrfs > index 96c3635..64c40bd 100644 > --- a/common/btrfs > +++ b/common/btrfs > @@ -105,24 +105,24 @@ _check_btrfs_filesystem() > if [ -f ${RESULT_DIR}/require_scratch.require_qgroup_report ]; then > $BTRFS_UTIL_PROG check $device --qgroup-report > $tmp.qgroup_report 2>&1 > if grep -qE "Counts for qgroup.*are different" $tmp.qgroup_report ; then > - echo "_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers (see $seqres.full)" > - echo "_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers" \ > - >> $seqres.full > + msg="_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers" > + echo "$msg" >> $seqres.full > echo "*** qgroup_report.$FSTYP output ***" >>$seqres.full > cat $tmp.qgroup_report >>$seqres.full > echo "*** qgroup_report.$FSTYP output ***" >>$seqres.full > + err_msg="$msg (see $seqres.full)" This only assigns err_msg but doesn't echo it to stdout like what the original code does. The same issue applies to other fsck error report path. And I noticed that similar pattern repeats time and time again, maybe they can be abstracted into a helper? Thanks, Eryu > fi > rm -f $tmp.qgroup_report > fi > > $BTRFS_UTIL_PROG check $device >$tmp.fsck 2>&1 > if [ $? -ne 0 ]; then > - echo "_check_btrfs_filesystem: filesystem on $device is inconsistent (see $seqres.full)" > - > - echo "_check_btrfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full > + msg="_check_btrfs_filesystem: filesystem on $device is inconsistent" > + echo "$msg" >>$seqres.full > echo "*** fsck.$FSTYP output ***" >>$seqres.full > cat $tmp.fsck >>$seqres.full > echo "*** end fsck.$FSTYP output" >>$seqres.full > + err_msg="$msg (see $seqres.full)" > > ok=0 > fi > diff --git a/common/rc b/common/rc > index 7e2eaee..f97b4d2 100644 > --- a/common/rc > +++ b/common/rc > @@ -1017,7 +1017,10 @@ _repair_scratch_fs() > _scratch_xfs_repair "$@" 2>&1 > res=$? > fi > - test $res -ne 0 && >&2 echo "xfs_repair failed, err=$res" > + if [ test $res -ne 0]; then > + err_msg="xfs_repair failed, err=$res" > + >&2 echo "$err_msg" > + fi > return $res > ;; > *) > @@ -1029,7 +1032,8 @@ _repair_scratch_fs() > res=0 > ;; > *) > - >&2 echo "fsck.$FSTYP failed, err=$res" > + err_msg="fsck.$FSTYP failed, err=$res" > + >&2 echo "$err_msg" > ;; > esac > return $res > @@ -2130,7 +2134,8 @@ _mount_or_remount_rw() > _overlay_mount $device $mountpoint > fi > if [ $? -ne 0 ]; then > - echo "!!! failed to remount $device on $mountpoint" > + err_msg="!!! failed to remount $device on $mountpoint" > + echo "$err_msg" > return 0 # ok=0 > fi > else > @@ -2164,12 +2169,12 @@ _check_generic_filesystem() > fsck -t $FSTYP $FSCK_OPTIONS $device >$tmp.fsck 2>&1 > if [ $? -ne 0 ] > then > - echo "_check_generic_filesystem: filesystem on $device is inconsistent (see $seqres.full)" > - > - echo "_check_generic filesystem: filesystem on $device is inconsistent" >>$seqres.full > + msg="_check_generic_filesystem: filesystem on $device is inconsistent" > + echo "$msg" >>$seqres.full > echo "*** fsck.$FSTYP output ***" >>$seqres.full > cat $tmp.fsck >>$seqres.full > echo "*** end fsck.$FSTYP output" >>$seqres.full > + err_msg="$msg (see $seqres.full)" > > ok=0 > fi > @@ -3023,7 +3028,8 @@ _check_dmesg() > -e "general protection fault:" \ > $seqres.dmesg > if [ $? -eq 0 ]; then > - echo "_check_dmesg: something found in dmesg (see $seqres.dmesg)" > + err_msg="_check_dmesg: something found in dmesg (see $seqres.dmesg)" > + echo "$err_msg" > return 1 > else > rm -f $seqres.dmesg > diff --git a/common/xfs b/common/xfs > index 767a481..ad4c505 100644 > --- a/common/xfs > +++ b/common/xfs > @@ -333,7 +333,8 @@ _check_xfs_filesystem() > if [ -n "$TEST_XFS_SCRUB" ] && [ -x "$XFS_SCRUB_PROG" ]; then > "$XFS_SCRUB_PROG" $scrubflag -vd $device >>$seqres.full > if [ $? -ne 0 ]; then > - echo "filesystem on $device failed scrub (see $seqres.full)" > + err_msg="filesystem on $device failed scrub (see $seqres.full)" > + echo "$err_msg" > ok=0 > fi > fi > @@ -344,12 +345,12 @@ _check_xfs_filesystem() > $XFS_LOGPRINT_PROG -t $extra_log_options $device 2>&1 \ > | tee $tmp.logprint | grep -q "<CLEAN>" > if [ $? -ne 0 -a "$HOSTOS" = "Linux" ]; then > - echo "_check_xfs_filesystem: filesystem on $device has dirty log (see $seqres.full)" > - > - echo "_check_xfs_filesystem: filesystem on $device has dirty log" >>$seqres.full > + msg="_check_xfs_filesystem: filesystem on $device has dirty log" > + echo "$msg" >>$seqres.full > echo "*** xfs_logprint -t output ***" >>$seqres.full > cat $tmp.logprint >>$seqres.full > echo "*** end xfs_logprint output" >>$seqres.full > + err_msg="$msg (see $seqres.full)" > > ok=0 > fi > @@ -362,24 +363,24 @@ _check_xfs_filesystem() > _fix_malloc >$tmp.fs_check > fi > if [ -s $tmp.fs_check ]; then > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (c) (see $seqres.full)" > - > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full > + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (c)" > + echo "$msg" >>$seqres.full > echo "*** xfs_check output ***" >>$seqres.full > cat $tmp.fs_check >>$seqres.full > echo "*** end xfs_check output" >>$seqres.full > + err_msg="$msg (see $seqres.full)" > > ok=0 > fi > > $XFS_REPAIR_PROG -n $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 > if [ $? -ne 0 ]; then > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (r) (see $seqres.full)" > - > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full > + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (r)" > + echo "$msg" >>$seqres.full > echo "*** xfs_repair -n output ***" >>$seqres.full > cat $tmp.repair | _fix_malloc >>$seqres.full > echo "*** end xfs_repair output" >>$seqres.full > + err_msg="$msg (see $seqres.full)" > > ok=0 > fi > @@ -389,12 +390,12 @@ _check_xfs_filesystem() > if [ -n "$TEST_XFS_REPAIR_REBUILD" ]; then > $XFS_REPAIR_PROG $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 > if [ $? -ne 0 ]; then > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild) (see $seqres.full)" > - > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild)" >>$seqres.full > + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild)" > + echo "$msg" >>$seqres.full > echo "*** xfs_repair output ***" >>$seqres.full > cat $tmp.repair | _fix_malloc >>$seqres.full > echo "*** end xfs_repair output" >>$seqres.full > + err_msg="$msg (see $seqres.full)" > > ok=0 > fi > @@ -402,12 +403,12 @@ _check_xfs_filesystem() > > $XFS_REPAIR_PROG -n $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 > if [ $? -ne 0 ]; then > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify) (see $seqres.full)" > - > - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify)" >>$seqres.full > + err_msg="_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify)" > + echo "$msg" >>$seqres.full > echo "*** xfs_repair -n output ***" >>$seqres.full > cat $tmp.repair | _fix_malloc >>$seqres.full > echo "*** end xfs_repair output" >>$seqres.full > + err_msg="$msg (see $seqres.full)" > > ok=0 > fi > -- > 2.9.3 > > -- > To unsubscribe from this list: send the line "unsubscribe fstests" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html ^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 1/2] check: prepare test report generator infrastructure 2017-02-16 9:18 ` Eryu Guan @ 2017-02-16 9:38 ` Dmitry Monakhov 0 siblings, 0 replies; 14+ messages in thread From: Dmitry Monakhov @ 2017-02-16 9:38 UTC (permalink / raw) To: Eryu Guan, Dmitry Monakhov; +Cc: fstests Eryu Guan <eguan@redhat.com> writes: > On Tue, Jan 31, 2017 at 09:43:20PM +0400, Dmitry Monakhov wrote: >> Save testcase data which later may be used by report generators >> - Save failure reason to $err_msg variable >> - Save number of notrun tests to $n_notrun counter, similar to $n_try,$n_bad > > Sorry for the late review. > >> >> Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> >> --- >> check | 32 ++++++++++++++++++++++---------- >> common/btrfs | 12 ++++++------ >> common/rc | 20 +++++++++++++------- >> common/xfs | 33 +++++++++++++++++---------------- >> 4 files changed, 58 insertions(+), 39 deletions(-) >> >> diff --git a/check b/check >> index 5a93c94..edc0899 100755 >> --- a/check >> +++ b/check >> @@ -28,6 +28,7 @@ try="" >> n_bad=0 >> sum_bad=0 >> bad="" >> +n_notrun=0 >> notrun="" >> interrupt=true >> diff="diff -u" >> @@ -37,6 +38,7 @@ randomize=false >> export here=`pwd` >> xfile="" >> brief_test_summary=false >> +err_msg="" >> >> DUMP_OUTPUT=false >> >> @@ -368,6 +370,7 @@ _wipe_counters() >> { >> n_try="0" >> n_bad="0" >> + n_notrun="0" >> unset try notrun bad >> } >> >> @@ -596,6 +599,7 @@ for section in $HOST_OPTIONS_SECTIONS; do >> for seq in $list >> do >> err=false >> + err_msg="" >> if [ ! -f $seq ]; then >> # Try to get full name in case the user supplied only seq id >> # and the test has a name. A bit of hassle to find really >> @@ -629,14 +633,17 @@ for section in $HOST_OPTIONS_SECTIONS; do >> >> echo -n "$seqnum" >> >> - if $showme; then >> - echo >> - continue >> - fi >> + if $showme; then >> + echo >> + start=0 >> + stop=0 >> + n_notrun=`expr $n_notrun + 1` >> + continue >> + fi >> >> - if [ ! -f $seq ]; then >> - echo " - no such test?" >> - else >> + if [ ! -f $seq ]; then >> + echo " - no such test?" >> + else >> # really going to try and run this one >> # >> rm -f $seqres.out.bad >> @@ -684,7 +691,8 @@ for section in $HOST_OPTIONS_SECTIONS; do >> >> if [ -f core ] >> then >> - echo -n " [dumped core]" >> + err_msg="[dumped core]" >> + echo -n " $err_msg" >> mv core $RESULT_BASE/$seqnum.core >> err=true >> fi >> @@ -695,15 +703,18 @@ for section in $HOST_OPTIONS_SECTIONS; do >> $timestamp && echo " [not run]" && echo -n " $seqnum -- " >> cat $seqres.notrun >> notrun="$notrun $seqnum" >> + n_notrun=`expr $n_notrun + 1` >> else >> if [ $sts -ne 0 ] >> then >> - echo -n " [failed, exit status $sts]" >> + err_msg="[failed, exit status $sts]" >> + echo -n " $err_msg" >> err=true >> fi >> if [ ! -f $seq.out ] >> then >> - echo " - no qualified output" >> + err_msg="no qualified output" >> + echo "- $err_msg" >> err=true >> else >> >> @@ -733,6 +744,7 @@ for section in $HOST_OPTIONS_SECTIONS; do >> " to see the entire diff)" >> fi; } | \ >> sed -e 's/^\(.\)/ \1/' >> + err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)" >> err=true >> fi >> fi >> diff --git a/common/btrfs b/common/btrfs >> index 96c3635..64c40bd 100644 >> --- a/common/btrfs >> +++ b/common/btrfs >> @@ -105,24 +105,24 @@ _check_btrfs_filesystem() >> if [ -f ${RESULT_DIR}/require_scratch.require_qgroup_report ]; then >> $BTRFS_UTIL_PROG check $device --qgroup-report > $tmp.qgroup_report 2>&1 >> if grep -qE "Counts for qgroup.*are different" $tmp.qgroup_report ; then >> - echo "_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers (see $seqres.full)" >> - echo "_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers" \ >> - >> $seqres.full >> + msg="_check_btrfs_filesystem: filesystem on $device has wrong qgroup numbers" >> + echo "$msg" >> $seqres.full >> echo "*** qgroup_report.$FSTYP output ***" >>$seqres.full >> cat $tmp.qgroup_report >>$seqres.full >> echo "*** qgroup_report.$FSTYP output ***" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" > > This only assigns err_msg but doesn't echo it to stdout like what the > original code does. The same issue applies to other fsck error report > path. > > And I noticed that similar pattern repeats time and time again, maybe > they can be abstracted into a helper? Definitely. This is reasonable idea, I'll resend updated version. > > Thanks, > Eryu > >> fi >> rm -f $tmp.qgroup_report >> fi >> >> $BTRFS_UTIL_PROG check $device >$tmp.fsck 2>&1 >> if [ $? -ne 0 ]; then >> - echo "_check_btrfs_filesystem: filesystem on $device is inconsistent (see $seqres.full)" >> - >> - echo "_check_btrfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full >> + msg="_check_btrfs_filesystem: filesystem on $device is inconsistent" >> + echo "$msg" >>$seqres.full >> echo "*** fsck.$FSTYP output ***" >>$seqres.full >> cat $tmp.fsck >>$seqres.full >> echo "*** end fsck.$FSTYP output" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" >> >> ok=0 >> fi >> diff --git a/common/rc b/common/rc >> index 7e2eaee..f97b4d2 100644 >> --- a/common/rc >> +++ b/common/rc >> @@ -1017,7 +1017,10 @@ _repair_scratch_fs() >> _scratch_xfs_repair "$@" 2>&1 >> res=$? >> fi >> - test $res -ne 0 && >&2 echo "xfs_repair failed, err=$res" >> + if [ test $res -ne 0]; then >> + err_msg="xfs_repair failed, err=$res" >> + >&2 echo "$err_msg" >> + fi >> return $res >> ;; >> *) >> @@ -1029,7 +1032,8 @@ _repair_scratch_fs() >> res=0 >> ;; >> *) >> - >&2 echo "fsck.$FSTYP failed, err=$res" >> + err_msg="fsck.$FSTYP failed, err=$res" >> + >&2 echo "$err_msg" >> ;; >> esac >> return $res >> @@ -2130,7 +2134,8 @@ _mount_or_remount_rw() >> _overlay_mount $device $mountpoint >> fi >> if [ $? -ne 0 ]; then >> - echo "!!! failed to remount $device on $mountpoint" >> + err_msg="!!! failed to remount $device on $mountpoint" >> + echo "$err_msg" >> return 0 # ok=0 >> fi >> else >> @@ -2164,12 +2169,12 @@ _check_generic_filesystem() >> fsck -t $FSTYP $FSCK_OPTIONS $device >$tmp.fsck 2>&1 >> if [ $? -ne 0 ] >> then >> - echo "_check_generic_filesystem: filesystem on $device is inconsistent (see $seqres.full)" >> - >> - echo "_check_generic filesystem: filesystem on $device is inconsistent" >>$seqres.full >> + msg="_check_generic_filesystem: filesystem on $device is inconsistent" >> + echo "$msg" >>$seqres.full >> echo "*** fsck.$FSTYP output ***" >>$seqres.full >> cat $tmp.fsck >>$seqres.full >> echo "*** end fsck.$FSTYP output" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" >> >> ok=0 >> fi >> @@ -3023,7 +3028,8 @@ _check_dmesg() >> -e "general protection fault:" \ >> $seqres.dmesg >> if [ $? -eq 0 ]; then >> - echo "_check_dmesg: something found in dmesg (see $seqres.dmesg)" >> + err_msg="_check_dmesg: something found in dmesg (see $seqres.dmesg)" >> + echo "$err_msg" >> return 1 >> else >> rm -f $seqres.dmesg >> diff --git a/common/xfs b/common/xfs >> index 767a481..ad4c505 100644 >> --- a/common/xfs >> +++ b/common/xfs >> @@ -333,7 +333,8 @@ _check_xfs_filesystem() >> if [ -n "$TEST_XFS_SCRUB" ] && [ -x "$XFS_SCRUB_PROG" ]; then >> "$XFS_SCRUB_PROG" $scrubflag -vd $device >>$seqres.full >> if [ $? -ne 0 ]; then >> - echo "filesystem on $device failed scrub (see $seqres.full)" >> + err_msg="filesystem on $device failed scrub (see $seqres.full)" >> + echo "$err_msg" >> ok=0 >> fi >> fi >> @@ -344,12 +345,12 @@ _check_xfs_filesystem() >> $XFS_LOGPRINT_PROG -t $extra_log_options $device 2>&1 \ >> | tee $tmp.logprint | grep -q "<CLEAN>" >> if [ $? -ne 0 -a "$HOSTOS" = "Linux" ]; then >> - echo "_check_xfs_filesystem: filesystem on $device has dirty log (see $seqres.full)" >> - >> - echo "_check_xfs_filesystem: filesystem on $device has dirty log" >>$seqres.full >> + msg="_check_xfs_filesystem: filesystem on $device has dirty log" >> + echo "$msg" >>$seqres.full >> echo "*** xfs_logprint -t output ***" >>$seqres.full >> cat $tmp.logprint >>$seqres.full >> echo "*** end xfs_logprint output" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" >> >> ok=0 >> fi >> @@ -362,24 +363,24 @@ _check_xfs_filesystem() >> _fix_malloc >$tmp.fs_check >> fi >> if [ -s $tmp.fs_check ]; then >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (c) (see $seqres.full)" >> - >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full >> + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (c)" >> + echo "$msg" >>$seqres.full >> echo "*** xfs_check output ***" >>$seqres.full >> cat $tmp.fs_check >>$seqres.full >> echo "*** end xfs_check output" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" >> >> ok=0 >> fi >> >> $XFS_REPAIR_PROG -n $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 >> if [ $? -ne 0 ]; then >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (r) (see $seqres.full)" >> - >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent" >>$seqres.full >> + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (r)" >> + echo "$msg" >>$seqres.full >> echo "*** xfs_repair -n output ***" >>$seqres.full >> cat $tmp.repair | _fix_malloc >>$seqres.full >> echo "*** end xfs_repair output" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" >> >> ok=0 >> fi >> @@ -389,12 +390,12 @@ _check_xfs_filesystem() >> if [ -n "$TEST_XFS_REPAIR_REBUILD" ]; then >> $XFS_REPAIR_PROG $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 >> if [ $? -ne 0 ]; then >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild) (see $seqres.full)" >> - >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild)" >>$seqres.full >> + msg="_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild)" >> + echo "$msg" >>$seqres.full >> echo "*** xfs_repair output ***" >>$seqres.full >> cat $tmp.repair | _fix_malloc >>$seqres.full >> echo "*** end xfs_repair output" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" >> >> ok=0 >> fi >> @@ -402,12 +403,12 @@ _check_xfs_filesystem() >> >> $XFS_REPAIR_PROG -n $extra_options $extra_log_options $extra_rt_options $device >$tmp.repair 2>&1 >> if [ $? -ne 0 ]; then >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify) (see $seqres.full)" >> - >> - echo "_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify)" >>$seqres.full >> + err_msg="_check_xfs_filesystem: filesystem on $device is inconsistent (rebuild-reverify)" >> + echo "$msg" >>$seqres.full >> echo "*** xfs_repair -n output ***" >>$seqres.full >> cat $tmp.repair | _fix_malloc >>$seqres.full >> echo "*** end xfs_repair output" >>$seqres.full >> + err_msg="$msg (see $seqres.full)" >> >> ok=0 >> fi >> -- >> 2.9.3 >> >> -- >> To unsubscribe from this list: send the line "unsubscribe fstests" in >> the body of a message to majordomo@vger.kernel.org >> More majordomo info at http://vger.kernel.org/majordomo-info.html ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/2] report: Add xunit format report generator 2017-01-31 17:43 [PATCH 0/2][RESEND] Add report generators support Dmitry Monakhov 2017-01-31 17:43 ` [PATCH 1/2] check: prepare test report generator infrastructure Dmitry Monakhov @ 2017-01-31 17:43 ` Dmitry Monakhov 2017-02-16 10:08 ` Eryu Guan 1 sibling, 1 reply; 14+ messages in thread From: Dmitry Monakhov @ 2017-01-31 17:43 UTC (permalink / raw) To: fstests; +Cc: Dmitry Monakhov xunit[1]/junit[2] are well known report formats for tests frameworks which supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) Basically this is just xml document which can be easily parsed later by external tools. EXAMPLE: #./check --xunit -s ext4 generic/001 generic/010 generic/013 #cat results/ext4/result.xml <?xml version="1.0" encoding="UTF-8"?> <testsuite name="xfstests" errors="0" skipped="1" tests="3" time="12" timestamp="2017-01-31 18:24:06" > <testcase classname="ext4" name="generic/001" time="6"> </testcase> <testcase classname="ext4" name="generic/010" time="0"> <skipped message="src/dbtest not built" /> </testcase> <testcase classname="ext4" name="generic/013" time="3"> </testcase> </testsuite> Footnotes: [1] https://xunit.github.io/docs/format-xml-v2.html [2] http://help.catchsoftware.com/display/ET/JUnit+Format [3] jenkins.io [4] https://www.atlassian.com/software/bamboo [5] https://github.com/avocado-framework/avocado Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> --- check | 22 +++++++++--- common/rc | 3 ++ common/report | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 common/report diff --git a/check b/check index edc0899..1442b6c 100755 --- a/check +++ b/check @@ -54,7 +54,7 @@ export DIFF_LENGTH=${DIFF_LENGTH:=10} # by default don't output timestamps timestamp=${TIMESTAMP:=false} -rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist +rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.* SRC_GROUPS="generic shared" export SRC_DIR="tests" @@ -75,6 +75,7 @@ check options -r randomize test order -d dump test output to stdout -b brief test summary + --xunit generate summary in xunit format --large-fs optimise scratch device for large filesystems -s section run only specified section from config file -S section exclude the specified section from the config file @@ -296,6 +297,7 @@ while [ $# -gt 0 ]; do -d) DUMP_OUTPUT=true ;; -b) brief_test_summary=true;; + --xunit) REPORT_XUNIT=yes; DO_REPORT=yes ;; --large-fs) export LARGE_SCRATCH_DEV=yes ;; --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;; @@ -380,7 +382,10 @@ _wrapup() check="$RESULT_BASE/check" if $showme; then - : + if $needwrap; then + _make_section_report + needwrap=false + fi elif $needwrap; then if [ -f $check.time -a -f $tmp.time ]; then cat $check.time $tmp.time \ @@ -431,9 +436,9 @@ _wrapup() echo "Passed all $n_try tests" >>$tmp.summary fi echo "" >>$tmp.summary + _make_section_report needwrap=false fi - sum_bad=`expr $sum_bad + $n_bad` _wipe_counters rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time @@ -519,6 +524,7 @@ for section in $HOST_OPTIONS_SECTIONS; do echo "SECTION -- $section" fi + sect_start=`_wallclock` if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then echo "RECREATING -- $FSTYP on $TEST_DEV" _test_unmount 2> /dev/null @@ -624,9 +630,11 @@ for section in $HOST_OPTIONS_SECTIONS; do if $OPTIONS_HAVE_SECTIONS; then export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"` seqres="$RESULT_BASE/$section/$seqnum" + sectdir_res="$RESULT_BASE/$section" else export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"` seqres="$RESULT_BASE/$seqnum" + sectdir_res="$RESULT_BASE" fi mkdir -p $RESULT_DIR @@ -638,9 +646,10 @@ for section in $HOST_OPTIONS_SECTIONS; do start=0 stop=0 n_notrun=`expr $n_notrun + 1` + _make_testcase_report "list" continue fi - + tc_status="pass" if [ ! -f $seq ]; then echo " - no such test?" else @@ -704,6 +713,7 @@ for section in $HOST_OPTIONS_SECTIONS; do cat $seqres.notrun notrun="$notrun $seqnum" n_notrun=`expr $n_notrun + 1` + tc_status="notrun" else if [ $sts -ne 0 ] then @@ -763,10 +773,12 @@ for section in $HOST_OPTIONS_SECTIONS; do bad="$bad $seqnum" n_bad=`expr $n_bad + 1` quick=false + tc_status="fail" fi - + _make_testcase_report "$tc_status" seq="after_$seqnum" done + sect_stop=`_wallclock` _wrapup echo diff --git a/common/rc b/common/rc index f97b4d2..9f1a068 100644 --- a/common/rc +++ b/common/rc @@ -3246,6 +3246,9 @@ _get_fs_sysfs_attr() } +if [ ! -z "$DO_REPORT" ]; then + . ./common/report +fi init_rc diff --git a/common/report b/common/report new file mode 100644 index 0000000..05496c5 --- /dev/null +++ b/common/report @@ -0,0 +1,108 @@ +# +# Reports generator funcitons lives here +# + + +# +# Xunit format report functions +_xunit_make_section_report() +{ + # section ==> testsuite + #<?xml version="1.0" encoding="UTF-8"?> + #<testsuite errors="0" failures="1" name="avocado" skipped="1" tests="3" time="0.0942118167877" timestamp="2017-01-30 21:43:42.117398"> + sect_name=$section + sect_time=`expr $sect_stop - $sect_start` + + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + fi + report=$tmp.report.xunit.$sect_name.xml + + n_total=`expr $n_try + $n_notrun` + echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $sectdir_res/result.xml + stats="errors=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\" timestamp=\"$date_time\"" + echo "<testsuite name=\"xfstests\" $stats >" >> $sectdir_res/result.xml + cat $tmp.report.xunit.$sect_name.xml >> $sectdir_res/result.xml + echo "</testsuite>" >> $sectdir_res/result.xml + echo "Xunit report: $sectdir_res/result.xml" +} + +_xunit_make_testcase_report() +{ + test_status="$1" + test_time=`expr $stop - $start` + sect_name=$section + + #TODO: other places may also win if no-section mode will be named like 'default/global' + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + fi + report=$tmp.report.xunit.$sect_name.xml + strip="$SRC_DIR/" + test_name=${seq#$strip} + echo -e "\t<testcase classname=\"$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report + case $test_status in + "pass") + ;; + "notrun") + if [ -f $seqres.notrun ]; then + msg=`cat $seqres.notrun` + echo -e "\t\t<skipped message=\"$msg\" />" >> $report + else + echo -e "\t\t<skipped/>" >> $report + fi + ;; + "list") + echo -e "\t\t<skipped/>" >> $report + ;; + "fail") + if [ -z "$err_msg" ]; then + err_msg="Test $sequm failed, reason unknown" + fi + echo -e "\t\t<failure message=\"$err_msg\" type=\"TestFail\">" >> $report + if [ -f $seqres.full ]; then + echo -e "\t\t\t<system-out>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.full | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t\t</system-out>" >> $report + fi + if [ -f $seqres.dmesg ]; then + echo -e "\t\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.dmesg | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t\t</system-err>" >> $report + elif [ -f $seqres.out.bad ]; then + echo -e "\t\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + $diff $seq.out $seqres.out.bad >>$report + printf ']]>\n' >>$report + echo -e "\t\t\t</system-err>" >> $report + fi + echo -e "\t\t</failure>" >> $report + ;; + *) + echo -e "\t\t<failure message=\"Unknown ret_state=$ret_state\" type=\"TestFail\"/>" >> $report + ;; + esac + echo -e "\t</testcase>" >> $report +} + + +# +# Common report generator entry points +_make_section_report() +{ + if [ ! -z "$REPORT_XUNIT" ]; then + _xunit_make_section_report + fi +} + +_make_testcase_report() +{ + test_status="$1" + if [ ! -z "$REPORT_XUNIT" ]; then + _xunit_make_testcase_report "$test_status" + fi +} -- 2.9.3 ^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: [PATCH 2/2] report: Add xunit format report generator 2017-01-31 17:43 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov @ 2017-02-16 10:08 ` Eryu Guan 0 siblings, 0 replies; 14+ messages in thread From: Eryu Guan @ 2017-02-16 10:08 UTC (permalink / raw) To: Dmitry Monakhov; +Cc: fstests, Dmitry Monakhov On Tue, Jan 31, 2017 at 09:43:21PM +0400, Dmitry Monakhov wrote: > xunit[1]/junit[2] are well known report formats for tests frameworks which > supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) > Basically this is just xml document which can be easily parsed later by external tools. > > EXAMPLE: > #./check --xunit -s ext4 generic/001 generic/010 generic/013 > #cat results/ext4/result.xml > <?xml version="1.0" encoding="UTF-8"?> > <testsuite name="xfstests" errors="0" skipped="1" tests="3" time="12" timestamp="2017-01-31 18:24:06" > > <testcase classname="ext4" name="generic/001" time="6"> > </testcase> > <testcase classname="ext4" name="generic/010" time="0"> > <skipped message="src/dbtest not built" /> > </testcase> > <testcase classname="ext4" name="generic/013" time="3"> > </testcase> > </testsuite> > > Footnotes: > [1] https://xunit.github.io/docs/format-xml-v2.html > [2] http://help.catchsoftware.com/display/ET/JUnit+Format > [3] jenkins.io > [4] https://www.atlassian.com/software/bamboo > [5] https://github.com/avocado-framework/avocado > > Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> > --- > check | 22 +++++++++--- > common/rc | 3 ++ > common/report | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ > 3 files changed, 128 insertions(+), 5 deletions(-) > create mode 100644 common/report > > diff --git a/check b/check > index edc0899..1442b6c 100755 > --- a/check > +++ b/check > @@ -54,7 +54,7 @@ export DIFF_LENGTH=${DIFF_LENGTH:=10} > # by default don't output timestamps > timestamp=${TIMESTAMP:=false} > > -rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist > +rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.* > > SRC_GROUPS="generic shared" > export SRC_DIR="tests" > @@ -75,6 +75,7 @@ check options > -r randomize test order > -d dump test output to stdout > -b brief test summary > + --xunit generate summary in xunit format > --large-fs optimise scratch device for large filesystems > -s section run only specified section from config file > -S section exclude the specified section from the config file > @@ -296,6 +297,7 @@ while [ $# -gt 0 ]; do > -d) DUMP_OUTPUT=true ;; > -b) brief_test_summary=true;; > > + --xunit) REPORT_XUNIT=yes; DO_REPORT=yes ;; So adding a new report format needs a new check option. How about introducing only one new option, e.g. "-R", which accepts the report format as argument? e.g. -R) REPORT_FMT=$2;; And do case switch against $REPORT_FMT in _make_section_report() and _make_testcase_report(), and from check command line we do ./check -R xunit -g auto > --large-fs) export LARGE_SCRATCH_DEV=yes ;; > --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;; > > @@ -380,7 +382,10 @@ _wrapup() > check="$RESULT_BASE/check" > > if $showme; then > - : > + if $needwrap; then > + _make_section_report > + needwrap=false > + fi > elif $needwrap; then > if [ -f $check.time -a -f $tmp.time ]; then > cat $check.time $tmp.time \ > @@ -431,9 +436,9 @@ _wrapup() > echo "Passed all $n_try tests" >>$tmp.summary > fi > echo "" >>$tmp.summary > + _make_section_report > needwrap=false > fi > - > sum_bad=`expr $sum_bad + $n_bad` > _wipe_counters > rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time > @@ -519,6 +524,7 @@ for section in $HOST_OPTIONS_SECTIONS; do > echo "SECTION -- $section" > fi > > + sect_start=`_wallclock` > if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then > echo "RECREATING -- $FSTYP on $TEST_DEV" > _test_unmount 2> /dev/null > @@ -624,9 +630,11 @@ for section in $HOST_OPTIONS_SECTIONS; do > if $OPTIONS_HAVE_SECTIONS; then > export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"` > seqres="$RESULT_BASE/$section/$seqnum" > + sectdir_res="$RESULT_BASE/$section" > else > export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"` > seqres="$RESULT_BASE/$seqnum" > + sectdir_res="$RESULT_BASE" sectdir_res seems vague from its name to me, after looking at the code I notice it's the dir to store report xml. How about name like REPORT_DIR? After thinking about it more, I think RESULT_DIR probably could be just $RESULT_BASE/$section (have section) or $RESULT_BASE (not tested)? And we can save report.xml to $RESULT_DIR. > fi > > mkdir -p $RESULT_DIR > @@ -638,9 +646,10 @@ for section in $HOST_OPTIONS_SECTIONS; do > start=0 > stop=0 > n_notrun=`expr $n_notrun + 1` > + _make_testcase_report "list" > continue > fi > - > + tc_status="pass" > if [ ! -f $seq ]; then > echo " - no such test?" > else > @@ -704,6 +713,7 @@ for section in $HOST_OPTIONS_SECTIONS; do > cat $seqres.notrun > notrun="$notrun $seqnum" > n_notrun=`expr $n_notrun + 1` > + tc_status="notrun" > else > if [ $sts -ne 0 ] > then > @@ -763,10 +773,12 @@ for section in $HOST_OPTIONS_SECTIONS; do > bad="$bad $seqnum" > n_bad=`expr $n_bad + 1` > quick=false > + tc_status="fail" > fi > - > + _make_testcase_report "$tc_status" > seq="after_$seqnum" > done > + sect_stop=`_wallclock` > _wrapup > echo > > diff --git a/common/rc b/common/rc > index f97b4d2..9f1a068 100644 > --- a/common/rc > +++ b/common/rc > @@ -3246,6 +3246,9 @@ _get_fs_sysfs_attr() > } > > > +if [ ! -z "$DO_REPORT" ]; then > + . ./common/report > +fi Better move this part to be near the per-fs setup in common/rc. > > init_rc > > diff --git a/common/report b/common/report > new file mode 100644 > index 0000000..05496c5 > --- /dev/null > +++ b/common/report > @@ -0,0 +1,108 @@ > +# > +# Reports generator funcitons lives here > +# > + > + > +# > +# Xunit format report functions > +_xunit_make_section_report() > +{ > + # section ==> testsuite > + #<?xml version="1.0" encoding="UTF-8"?> > + #<testsuite errors="0" failures="1" name="avocado" skipped="1" tests="3" time="0.0942118167877" timestamp="2017-01-30 21:43:42.117398"> > + sect_name=$section > + sect_time=`expr $sect_stop - $sect_start` Better to declare local variables with "local". > + > + if [ $sect_name == '-no-sections-' ]; then > + sect_name='global' > + fi > + report=$tmp.report.xunit.$sect_name.xml > + > + n_total=`expr $n_try + $n_notrun` > + echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $sectdir_res/result.xml > + stats="errors=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\" timestamp=\"$date_time\"" > + echo "<testsuite name=\"xfstests\" $stats >" >> $sectdir_res/result.xml > + cat $tmp.report.xunit.$sect_name.xml >> $sectdir_res/result.xml > + echo "</testsuite>" >> $sectdir_res/result.xml > + echo "Xunit report: $sectdir_res/result.xml" > +} > + > +_xunit_make_testcase_report() > +{ > + test_status="$1" > + test_time=`expr $stop - $start` > + sect_name=$section > + > + #TODO: other places may also win if no-section mode will be named like 'default/global' > + if [ $sect_name == '-no-sections-' ]; then > + sect_name='global' > + fi > + report=$tmp.report.xunit.$sect_name.xml > + strip="$SRC_DIR/" > + test_name=${seq#$strip} > + echo -e "\t<testcase classname=\"$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report > + case $test_status in > + "pass") > + ;; > + "notrun") > + if [ -f $seqres.notrun ]; then > + msg=`cat $seqres.notrun` > + echo -e "\t\t<skipped message=\"$msg\" />" >> $report > + else > + echo -e "\t\t<skipped/>" >> $report > + fi > + ;; > + "list") > + echo -e "\t\t<skipped/>" >> $report > + ;; > + "fail") > + if [ -z "$err_msg" ]; then > + err_msg="Test $sequm failed, reason unknown" > + fi > + echo -e "\t\t<failure message=\"$err_msg\" type=\"TestFail\">" >> $report > + if [ -f $seqres.full ]; then Sometimes I see empty $seqres.full, so does this system-out section. Perhaps use "if [ -s $seqres.full ]"? Thanks, Eryu > + echo -e "\t\t\t<system-out>" >> $report > + printf '<![CDATA[\n' >>$report > + cat $seqres.full | tr -dc '[:print:][:space:]' >>$report > + printf ']]>\n' >>$report > + echo -e "\t\t\t</system-out>" >> $report > + fi > + if [ -f $seqres.dmesg ]; then > + echo -e "\t\t\t<system-err>" >> $report > + printf '<![CDATA[\n' >>$report > + cat $seqres.dmesg | tr -dc '[:print:][:space:]' >>$report > + printf ']]>\n' >>$report > + echo -e "\t\t\t</system-err>" >> $report > + elif [ -f $seqres.out.bad ]; then > + echo -e "\t\t\t<system-err>" >> $report > + printf '<![CDATA[\n' >>$report > + $diff $seq.out $seqres.out.bad >>$report > + printf ']]>\n' >>$report > + echo -e "\t\t\t</system-err>" >> $report > + fi > + echo -e "\t\t</failure>" >> $report > + ;; > + *) > + echo -e "\t\t<failure message=\"Unknown ret_state=$ret_state\" type=\"TestFail\"/>" >> $report > + ;; > + esac > + echo -e "\t</testcase>" >> $report > +} > + > + > +# > +# Common report generator entry points > +_make_section_report() > +{ > + if [ ! -z "$REPORT_XUNIT" ]; then > + _xunit_make_section_report > + fi > +} > + > +_make_testcase_report() > +{ > + test_status="$1" > + if [ ! -z "$REPORT_XUNIT" ]; then > + _xunit_make_testcase_report "$test_status" > + fi > +} > -- > 2.9.3 > > -- > To unsubscribe from this list: send the line "unsubscribe fstests" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 0/2] Add report generators support v3 @ 2017-03-03 8:26 Dmitry Monakhov 2017-03-03 8:26 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 0 siblings, 1 reply; 14+ messages in thread From: Dmitry Monakhov @ 2017-03-03 8:26 UTC (permalink / raw) To: fstests; +Cc: eguan, Dmitry Monakhov xfstests has no stable logging API which is suitable for machine parsing. This makes it difficult to integrate with external atomation tools (Jenkins.io, k8s jobs) In fact thare are alot of well known parsable report formats TAP [1], Junit [2], yaml. Let's add basic infrastructure so people may add report formats they do care about. Usecase example with Jenkins: - Split huge xfstest job to smaller peaces. - Submit it as separate jobs and run it in parallel on different VMs - Fetch and merge results to single job report. This approach allow me to speedup xfstests x50 times, now full run of '-g auto' takes only 3minutes TOC - check: prepare test report generator infrastructure - report:Add xunit format report generator changes from v2: - fixes in response to comments from eguan@ - update against recent HEAD changes from v1: - fixes in response to comments from eguan@ - Add env variables to report . Footnotes: [1] https://testanything.org [2] http://help.catchsoftware.com/display/ET/JUnit+Format ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/2] report: Add xunit format report generator 2017-03-03 8:26 [PATCH 0/2] Add report generators support v3 Dmitry Monakhov @ 2017-03-03 8:26 ` Dmitry Monakhov 2017-03-07 12:05 ` Eryu Guan 0 siblings, 1 reply; 14+ messages in thread From: Dmitry Monakhov @ 2017-03-03 8:26 UTC (permalink / raw) To: fstests; +Cc: eguan, Dmitry Monakhov xunit[1]/junit[2] are well known report formats for tests frameworks which supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) Basically this is just xml document which can be easily parsed later by external tools. changes since v2: (in response to eguan@) - assert repot format on initialization - Tabify new files changes since v1: - Use generic '-R fmt' option for report generation, asked by eguan@ - localize variables, asked by eguan@ - do not add empty files to report asked by eguan@ - Add properties - Fix system-err/system-out node nesting EXAMPLE: #./check -R xunit -s ext4 generic/001 generic/010 #cat results/ext4/result.xml <?xml version="1.0" encoding="UTF-8"?> <testsuite name="xfstests" errors="0" skipped="1" tests="2" time="7" hostname="alice" timestamp="2017-02-21T15:15:06" > <properties> <property name="SECTION" value="ext4"/> <property name="FSTYP" value="ext4"/> <property name="MOUNT_OPTIONS" value="-o acl,user_xattr "/> <property name="HOST_OPTIONS" value="/devel/xfstests-dev.git/configs/alice.config"/> <property name="XFS_MKFS_OPTIONS" value="-bsize=4096"/> <property name="TIME_FACTOR" value="1"/> <property name="LOAD_FACTOR" value="1"/> <property name="TEST_DIR" value="/mnt/test"/> <property name="TEST_DEV" value="/dev/ram0"/> <property name="SCRATCH_DEV" value="/dev/ram1"/> <property name="SCRATCH_MNT" value="/mnt/scratch"/> <property name="OVERLAY_UPPER_DIR" value="upper"/> <property name="OVERLAY_LOWER_DIR" value="lower"/> <property name="OVERLAY_WORK_DIR" value="work"/> </properties> <testcase classname="xfstests.ext4" name="generic/001" time="5"> </testcase> <testcase classname="xfstests.ext4" name="generic/010" time="1"> <skipped message="src/dbtest not built" /> </testcase> </testsuite> Footnotes: [1] https://xunit.github.io/docs/format-xml-v2.html [2] http://help.catchsoftware.com/display/ET/JUnit+Format [3] https://jenkins.io [4] https://www.atlassian.com/software/bamboo [5] https://github.com/avocado-framework/avocado Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> --- check | 38 ++++++++++--- common/rc | 6 +- common/report | 174 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 9 deletions(-) create mode 100644 common/report diff --git a/check b/check index c4deb4d..a3ee591 100755 --- a/check +++ b/check @@ -39,7 +39,7 @@ export here=`pwd` xfile="" brief_test_summary=false err_msg="" - +do_report=false DUMP_OUTPUT=false # start the initialisation work now @@ -54,7 +54,7 @@ export DIFF_LENGTH=${DIFF_LENGTH:=10} # by default don't output timestamps timestamp=${TIMESTAMP:=false} -rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist +rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.* SRC_GROUPS="generic shared" export SRC_DIR="tests" @@ -75,6 +75,7 @@ check options -r randomize test order -d dump test output to stdout -b brief test summary + -R fmt[,fmt] generate report in formats specified. Supported format: [xunit] --large-fs optimise scratch device for large filesystems -s section run only specified section from config file -S section exclude the specified section from the config file @@ -295,7 +296,10 @@ while [ $# -gt 0 ]; do -T) timestamp=true ;; -d) DUMP_OUTPUT=true ;; -b) brief_test_summary=true;; - + -R) report_fmt=$2 ; shift ; + REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }" + do_report=true + ;; --large-fs) export LARGE_SCRATCH_DEV=yes ;; --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;; @@ -380,7 +384,12 @@ _wrapup() check="$RESULT_BASE/check" if $showme; then - : + if $needwrap; then + if $do_report; then + _make_section_report + fi + needwrap=false + fi elif $needwrap; then if [ -f $check.time -a -f $tmp.time ]; then cat $check.time $tmp.time \ @@ -431,6 +440,9 @@ _wrapup() echo "Passed all $n_try tests" >>$tmp.summary fi echo "" >>$tmp.summary + if $do_report; then + _make_section_report + fi needwrap=false fi @@ -519,6 +531,7 @@ for section in $HOST_OPTIONS_SECTIONS; do echo "SECTION -- $section" fi + sect_start=`_wallclock` if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then echo "RECREATING -- $FSTYP on $TEST_DEV" _test_unmount 2> /dev/null @@ -623,11 +636,12 @@ for section in $HOST_OPTIONS_SECTIONS; do group=`dirname $seq` if $OPTIONS_HAVE_SECTIONS; then export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"` - seqres="$RESULT_BASE/$section/$seqnum" + REPORT_DIR="$RESULT_BASE/$section" else export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"` - seqres="$RESULT_BASE/$seqnum" + REPORT_DIR="$RESULT_BASE" fi + seqres="$REPORT_DIR/$seqnum" mkdir -p $RESULT_DIR @@ -638,9 +652,12 @@ for section in $HOST_OPTIONS_SECTIONS; do start=0 stop=0 n_notrun=`expr $n_notrun + 1` + if $do_report; then + _make_testcase_report "list" + fi continue fi - + tc_status="pass" if [ ! -f $seq ]; then echo " - no such test?" else @@ -704,6 +721,7 @@ for section in $HOST_OPTIONS_SECTIONS; do cat $seqres.notrun notrun="$notrun $seqnum" n_notrun=`expr $n_notrun + 1` + tc_status="notrun" else if [ $sts -ne 0 ] then @@ -762,10 +780,14 @@ for section in $HOST_OPTIONS_SECTIONS; do bad="$bad $seqnum" n_bad=`expr $n_bad + 1` quick=false + tc_status="fail" + fi + if $do_report; then + _make_testcase_report "$tc_status" fi - seq="after_$seqnum" done + sect_stop=`_wallclock` interrupt=false _wrapup interrupt=true diff --git a/common/rc b/common/rc index 166400f..a49f14a 100644 --- a/common/rc +++ b/common/rc @@ -174,6 +174,11 @@ case "$FSTYP" in ;; esac +if [ ! -z "$REPORT_LIST" ]; then + . ./common/report + _assert_report_list +fi + _mount() { $MOUNT_PROG `_mount_ops_filter $*` @@ -3325,7 +3330,6 @@ _get_fs_sysfs_attr() } - init_rc ################################################################################ diff --git a/common/report b/common/report new file mode 100644 index 0000000..4b2d38f --- /dev/null +++ b/common/report @@ -0,0 +1,174 @@ +# +# Reports generator funcitons lives here +# + +# List of xfstests's enviroment variables to include reports +## TODO automate list population inside common/conf +REPORT_ENV_LIST="$REPORT_ENV_LIST SECTION" +REPORT_ENV_LIST="$REPORT_ENV_LIST FSTYP" +REPORT_ENV_LIST="$REPORT_ENV_LIST PLATFORM" +REPORT_ENV_LIST="$REPORT_ENV_LIST MKFS_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST MOUNT_OPTIONS" + +REPORT_ENV_LIST="$REPORT_ENV_LIST HOST_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST CHECK_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST XFS_MKFS_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST TIME_FACTOR" +REPORT_ENV_LIST="$REPORT_ENV_LIST LOAD_FACTOR" + +REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DIR" +REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DEV" +REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_DEV" +REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_MNT" + +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_UPPER_DIR" +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_LOWER_DIR" +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_WORK_DIR" + +# +# Xunit format report functions +_xunit_add_property() +{ + local name="$1" + local value="${!name}" + + if [ ! -z "$value" ]; then + echo -e "\t\t<property name=\"$name\" value=\"$value\"/>" >> $REPORT_DIR/result.xml + fi +} +_xunit_make_section_report() +{ + # xfstest:section ==> xunit:testsuite + local sect_name=$section + local sect_time=`expr $sect_stop - $sect_start` + local n_total=`expr $n_try + $n_notrun` + + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + fi + local report=$tmp.report.xunit.$sect_name.xml + #Header + echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $REPORT_DIR/result.xml + local dtime=`echo $date_time| tr " " 'T'` + local stats="errors=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\"" + local hw_info="hostname=\"$HOST\" timestamp=\"$dtime\" " + echo "<testsuite name=\"xfstests\" $stats $hw_info >" >> $REPORT_DIR/result.xml + #Properties + + echo -e "\t<properties>" >> $REPORT_DIR/result.xml + for p in $REPORT_ENV_LIST;do + _xunit_add_property "$p" + done + echo -e "\t</properties>" >> $REPORT_DIR/result.xml + cat $tmp.report.xunit.$sect_name.xml >> $REPORT_DIR/result.xml + echo "</testsuite>" >> $REPORT_DIR/result.xml + echo "Xunit report: $REPORT_DIR/result.xml" +} + +_xunit_make_testcase_report() +{ + local test_status="$1" + local test_time=`expr $stop - $start` + local strip="$SRC_DIR/" + local test_name=${seq#$strip} + local sect_name=$section + + #TODO: other places may also win if no-section mode will be named like 'default/global' + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + + fi + local report=$tmp.report.xunit.$sect_name.xml + + echo -e "\t<testcase classname=\"xfstests.$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report + case $test_status in + "pass") + ;; + "notrun") + if [ -f $seqres.notrun ]; then + local msg=`cat $seqres.notrun` + echo -e "\t\t<skipped message=\"$msg\" />" >> $report + else + echo -e "\t\t<skipped/>" >> $report + fi + ;; + "list") + echo -e "\t\t<skipped/>" >> $report + ;; + "fail") + if [ -z "$err_msg" ]; then + err_msg="Test $sequm failed, reason unknown" + fi + echo -e "\t\t<failure message=\"$err_msg\" type=\"TestFail\" />" >> $report + if [ -s $seqres.full ]; then + echo -e "\t\t<system-out>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.full | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t</system-out>" >> $report + fi + if [ -f $seqres.dmesg ]; then + echo -e "\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.dmesg | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t</system-err>" >> $report + elif [ -s $seqres.out.bad ]; then + echo -e "\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + $diff $seq.out $seqres.out.bad >>$report + printf ']]>\n' >>$report + echo -e "\t\t</system-err>" >> $report + fi + ;; + *) + echo -e "\t\t<failure message=\"Unknown ret_state=$ret_state\" type=\"TestFail\"/>" >> $report + ;; + esac + echo -e "\t</testcase>" >> $report +} + + +# +# Common report generator entry points +_make_section_report() +{ + for report in $REPORT_LIST; do + case "$report" in + "xunit") + _xunit_make_section_report "$test_status" + ;; + *) + _dump_err "format '$report' is not supported" + ;; + esac + done +} + +_make_testcase_report() +{ + test_status="$1" + for report in $REPORT_LIST; do + case "$report" in + "xunit") + _xunit_make_testcase_report "$test_status" + ;; + *) + _dump_err "report format '$report' is not supported" + ;; + esac + done +} + +_assert_report_list() { + test_status="$1" + for report in $REPORT_LIST; do + case "$report" in + "xunit") + ;; + *) + _fatal "report format '$report' is not supported" + ;; + esac + done +} -- 2.9.3 ^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: [PATCH 2/2] report: Add xunit format report generator 2017-03-03 8:26 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov @ 2017-03-07 12:05 ` Eryu Guan 2017-03-07 13:13 ` Amir Goldstein 0 siblings, 1 reply; 14+ messages in thread From: Eryu Guan @ 2017-03-07 12:05 UTC (permalink / raw) To: Dmitry Monakhov; +Cc: fstests On Fri, Mar 03, 2017 at 12:26:16PM +0400, Dmitry Monakhov wrote: > xunit[1]/junit[2] are well known report formats for tests frameworks which > supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) > Basically this is just xml document which can be easily parsed later by external tools. > > changes since v2: (in response to eguan@) > - assert repot format on initialization > - Tabify new files > > changes since v1: > - Use generic '-R fmt' option for report generation, asked by eguan@ > - localize variables, asked by eguan@ > - do not add empty files to report asked by eguan@ > - Add properties > - Fix system-err/system-out node nesting > > EXAMPLE: > #./check -R xunit -s ext4 generic/001 generic/010 > #cat results/ext4/result.xml > <?xml version="1.0" encoding="UTF-8"?> > <testsuite name="xfstests" errors="0" skipped="1" tests="2" time="7" hostname="alice" timestamp="2017-02-21T15:15:06" > > <properties> > <property name="SECTION" value="ext4"/> > <property name="FSTYP" value="ext4"/> > <property name="MOUNT_OPTIONS" value="-o acl,user_xattr "/> > <property name="HOST_OPTIONS" value="/devel/xfstests-dev.git/configs/alice.config"/> > <property name="XFS_MKFS_OPTIONS" value="-bsize=4096"/> > <property name="TIME_FACTOR" value="1"/> > <property name="LOAD_FACTOR" value="1"/> > <property name="TEST_DIR" value="/mnt/test"/> > <property name="TEST_DEV" value="/dev/ram0"/> > <property name="SCRATCH_DEV" value="/dev/ram1"/> > <property name="SCRATCH_MNT" value="/mnt/scratch"/> > <property name="OVERLAY_UPPER_DIR" value="upper"/> > <property name="OVERLAY_LOWER_DIR" value="lower"/> > <property name="OVERLAY_WORK_DIR" value="work"/> > </properties> > <testcase classname="xfstests.ext4" name="generic/001" time="5"> > </testcase> > <testcase classname="xfstests.ext4" name="generic/010" time="1"> > <skipped message="src/dbtest not built" /> > </testcase> > </testsuite> > > Footnotes: > [1] https://xunit.github.io/docs/format-xml-v2.html > [2] http://help.catchsoftware.com/display/ET/JUnit+Format > [3] https://jenkins.io > [4] https://www.atlassian.com/software/bamboo > [5] https://github.com/avocado-framework/avocado > > Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> I renamed OVERLAY_{UPPER,LOWER,WORK}_DIR to OVL_{UPPER,LOWER,WORK} in this patch, as they've been updated by recent overlayfs patch. I also fixed some minor indention issues and removed unused test_status in _assert_report_list() function. Thanks for the update! Eryu ^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 2/2] report: Add xunit format report generator 2017-03-07 12:05 ` Eryu Guan @ 2017-03-07 13:13 ` Amir Goldstein 2017-03-08 3:00 ` Eryu Guan 0 siblings, 1 reply; 14+ messages in thread From: Amir Goldstein @ 2017-03-07 13:13 UTC (permalink / raw) To: Eryu Guan; +Cc: Dmitry Monakhov, fstests On Tue, Mar 7, 2017 at 2:05 PM, Eryu Guan <eguan@redhat.com> wrote: > On Fri, Mar 03, 2017 at 12:26:16PM +0400, Dmitry Monakhov wrote: >> xunit[1]/junit[2] are well known report formats for tests frameworks which >> supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) >> Basically this is just xml document which can be easily parsed later by external tools. >> >> changes since v2: (in response to eguan@) >> - assert repot format on initialization >> - Tabify new files >> >> changes since v1: >> - Use generic '-R fmt' option for report generation, asked by eguan@ >> - localize variables, asked by eguan@ >> - do not add empty files to report asked by eguan@ >> - Add properties >> - Fix system-err/system-out node nesting >> >> EXAMPLE: >> #./check -R xunit -s ext4 generic/001 generic/010 >> #cat results/ext4/result.xml >> <?xml version="1.0" encoding="UTF-8"?> >> <testsuite name="xfstests" errors="0" skipped="1" tests="2" time="7" hostname="alice" timestamp="2017-02-21T15:15:06" > >> <properties> >> <property name="SECTION" value="ext4"/> >> <property name="FSTYP" value="ext4"/> >> <property name="MOUNT_OPTIONS" value="-o acl,user_xattr "/> >> <property name="HOST_OPTIONS" value="/devel/xfstests-dev.git/configs/alice.config"/> >> <property name="XFS_MKFS_OPTIONS" value="-bsize=4096"/> >> <property name="TIME_FACTOR" value="1"/> >> <property name="LOAD_FACTOR" value="1"/> >> <property name="TEST_DIR" value="/mnt/test"/> >> <property name="TEST_DEV" value="/dev/ram0"/> >> <property name="SCRATCH_DEV" value="/dev/ram1"/> >> <property name="SCRATCH_MNT" value="/mnt/scratch"/> >> <property name="OVERLAY_UPPER_DIR" value="upper"/> >> <property name="OVERLAY_LOWER_DIR" value="lower"/> >> <property name="OVERLAY_WORK_DIR" value="work"/> >> </properties> >> <testcase classname="xfstests.ext4" name="generic/001" time="5"> >> </testcase> >> <testcase classname="xfstests.ext4" name="generic/010" time="1"> >> <skipped message="src/dbtest not built" /> >> </testcase> >> </testsuite> >> >> Footnotes: >> [1] https://xunit.github.io/docs/format-xml-v2.html >> [2] http://help.catchsoftware.com/display/ET/JUnit+Format >> [3] https://jenkins.io >> [4] https://www.atlassian.com/software/bamboo >> [5] https://github.com/avocado-framework/avocado >> >> Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> > > I renamed OVERLAY_{UPPER,LOWER,WORK}_DIR to OVL_{UPPER,LOWER,WORK} in > this patch, as they've been updated by recent overlayfs patch. > I would prefer if those were not configuration options at all. As I wrote in commit 6f5ba68b: - Make those vars values non-configurable, because lower/upper/work directory names are an internal test detail which should not concern the user and because we wish to simplify and document the overlay tests setup > I also fixed some minor indention issues and removed unused test_status > in _assert_report_list() function. > > Thanks for the update! > > Eryu > -- > To unsubscribe from this list: send the line "unsubscribe fstests" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html ^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 2/2] report: Add xunit format report generator 2017-03-07 13:13 ` Amir Goldstein @ 2017-03-08 3:00 ` Eryu Guan 2017-03-08 12:16 ` Dmitry Monakhov 0 siblings, 1 reply; 14+ messages in thread From: Eryu Guan @ 2017-03-08 3:00 UTC (permalink / raw) To: Amir Goldstein; +Cc: Dmitry Monakhov, fstests On Tue, Mar 07, 2017 at 03:13:01PM +0200, Amir Goldstein wrote: > On Tue, Mar 7, 2017 at 2:05 PM, Eryu Guan <eguan@redhat.com> wrote: > > On Fri, Mar 03, 2017 at 12:26:16PM +0400, Dmitry Monakhov wrote: > >> xunit[1]/junit[2] are well known report formats for tests frameworks which > >> supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) > >> Basically this is just xml document which can be easily parsed later by external tools. > >> > >> changes since v2: (in response to eguan@) > >> - assert repot format on initialization > >> - Tabify new files > >> > >> changes since v1: > >> - Use generic '-R fmt' option for report generation, asked by eguan@ > >> - localize variables, asked by eguan@ > >> - do not add empty files to report asked by eguan@ > >> - Add properties > >> - Fix system-err/system-out node nesting > >> > >> EXAMPLE: > >> #./check -R xunit -s ext4 generic/001 generic/010 > >> #cat results/ext4/result.xml > >> <?xml version="1.0" encoding="UTF-8"?> > >> <testsuite name="xfstests" errors="0" skipped="1" tests="2" time="7" hostname="alice" timestamp="2017-02-21T15:15:06" > > >> <properties> > >> <property name="SECTION" value="ext4"/> > >> <property name="FSTYP" value="ext4"/> > >> <property name="MOUNT_OPTIONS" value="-o acl,user_xattr "/> > >> <property name="HOST_OPTIONS" value="/devel/xfstests-dev.git/configs/alice.config"/> > >> <property name="XFS_MKFS_OPTIONS" value="-bsize=4096"/> > >> <property name="TIME_FACTOR" value="1"/> > >> <property name="LOAD_FACTOR" value="1"/> > >> <property name="TEST_DIR" value="/mnt/test"/> > >> <property name="TEST_DEV" value="/dev/ram0"/> > >> <property name="SCRATCH_DEV" value="/dev/ram1"/> > >> <property name="SCRATCH_MNT" value="/mnt/scratch"/> > >> <property name="OVERLAY_UPPER_DIR" value="upper"/> > >> <property name="OVERLAY_LOWER_DIR" value="lower"/> > >> <property name="OVERLAY_WORK_DIR" value="work"/> > >> </properties> > >> <testcase classname="xfstests.ext4" name="generic/001" time="5"> > >> </testcase> > >> <testcase classname="xfstests.ext4" name="generic/010" time="1"> > >> <skipped message="src/dbtest not built" /> > >> </testcase> > >> </testsuite> > >> > >> Footnotes: > >> [1] https://xunit.github.io/docs/format-xml-v2.html > >> [2] http://help.catchsoftware.com/display/ET/JUnit+Format > >> [3] https://jenkins.io > >> [4] https://www.atlassian.com/software/bamboo > >> [5] https://github.com/avocado-framework/avocado > >> > >> Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> > > > > I renamed OVERLAY_{UPPER,LOWER,WORK}_DIR to OVL_{UPPER,LOWER,WORK} in > > this patch, as they've been updated by recent overlayfs patch. > > > > I would prefer if those were not configuration options at all. > > As I wrote in commit 6f5ba68b: > > - Make those vars values non-configurable, because lower/upper/work > directory names are an internal test detail which should > not concern the user and because we wish to simplify > and document the overlay tests setup Thanks for the reminder! Dmitry, do you mind I removing all these overlay variables from REPORT_ENV_LIST? Thanks, Eryu ^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 2/2] report: Add xunit format report generator 2017-03-08 3:00 ` Eryu Guan @ 2017-03-08 12:16 ` Dmitry Monakhov 0 siblings, 0 replies; 14+ messages in thread From: Dmitry Monakhov @ 2017-03-08 12:16 UTC (permalink / raw) To: Eryu Guan, Amir Goldstein; +Cc: fstests Eryu Guan <eguan@redhat.com> writes: > On Tue, Mar 07, 2017 at 03:13:01PM +0200, Amir Goldstein wrote: >> On Tue, Mar 7, 2017 at 2:05 PM, Eryu Guan <eguan@redhat.com> wrote: >> > On Fri, Mar 03, 2017 at 12:26:16PM +0400, Dmitry Monakhov wrote: >> >> xunit[1]/junit[2] are well known report formats for tests frameworks which >> >> supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) >> >> Basically this is just xml document which can be easily parsed later by external tools. >> >> >> >> changes since v2: (in response to eguan@) >> >> - assert repot format on initialization >> >> - Tabify new files >> >> >> >> changes since v1: >> >> - Use generic '-R fmt' option for report generation, asked by eguan@ >> >> - localize variables, asked by eguan@ >> >> - do not add empty files to report asked by eguan@ >> >> - Add properties >> >> - Fix system-err/system-out node nesting >> >> >> >> EXAMPLE: >> >> #./check -R xunit -s ext4 generic/001 generic/010 >> >> #cat results/ext4/result.xml >> >> <?xml version="1.0" encoding="UTF-8"?> >> >> <testsuite name="xfstests" errors="0" skipped="1" tests="2" time="7" hostname="alice" timestamp="2017-02-21T15:15:06" > >> >> <properties> >> >> <property name="SECTION" value="ext4"/> >> >> <property name="FSTYP" value="ext4"/> >> >> <property name="MOUNT_OPTIONS" value="-o acl,user_xattr "/> >> >> <property name="HOST_OPTIONS" value="/devel/xfstests-dev.git/configs/alice.config"/> >> >> <property name="XFS_MKFS_OPTIONS" value="-bsize=4096"/> >> >> <property name="TIME_FACTOR" value="1"/> >> >> <property name="LOAD_FACTOR" value="1"/> >> >> <property name="TEST_DIR" value="/mnt/test"/> >> >> <property name="TEST_DEV" value="/dev/ram0"/> >> >> <property name="SCRATCH_DEV" value="/dev/ram1"/> >> >> <property name="SCRATCH_MNT" value="/mnt/scratch"/> >> >> <property name="OVERLAY_UPPER_DIR" value="upper"/> >> >> <property name="OVERLAY_LOWER_DIR" value="lower"/> >> >> <property name="OVERLAY_WORK_DIR" value="work"/> >> >> </properties> >> >> <testcase classname="xfstests.ext4" name="generic/001" time="5"> >> >> </testcase> >> >> <testcase classname="xfstests.ext4" name="generic/010" time="1"> >> >> <skipped message="src/dbtest not built" /> >> >> </testcase> >> >> </testsuite> >> >> >> >> Footnotes: >> >> [1] https://xunit.github.io/docs/format-xml-v2.html >> >> [2] http://help.catchsoftware.com/display/ET/JUnit+Format >> >> [3] https://jenkins.io >> >> [4] https://www.atlassian.com/software/bamboo >> >> [5] https://github.com/avocado-framework/avocado >> >> >> >> Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> >> > >> > I renamed OVERLAY_{UPPER,LOWER,WORK}_DIR to OVL_{UPPER,LOWER,WORK} in >> > this patch, as they've been updated by recent overlayfs patch. >> > >> >> I would prefer if those were not configuration options at all. >> >> As I wrote in commit 6f5ba68b: >> >> - Make those vars values non-configurable, because lower/upper/work >> directory names are an internal test detail which should >> not concern the user and because we wish to simplify >> and document the overlay tests setup > > Thanks for the reminder! > > Dmitry, do you mind I removing all these overlay variables from > REPORT_ENV_LIST? Off course. Please. This list should contains only externally configurable options necessery for later reproduction. > > Thanks, > Eryu ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 0/2] Add report generators support v2 @ 2017-02-21 12:44 Dmitry Monakhov 2017-02-21 12:44 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 0 siblings, 1 reply; 14+ messages in thread From: Dmitry Monakhov @ 2017-02-21 12:44 UTC (permalink / raw) To: fstests; +Cc: Dmitry Monakhov xfstests has no stable logging API which is suitable for machine parsing. This makes it difficult to integrate with external atomation tools (Jenkins.io, k8s jobs) In fact thare are alot of well known parsable report formats TAP [1], Junit [2], yaml. Let's add basic infrastructure so people may add report formats they do care about. Usecase example with Jenkins: - Split huge xfstest job to smaller peaces. - Submit it as separate jobs and run it in parallel on different VMs - Fetch and merge results to single job report. This approach allow me to speedup xfstests x50 times, now full run of '-g auto' takes only 3minutes TOC - check: prepare test report generator infrastructure - report:Add xunit format report generator changes from v1: - fixes in response to comments from eguan@ - Add env variables to report . Footnotes: [1] https://testanything.org [2] http://help.catchsoftware.com/display/ET/JUnit+Format ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/2] report: Add xunit format report generator 2017-02-21 12:44 [PATCH 0/2] Add report generators support v2 Dmitry Monakhov @ 2017-02-21 12:44 ` Dmitry Monakhov 2017-03-02 8:13 ` Eryu Guan 0 siblings, 1 reply; 14+ messages in thread From: Dmitry Monakhov @ 2017-02-21 12:44 UTC (permalink / raw) To: fstests; +Cc: Dmitry Monakhov xunit[1]/junit[2] are well known report formats for tests frameworks which supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) Basically this is just xml document which can be easily parsed later by external tools. changes since v1: - Use generic '-R fmt' option for report generation, asked by eguan@ - localize variables, asked by eguan@ - do not add empty files to report asked by eguan@ - Add properties - Fix system-err/system-out node nesting EXAMPLE: #./check -R xunit -s ext4 generic/001 generic/010 #cat results/ext4/result.xml <?xml version="1.0" encoding="UTF-8"?> <testsuite name="xfstests" errors="0" skipped="1" tests="2" time="7" hostname="alice" timestamp="2017-02-21T15:15:06" > <properties> <property name="SECTION" value="ext4"/> <property name="FSTYP" value="ext4"/> <property name="MOUNT_OPTIONS" value="-o acl,user_xattr "/> <property name="HOST_OPTIONS" value="/devel/xfstests-dev.git/configs/alice.config"/> <property name="XFS_MKFS_OPTIONS" value="-bsize=4096"/> <property name="TIME_FACTOR" value="1"/> <property name="LOAD_FACTOR" value="1"/> <property name="TEST_DIR" value="/mnt/test"/> <property name="TEST_DEV" value="/dev/ram0"/> <property name="SCRATCH_DEV" value="/dev/ram1"/> <property name="SCRATCH_MNT" value="/mnt/scratch"/> <property name="OVERLAY_UPPER_DIR" value="upper"/> <property name="OVERLAY_LOWER_DIR" value="lower"/> <property name="OVERLAY_WORK_DIR" value="work"/> </properties> <testcase classname="xfstests.ext4" name="generic/001" time="5"> </testcase> <testcase classname="xfstests.ext4" name="generic/010" time="1"> <skipped message="src/dbtest not built" /> </testcase> </testsuite> Footnotes: [1] https://xunit.github.io/docs/format-xml-v2.html [2] http://help.catchsoftware.com/display/ET/JUnit+Format [3] https://jenkins.io [4] https://www.atlassian.com/software/bamboo [5] https://github.com/avocado-framework/avocado Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> --- check | 38 +++++++++++--- common/rc | 5 +- common/report | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 common/report diff --git a/check b/check index 8ee2f6f..bd2d763 100755 --- a/check +++ b/check @@ -39,7 +39,7 @@ export here=`pwd` xfile="" brief_test_summary=false err_msg="" - +do_report=false DUMP_OUTPUT=false # start the initialisation work now @@ -54,7 +54,7 @@ export DIFF_LENGTH=${DIFF_LENGTH:=10} # by default don't output timestamps timestamp=${TIMESTAMP:=false} -rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist +rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.* SRC_GROUPS="generic shared" export SRC_DIR="tests" @@ -75,6 +75,7 @@ check options -r randomize test order -d dump test output to stdout -b brief test summary + -R fmt[,fmt] generate report in formats specified. Supported format: [xunit] --large-fs optimise scratch device for large filesystems -s section run only specified section from config file -S section exclude the specified section from the config file @@ -295,7 +296,10 @@ while [ $# -gt 0 ]; do -T) timestamp=true ;; -d) DUMP_OUTPUT=true ;; -b) brief_test_summary=true;; - + -R) report_fmt=$2 ; shift ; + REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }" + do_report=true + ;; --large-fs) export LARGE_SCRATCH_DEV=yes ;; --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;; @@ -380,7 +384,12 @@ _wrapup() check="$RESULT_BASE/check" if $showme; then - : + if $needwrap; then + if $do_report; then + _make_section_report + fi + needwrap=false + fi elif $needwrap; then if [ -f $check.time -a -f $tmp.time ]; then cat $check.time $tmp.time \ @@ -431,6 +440,9 @@ _wrapup() echo "Passed all $n_try tests" >>$tmp.summary fi echo "" >>$tmp.summary + if $do_report; then + _make_section_report + fi needwrap=false fi @@ -519,6 +531,7 @@ for section in $HOST_OPTIONS_SECTIONS; do echo "SECTION -- $section" fi + sect_start=`_wallclock` if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then echo "RECREATING -- $FSTYP on $TEST_DEV" _test_unmount 2> /dev/null @@ -623,11 +636,12 @@ for section in $HOST_OPTIONS_SECTIONS; do group=`dirname $seq` if $OPTIONS_HAVE_SECTIONS; then export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"` - seqres="$RESULT_BASE/$section/$seqnum" + REPORT_DIR="$RESULT_BASE/$section" else export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"` - seqres="$RESULT_BASE/$seqnum" + REPORT_DIR="$RESULT_BASE" fi + seqres="$REPORT_DIR/$seqnum" mkdir -p $RESULT_DIR @@ -638,9 +652,12 @@ for section in $HOST_OPTIONS_SECTIONS; do start=0 stop=0 n_notrun=`expr $n_notrun + 1` + if $do_report; then + _make_testcase_report "list" + fi continue fi - + tc_status="pass" if [ ! -f $seq ]; then echo " - no such test?" else @@ -704,6 +721,7 @@ for section in $HOST_OPTIONS_SECTIONS; do cat $seqres.notrun notrun="$notrun $seqnum" n_notrun=`expr $n_notrun + 1` + tc_status="notrun" else if [ $sts -ne 0 ] then @@ -762,10 +780,14 @@ for section in $HOST_OPTIONS_SECTIONS; do bad="$bad $seqnum" n_bad=`expr $n_bad + 1` quick=false + tc_status="fail" + fi + if $do_report; then + _make_testcase_report "$tc_status" fi - seq="after_$seqnum" done + sect_stop=`_wallclock` _wrapup echo diff --git a/common/rc b/common/rc index ec9b2a0..6ac8ea3 100644 --- a/common/rc +++ b/common/rc @@ -155,6 +155,10 @@ case "$FSTYP" in ;; esac +if [ ! -z "$REPORT_LIST" ]; then + . ./common/report +fi + _mount() { $MOUNT_PROG `_mount_ops_filter $*` @@ -3297,7 +3301,6 @@ _get_fs_sysfs_attr() } - init_rc ################################################################################ diff --git a/common/report b/common/report new file mode 100644 index 0000000..1a1eadd --- /dev/null +++ b/common/report @@ -0,0 +1,161 @@ +# +# Reports generator funcitons lives here +# + +# List of xfstests's enviroment variables to include reports +## TODO automate list population inside common/conf +REPORT_ENV_LIST="$REPORT_ENV_LIST SECTION" +REPORT_ENV_LIST="$REPORT_ENV_LIST FSTYP" +REPORT_ENV_LIST="$REPORT_ENV_LIST PLATFORM" +REPORT_ENV_LIST="$REPORT_ENV_LIST MKFS_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST MOUNT_OPTIONS" + +REPORT_ENV_LIST="$REPORT_ENV_LIST HOST_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST CHECK_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST XFS_MKFS_OPTIONS" +REPORT_ENV_LIST="$REPORT_ENV_LIST TIME_FACTOR" +REPORT_ENV_LIST="$REPORT_ENV_LIST LOAD_FACTOR" + +REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DIR" +REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DEV" +REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_DEV" +REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_MNT" + +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_UPPER_DIR" +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_LOWER_DIR" +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_WORK_DIR" + +# +# Xunit format report functions +_xunit_add_property() +{ + local name="$1" + local value="${!name}" + + if [ ! -z "$value" ]; then + echo -e "\t\t<property name=\"$name\" value=\"$value\"/>" >> $REPORT_DIR/result.xml + fi +} +_xunit_make_section_report() +{ + # xfstest:section ==> xunit:testsuite + local sect_name=$section + local sect_time=`expr $sect_stop - $sect_start` + local n_total=`expr $n_try + $n_notrun` + + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + fi + local report=$tmp.report.xunit.$sect_name.xml + #Header + echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $REPORT_DIR/result.xml + local dtime=`echo $date_time| tr " " 'T'` + local stats="errors=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\"" + local hw_info="hostname=\"$HOST\" timestamp=\"$dtime\" " + echo "<testsuite name=\"xfstests\" $stats $hw_info >" >> $REPORT_DIR/result.xml + #Properties + + echo -e "\t<properties>" >> $REPORT_DIR/result.xml + for p in $REPORT_ENV_LIST;do + _xunit_add_property "$p" + done + echo -e "\t</properties>" >> $REPORT_DIR/result.xml + cat $tmp.report.xunit.$sect_name.xml >> $REPORT_DIR/result.xml + echo "</testsuite>" >> $REPORT_DIR/result.xml + echo "Xunit report: $REPORT_DIR/result.xml" +} + +_xunit_make_testcase_report() +{ + local test_status="$1" + local test_time=`expr $stop - $start` + local strip="$SRC_DIR/" + local test_name=${seq#$strip} + local sect_name=$section + + #TODO: other places may also win if no-section mode will be named like 'default/global' + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + + fi + local report=$tmp.report.xunit.$sect_name.xml + + echo -e "\t<testcase classname=\"xfstests.$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report + case $test_status in + "pass") + ;; + "notrun") + if [ -f $seqres.notrun ]; then + local msg=`cat $seqres.notrun` + echo -e "\t\t<skipped message=\"$msg\" />" >> $report + else + echo -e "\t\t<skipped/>" >> $report + fi + ;; + "list") + echo -e "\t\t<skipped/>" >> $report + ;; + "fail") + if [ -z "$err_msg" ]; then + err_msg="Test $sequm failed, reason unknown" + fi + echo -e "\t\t<failure message=\"$err_msg\" type=\"TestFail\" />" >> $report + if [ -s $seqres.full ]; then + echo -e "\t\t<system-out>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.full | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t</system-out>" >> $report + fi + if [ -f $seqres.dmesg ]; then + echo -e "\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.dmesg | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t</system-err>" >> $report + elif [ -s $seqres.out.bad ]; then + echo -e "\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + $diff $seq.out $seqres.out.bad >>$report + printf ']]>\n' >>$report + echo -e "\t\t</system-err>" >> $report + fi + ;; + *) + echo -e "\t\t<failure message=\"Unknown ret_state=$ret_state\" type=\"TestFail\"/>" >> $report + ;; + esac + echo -e "\t</testcase>" >> $report +} + + +# +# Common report generator entry points +_make_section_report() +{ + for report in $REPORT_LIST; do + case "$report" in + "xunit") + _xunit_make_section_report "$test_status" + ;; + *) + _fatal "report format '$report' is not supported" + ;; + esac + done +} + +_make_testcase_report() +{ + test_status="$1" + for report in $REPORT_LIST; do + case "$report" in + "xunit") + _xunit_make_testcase_report "$test_status" + ;; + *) + _fatal "report format '$report' is not supported" + ;; + esac + done +} -- 2.9.3 ^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: [PATCH 2/2] report: Add xunit format report generator 2017-02-21 12:44 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov @ 2017-03-02 8:13 ` Eryu Guan 0 siblings, 0 replies; 14+ messages in thread From: Eryu Guan @ 2017-03-02 8:13 UTC (permalink / raw) To: Dmitry Monakhov; +Cc: fstests On Tue, Feb 21, 2017 at 04:44:05PM +0400, Dmitry Monakhov wrote: > xunit[1]/junit[2] are well known report formats for tests frameworks which > supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) > Basically this is just xml document which can be easily parsed later by external tools. > > changes since v1: > - Use generic '-R fmt' option for report generation, asked by eguan@ > - localize variables, asked by eguan@ > - do not add empty files to report asked by eguan@ > - Add properties > - Fix system-err/system-out node nesting > > EXAMPLE: > #./check -R xunit -s ext4 generic/001 generic/010 > #cat results/ext4/result.xml > <?xml version="1.0" encoding="UTF-8"?> > <testsuite name="xfstests" errors="0" skipped="1" tests="2" time="7" hostname="alice" timestamp="2017-02-21T15:15:06" > > <properties> > <property name="SECTION" value="ext4"/> > <property name="FSTYP" value="ext4"/> > <property name="MOUNT_OPTIONS" value="-o acl,user_xattr "/> > <property name="HOST_OPTIONS" value="/devel/xfstests-dev.git/configs/alice.config"/> > <property name="XFS_MKFS_OPTIONS" value="-bsize=4096"/> > <property name="TIME_FACTOR" value="1"/> > <property name="LOAD_FACTOR" value="1"/> > <property name="TEST_DIR" value="/mnt/test"/> > <property name="TEST_DEV" value="/dev/ram0"/> > <property name="SCRATCH_DEV" value="/dev/ram1"/> > <property name="SCRATCH_MNT" value="/mnt/scratch"/> > <property name="OVERLAY_UPPER_DIR" value="upper"/> > <property name="OVERLAY_LOWER_DIR" value="lower"/> > <property name="OVERLAY_WORK_DIR" value="work"/> > </properties> > <testcase classname="xfstests.ext4" name="generic/001" time="5"> > </testcase> > <testcase classname="xfstests.ext4" name="generic/010" time="1"> > <skipped message="src/dbtest not built" /> > </testcase> > </testsuite> > > Footnotes: > [1] https://xunit.github.io/docs/format-xml-v2.html > [2] http://help.catchsoftware.com/display/ET/JUnit+Format > [3] https://jenkins.io > [4] https://www.atlassian.com/software/bamboo > [5] https://github.com/avocado-framework/avocado > > Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> > --- > check | 38 +++++++++++--- > common/rc | 5 +- > common/report | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ > 3 files changed, 195 insertions(+), 9 deletions(-) > create mode 100644 common/report > > diff --git a/check b/check > index 8ee2f6f..bd2d763 100755 > --- a/check > +++ b/check > @@ -39,7 +39,7 @@ export here=`pwd` > xfile="" > brief_test_summary=false > err_msg="" > - > +do_report=false > DUMP_OUTPUT=false > > # start the initialisation work now > @@ -54,7 +54,7 @@ export DIFF_LENGTH=${DIFF_LENGTH:=10} > # by default don't output timestamps > timestamp=${TIMESTAMP:=false} > > -rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist > +rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.* > > SRC_GROUPS="generic shared" > export SRC_DIR="tests" > @@ -75,6 +75,7 @@ check options > -r randomize test order > -d dump test output to stdout > -b brief test summary > + -R fmt[,fmt] generate report in formats specified. Supported format: [xunit] > --large-fs optimise scratch device for large filesystems > -s section run only specified section from config file > -S section exclude the specified section from the config file > @@ -295,7 +296,10 @@ while [ $# -gt 0 ]; do > -T) timestamp=true ;; > -d) DUMP_OUTPUT=true ;; > -b) brief_test_summary=true;; > - > + -R) report_fmt=$2 ; shift ; > + REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }" > + do_report=true > + ;; > --large-fs) export LARGE_SCRATCH_DEV=yes ;; > --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;; I think we should check, somewhere before running tests, if the report format is supported, not after running the test. Right now it fails like below if I use "-R <random_string>" [root@dhcp-66-86-11 xfstests]# ./check -s xfs_4k_crc -R teststr generic/001 generic/002 SECTION -- xfs_4k_crc RECREATING -- xfs on /dev/sda5 FSTYP -- xfs (non-debug) PLATFORM -- Linux/x86_64 dhcp-66-86-11 4.10.0 MKFS_OPTIONS -- -f -f -b size=4k -m crc=1 /dev/sda6 MOUNT_OPTIONS -- -o context=system_u:object_r:nfs_t:s0 /dev/sda6 /mnt/testarea/scratch generic/001 7s ... 7s report format 'teststr' is not supported Ran: generic/001 Interrupted! Passed all 1 tests report format 'teststr' is not supported > > @@ -380,7 +384,12 @@ _wrapup() > check="$RESULT_BASE/check" > > if $showme; then > - : > + if $needwrap; then > + if $do_report; then > + _make_section_report > + fi > + needwrap=false > + fi > elif $needwrap; then > if [ -f $check.time -a -f $tmp.time ]; then > cat $check.time $tmp.time \ > @@ -431,6 +440,9 @@ _wrapup() > echo "Passed all $n_try tests" >>$tmp.summary > fi > echo "" >>$tmp.summary > + if $do_report; then > + _make_section_report > + fi > needwrap=false > fi > > @@ -519,6 +531,7 @@ for section in $HOST_OPTIONS_SECTIONS; do > echo "SECTION -- $section" > fi > > + sect_start=`_wallclock` > if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then > echo "RECREATING -- $FSTYP on $TEST_DEV" > _test_unmount 2> /dev/null > @@ -623,11 +636,12 @@ for section in $HOST_OPTIONS_SECTIONS; do > group=`dirname $seq` > if $OPTIONS_HAVE_SECTIONS; then > export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"` > - seqres="$RESULT_BASE/$section/$seqnum" > + REPORT_DIR="$RESULT_BASE/$section" > else > export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"` > - seqres="$RESULT_BASE/$seqnum" > + REPORT_DIR="$RESULT_BASE" > fi > + seqres="$REPORT_DIR/$seqnum" > > mkdir -p $RESULT_DIR > > @@ -638,9 +652,12 @@ for section in $HOST_OPTIONS_SECTIONS; do > start=0 > stop=0 > n_notrun=`expr $n_notrun + 1` > + if $do_report; then > + _make_testcase_report "list" > + fi > continue > fi > - > + tc_status="pass" > if [ ! -f $seq ]; then > echo " - no such test?" > else > @@ -704,6 +721,7 @@ for section in $HOST_OPTIONS_SECTIONS; do > cat $seqres.notrun > notrun="$notrun $seqnum" > n_notrun=`expr $n_notrun + 1` > + tc_status="notrun" > else > if [ $sts -ne 0 ] > then > @@ -762,10 +780,14 @@ for section in $HOST_OPTIONS_SECTIONS; do > bad="$bad $seqnum" > n_bad=`expr $n_bad + 1` > quick=false > + tc_status="fail" > + fi > + if $do_report; then > + _make_testcase_report "$tc_status" > fi > - > seq="after_$seqnum" > done > + sect_stop=`_wallclock` > _wrapup > echo > > diff --git a/common/rc b/common/rc > index ec9b2a0..6ac8ea3 100644 > --- a/common/rc > +++ b/common/rc > @@ -155,6 +155,10 @@ case "$FSTYP" in > ;; > esac > > +if [ ! -z "$REPORT_LIST" ]; then > + . ./common/report > +fi > + > _mount() > { > $MOUNT_PROG `_mount_ops_filter $*` > @@ -3297,7 +3301,6 @@ _get_fs_sysfs_attr() > } > > > - > init_rc > > ################################################################################ > diff --git a/common/report b/common/report > new file mode 100644 > index 0000000..1a1eadd > --- /dev/null > +++ b/common/report > @@ -0,0 +1,161 @@ > +# > +# Reports generator funcitons lives here > +# > + > +# List of xfstests's enviroment variables to include reports > +## TODO automate list population inside common/conf > +REPORT_ENV_LIST="$REPORT_ENV_LIST SECTION" > +REPORT_ENV_LIST="$REPORT_ENV_LIST FSTYP" > +REPORT_ENV_LIST="$REPORT_ENV_LIST PLATFORM" > +REPORT_ENV_LIST="$REPORT_ENV_LIST MKFS_OPTIONS" > +REPORT_ENV_LIST="$REPORT_ENV_LIST MOUNT_OPTIONS" > + > +REPORT_ENV_LIST="$REPORT_ENV_LIST HOST_OPTIONS" > +REPORT_ENV_LIST="$REPORT_ENV_LIST CHECK_OPTIONS" > +REPORT_ENV_LIST="$REPORT_ENV_LIST XFS_MKFS_OPTIONS" > +REPORT_ENV_LIST="$REPORT_ENV_LIST TIME_FACTOR" > +REPORT_ENV_LIST="$REPORT_ENV_LIST LOAD_FACTOR" > + > +REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DIR" > +REPORT_ENV_LIST="$REPORT_ENV_LIST TEST_DEV" > +REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_DEV" > +REPORT_ENV_LIST="$REPORT_ENV_LIST SCRATCH_MNT" > + > +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_UPPER_DIR" > +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_LOWER_DIR" > +REPORT_ENV_LIST="$REPORT_ENV_LIST OVERLAY_WORK_DIR" > + > +# > +# Xunit format report functions > +_xunit_add_property() > +{ > + local name="$1" > + local value="${!name}" > + > + if [ ! -z "$value" ]; then > + echo -e "\t\t<property name=\"$name\" value=\"$value\"/>" >> $REPORT_DIR/result.xml > + fi > +} Please use one tab for indention for new code, common/report is a new file, we don't have to worry about legacy code :) > +_xunit_make_section_report() > +{ > + # xfstest:section ==> xunit:testsuite > + local sect_name=$section > + local sect_time=`expr $sect_stop - $sect_start` > + local n_total=`expr $n_try + $n_notrun` > + > + if [ $sect_name == '-no-sections-' ]; then > + sect_name='global' > + fi > + local report=$tmp.report.xunit.$sect_name.xml > + #Header > + echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $REPORT_DIR/result.xml > + local dtime=`echo $date_time| tr " " 'T'` > + local stats="errors=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\"" > + local hw_info="hostname=\"$HOST\" timestamp=\"$dtime\" " > + echo "<testsuite name=\"xfstests\" $stats $hw_info >" >> $REPORT_DIR/result.xml > + #Properties > + > + echo -e "\t<properties>" >> $REPORT_DIR/result.xml > + for p in $REPORT_ENV_LIST;do > + _xunit_add_property "$p" > + done > + echo -e "\t</properties>" >> $REPORT_DIR/result.xml > + cat $tmp.report.xunit.$sect_name.xml >> $REPORT_DIR/result.xml > + echo "</testsuite>" >> $REPORT_DIR/result.xml > + echo "Xunit report: $REPORT_DIR/result.xml" > +} > + > +_xunit_make_testcase_report() > +{ > + local test_status="$1" > + local test_time=`expr $stop - $start` > + local strip="$SRC_DIR/" > + local test_name=${seq#$strip} > + local sect_name=$section > + > + #TODO: other places may also win if no-section mode will be named like 'default/global' > + if [ $sect_name == '-no-sections-' ]; then > + sect_name='global' > + > + fi > + local report=$tmp.report.xunit.$sect_name.xml > + > + echo -e "\t<testcase classname=\"xfstests.$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report > + case $test_status in > + "pass") > + ;; > + "notrun") > + if [ -f $seqres.notrun ]; then > + local msg=`cat $seqres.notrun` > + echo -e "\t\t<skipped message=\"$msg\" />" >> $report > + else > + echo -e "\t\t<skipped/>" >> $report > + fi > + ;; > + "list") > + echo -e "\t\t<skipped/>" >> $report > + ;; > + "fail") > + if [ -z "$err_msg" ]; then > + err_msg="Test $sequm failed, reason unknown" > + fi > + echo -e "\t\t<failure message=\"$err_msg\" type=\"TestFail\" />" >> $report > + if [ -s $seqres.full ]; then > + echo -e "\t\t<system-out>" >> $report > + printf '<![CDATA[\n' >>$report > + cat $seqres.full | tr -dc '[:print:][:space:]' >>$report > + printf ']]>\n' >>$report > + echo -e "\t\t</system-out>" >> $report > + fi > + if [ -f $seqres.dmesg ]; then > + echo -e "\t\t<system-err>" >> $report > + printf '<![CDATA[\n' >>$report > + cat $seqres.dmesg | tr -dc '[:print:][:space:]' >>$report > + printf ']]>\n' >>$report > + echo -e "\t\t</system-err>" >> $report > + elif [ -s $seqres.out.bad ]; then > + echo -e "\t\t<system-err>" >> $report > + printf '<![CDATA[\n' >>$report > + $diff $seq.out $seqres.out.bad >>$report > + printf ']]>\n' >>$report > + echo -e "\t\t</system-err>" >> $report > + fi > + ;; > + *) > + echo -e "\t\t<failure message=\"Unknown ret_state=$ret_state\" type=\"TestFail\"/>" >> $report > + ;; > + esac > + echo -e "\t</testcase>" >> $report > +} > + > + > +# > +# Common report generator entry points > +_make_section_report() > +{ > + for report in $REPORT_LIST; do > + case "$report" in > + "xunit") > + _xunit_make_section_report "$test_status" > + ;; > + *) > + _fatal "report format '$report' is not supported" > + ;; If the supported report format has been checked before running tests, I think it's fine to do nothing here for unsupported format, or just give a warning? Thanks, Eryu > + esac > + done > +} > + > +_make_testcase_report() > +{ > + test_status="$1" > + for report in $REPORT_LIST; do > + case "$report" in > + "xunit") > + _xunit_make_testcase_report "$test_status" > + ;; > + *) > + _fatal "report format '$report' is not supported" > + ;; > + esac > + done > +} > -- > 2.9.3 > > -- > To unsubscribe from this list: send the line "unsubscribe fstests" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 0/2] Add report generators support @ 2017-01-31 17:16 Dmitry Monakhov 2017-01-31 17:16 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 0 siblings, 1 reply; 14+ messages in thread From: Dmitry Monakhov @ 2017-01-31 17:16 UTC (permalink / raw) To: fstests; +Cc: Dmitry Monakhov xfstests has no stable logging API which is suitable for machine parsing. This makes it difficult to integrate with external atomation tools (Jenkins.io, k8s jobs) In fact thare are alot of well known parsable report formats TAP [1], Junit [2], yaml. Let's add basic infrastructure so people may add report formats they do care about. Usecase example with Jenkins: - Split huge xfstest job to smaller peaces. - Submit it as separate jobs and run it in parallel on different VMs - Fetch and merge results to single job report. This approach allow me to speedup xfstests x50 times, now full run of '-g auto' takes only 3minutes TOC - check: prepare test report generator infrastructure - report:Add xunit format report generator Footnotes: [1] https://testanything.org [2] http://help.catchsoftware.com/display/ET/JUnit+Format ^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/2] report: Add xunit format report generator 2017-01-31 17:16 [PATCH 0/2] Add report generators support Dmitry Monakhov @ 2017-01-31 17:16 ` Dmitry Monakhov 0 siblings, 0 replies; 14+ messages in thread From: Dmitry Monakhov @ 2017-01-31 17:16 UTC (permalink / raw) To: fstests; +Cc: Dmitry Monakhov xunit[1]/junit[2] are well known report formats for tests frameworks which supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5]) Basically this is just xml document which can be easily parsed later by external tools. EXAMPLE: #./check --xunit -s ext4 generic/001 generic/010 generic/013 #cat results/ext4/result.xml <?xml version="1.0" encoding="UTF-8"?> <testsuite name="xfstests" errors="0" skipped="1" tests="3" time="12" timestamp="2017-01-31 18:24:06" > <testcase classname="ext4" name="generic/001" time="6"> </testcase> <testcase classname="ext4" name="generic/010" time="0"> <skipped message="src/dbtest not built" /> </testcase> <testcase classname="ext4" name="generic/013" time="3"> </testcase> </testsuite> Footnotes: [1] https://xunit.github.io/docs/format-xml-v2.html [2] http://help.catchsoftware.com/display/ET/JUnit+Format [3] jenkins.io [4] https://www.atlassian.com/software/bamboo [5] https://github.com/avocado-framework/avocado Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> --- check | 22 +++++++++--- common/rc | 3 ++ common/report | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 common/report diff --git a/check b/check index edc0899..1442b6c 100755 --- a/check +++ b/check @@ -54,7 +54,7 @@ export DIFF_LENGTH=${DIFF_LENGTH:=10} # by default don't output timestamps timestamp=${TIMESTAMP:=false} -rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist +rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.* SRC_GROUPS="generic shared" export SRC_DIR="tests" @@ -75,6 +75,7 @@ check options -r randomize test order -d dump test output to stdout -b brief test summary + --xunit generate summary in xunit format --large-fs optimise scratch device for large filesystems -s section run only specified section from config file -S section exclude the specified section from the config file @@ -296,6 +297,7 @@ while [ $# -gt 0 ]; do -d) DUMP_OUTPUT=true ;; -b) brief_test_summary=true;; + --xunit) REPORT_XUNIT=yes; DO_REPORT=yes ;; --large-fs) export LARGE_SCRATCH_DEV=yes ;; --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;; @@ -380,7 +382,10 @@ _wrapup() check="$RESULT_BASE/check" if $showme; then - : + if $needwrap; then + _make_section_report + needwrap=false + fi elif $needwrap; then if [ -f $check.time -a -f $tmp.time ]; then cat $check.time $tmp.time \ @@ -431,9 +436,9 @@ _wrapup() echo "Passed all $n_try tests" >>$tmp.summary fi echo "" >>$tmp.summary + _make_section_report needwrap=false fi - sum_bad=`expr $sum_bad + $n_bad` _wipe_counters rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time @@ -519,6 +524,7 @@ for section in $HOST_OPTIONS_SECTIONS; do echo "SECTION -- $section" fi + sect_start=`_wallclock` if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then echo "RECREATING -- $FSTYP on $TEST_DEV" _test_unmount 2> /dev/null @@ -624,9 +630,11 @@ for section in $HOST_OPTIONS_SECTIONS; do if $OPTIONS_HAVE_SECTIONS; then export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"` seqres="$RESULT_BASE/$section/$seqnum" + sectdir_res="$RESULT_BASE/$section" else export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"` seqres="$RESULT_BASE/$seqnum" + sectdir_res="$RESULT_BASE" fi mkdir -p $RESULT_DIR @@ -638,9 +646,10 @@ for section in $HOST_OPTIONS_SECTIONS; do start=0 stop=0 n_notrun=`expr $n_notrun + 1` + _make_testcase_report "list" continue fi - + tc_status="pass" if [ ! -f $seq ]; then echo " - no such test?" else @@ -704,6 +713,7 @@ for section in $HOST_OPTIONS_SECTIONS; do cat $seqres.notrun notrun="$notrun $seqnum" n_notrun=`expr $n_notrun + 1` + tc_status="notrun" else if [ $sts -ne 0 ] then @@ -763,10 +773,12 @@ for section in $HOST_OPTIONS_SECTIONS; do bad="$bad $seqnum" n_bad=`expr $n_bad + 1` quick=false + tc_status="fail" fi - + _make_testcase_report "$tc_status" seq="after_$seqnum" done + sect_stop=`_wallclock` _wrapup echo diff --git a/common/rc b/common/rc index f97b4d2..9f1a068 100644 --- a/common/rc +++ b/common/rc @@ -3246,6 +3246,9 @@ _get_fs_sysfs_attr() } +if [ ! -z "$DO_REPORT" ]; then + . ./common/report +fi init_rc diff --git a/common/report b/common/report new file mode 100644 index 0000000..05496c5 --- /dev/null +++ b/common/report @@ -0,0 +1,108 @@ +# +# Reports generator funcitons lives here +# + + +# +# Xunit format report functions +_xunit_make_section_report() +{ + # section ==> testsuite + #<?xml version="1.0" encoding="UTF-8"?> + #<testsuite errors="0" failures="1" name="avocado" skipped="1" tests="3" time="0.0942118167877" timestamp="2017-01-30 21:43:42.117398"> + sect_name=$section + sect_time=`expr $sect_stop - $sect_start` + + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + fi + report=$tmp.report.xunit.$sect_name.xml + + n_total=`expr $n_try + $n_notrun` + echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $sectdir_res/result.xml + stats="errors=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\" timestamp=\"$date_time\"" + echo "<testsuite name=\"xfstests\" $stats >" >> $sectdir_res/result.xml + cat $tmp.report.xunit.$sect_name.xml >> $sectdir_res/result.xml + echo "</testsuite>" >> $sectdir_res/result.xml + echo "Xunit report: $sectdir_res/result.xml" +} + +_xunit_make_testcase_report() +{ + test_status="$1" + test_time=`expr $stop - $start` + sect_name=$section + + #TODO: other places may also win if no-section mode will be named like 'default/global' + if [ $sect_name == '-no-sections-' ]; then + sect_name='global' + fi + report=$tmp.report.xunit.$sect_name.xml + strip="$SRC_DIR/" + test_name=${seq#$strip} + echo -e "\t<testcase classname=\"$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report + case $test_status in + "pass") + ;; + "notrun") + if [ -f $seqres.notrun ]; then + msg=`cat $seqres.notrun` + echo -e "\t\t<skipped message=\"$msg\" />" >> $report + else + echo -e "\t\t<skipped/>" >> $report + fi + ;; + "list") + echo -e "\t\t<skipped/>" >> $report + ;; + "fail") + if [ -z "$err_msg" ]; then + err_msg="Test $sequm failed, reason unknown" + fi + echo -e "\t\t<failure message=\"$err_msg\" type=\"TestFail\">" >> $report + if [ -f $seqres.full ]; then + echo -e "\t\t\t<system-out>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.full | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t\t</system-out>" >> $report + fi + if [ -f $seqres.dmesg ]; then + echo -e "\t\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + cat $seqres.dmesg | tr -dc '[:print:][:space:]' >>$report + printf ']]>\n' >>$report + echo -e "\t\t\t</system-err>" >> $report + elif [ -f $seqres.out.bad ]; then + echo -e "\t\t\t<system-err>" >> $report + printf '<![CDATA[\n' >>$report + $diff $seq.out $seqres.out.bad >>$report + printf ']]>\n' >>$report + echo -e "\t\t\t</system-err>" >> $report + fi + echo -e "\t\t</failure>" >> $report + ;; + *) + echo -e "\t\t<failure message=\"Unknown ret_state=$ret_state\" type=\"TestFail\"/>" >> $report + ;; + esac + echo -e "\t</testcase>" >> $report +} + + +# +# Common report generator entry points +_make_section_report() +{ + if [ ! -z "$REPORT_XUNIT" ]; then + _xunit_make_section_report + fi +} + +_make_testcase_report() +{ + test_status="$1" + if [ ! -z "$REPORT_XUNIT" ]; then + _xunit_make_testcase_report "$test_status" + fi +} -- 2.9.3 ^ permalink raw reply related [flat|nested] 14+ messages in thread
end of thread, other threads:[~2017-03-08 12:17 UTC | newest] Thread overview: 14+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2017-01-31 17:43 [PATCH 0/2][RESEND] Add report generators support Dmitry Monakhov 2017-01-31 17:43 ` [PATCH 1/2] check: prepare test report generator infrastructure Dmitry Monakhov 2017-02-16 9:18 ` Eryu Guan 2017-02-16 9:38 ` Dmitry Monakhov 2017-01-31 17:43 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 2017-02-16 10:08 ` Eryu Guan -- strict thread matches above, loose matches on Subject: below -- 2017-03-03 8:26 [PATCH 0/2] Add report generators support v3 Dmitry Monakhov 2017-03-03 8:26 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 2017-03-07 12:05 ` Eryu Guan 2017-03-07 13:13 ` Amir Goldstein 2017-03-08 3:00 ` Eryu Guan 2017-03-08 12:16 ` Dmitry Monakhov 2017-02-21 12:44 [PATCH 0/2] Add report generators support v2 Dmitry Monakhov 2017-02-21 12:44 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov 2017-03-02 8:13 ` Eryu Guan 2017-01-31 17:16 [PATCH 0/2] Add report generators support Dmitry Monakhov 2017-01-31 17:16 ` [PATCH 2/2] report: Add xunit format report generator Dmitry Monakhov
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.