From: Brendan Higgins <brendanhiggins@google.com>
To: gregkh@linuxfoundation.org, keescook@google.com,
mcgrof@kernel.org, shuah@kernel.org
Cc: joel@jms.id.au, mpe@ellerman.id.au, joe@perches.com,
brakmo@fb.com, rostedt@goodmis.org, Tim.Bird@sony.com,
khilman@baylibre.com, julia.lawall@lip6.fr,
linux-kselftest@vger.kernel.org, kunit-dev@googlegroups.com,
linux-kernel@vger.kernel.org, jdike@addtoit.com, richard@nod.at,
linux-um@lists.infradead.org, daniel@ffwll.ch,
dri-devel@lists.freedesktop.org, robh@kernel.org,
dan.j.williams@intel.com, linux-nvdimm@lists.01.org,
kieran.bingham@ideasonboard.com,
Brendan Higgins <brendanhiggins@google.com>,
Felix Guo <felixguoxiuping@gmail.com>
Subject: [RFC v2 12/14] kunit.py: improve output from python wrapper
Date: Tue, 23 Oct 2018 16:57:48 -0700 [thread overview]
Message-ID: <20181023235750.103146-13-brendanhiggins@google.com> (raw)
In-Reply-To: <20181023235750.103146-1-brendanhiggins@google.com>
- add colors to displayed output
- add timing and summary
Signed-off-by: Felix Guo <felixguoxiuping@gmail.com>
Signed-off-by: Brendan Higgins <brendanhiggins@google.com>
---
tools/testing/kunit/kunit.py | 20 ++++++-
tools/testing/kunit/kunit_parser.py | 93 ++++++++++++++++++++++++++++-
2 files changed, 109 insertions(+), 4 deletions(-)
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 1356be404996b..b36c7b7924567 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -6,6 +6,7 @@
import argparse
import sys
import os
+import time
import kunit_config
import kunit_kernel
@@ -24,17 +25,32 @@ parser.add_argument('--timeout', help='maximum number of seconds to allow for '
cli_args = parser.parse_args()
linux = kunit_kernel.LinuxSourceTree()
+config_start = time.time()
success = linux.build_reconfig()
+config_end = time.time()
if not success:
quit()
-print('Building KUnit Kernel ...')
+kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
+
+build_start = time.time()
success = linux.build_um_kernel()
+build_end = time.time()
if not success:
quit()
-print('Starting KUnit Kernel ...')
+kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
+test_start = time.time()
+
if cli_args.raw_output:
kunit_parser.raw_output(linux.run_kernel(timeout=cli_args.timeout))
else:
kunit_parser.parse_run_tests(linux.run_kernel(timeout=cli_args.timeout))
+
+test_end = time.time()
+
+kunit_parser.print_with_timestamp((
+ "Elapsed time: %.3fs total, %.3fs configuring, %.3fs " +
+ "building, %.3fs running.\n") % (test_end - config_start,
+ config_end - config_start, build_end - build_start,
+ test_end - test_start))
diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py
index 1dff3adb73bd3..d9051e407d5a7 100644
--- a/tools/testing/kunit/kunit_parser.py
+++ b/tools/testing/kunit/kunit_parser.py
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
import re
+from datetime import datetime
kunit_start_re = re.compile('console .* enabled')
kunit_end_re = re.compile('List of all partitions:')
@@ -19,6 +20,94 @@ def raw_output(kernel_output):
for line in kernel_output:
print(line)
+DIVIDER = "=" * 30
+
+RESET = '\033[0;0m'
+
+def red(text):
+ return '\033[1;31m' + text + RESET
+
+def yellow(text):
+ return '\033[1;33m' + text + RESET
+
+def green(text):
+ return '\033[1;32m' + text + RESET
+
+def print_with_timestamp(message):
+ print('[%s] %s' % (datetime.now().strftime('%H:%M:%S'), message))
+
+def print_log(log):
+ for m in log:
+ print_with_timestamp(m)
+
def parse_run_tests(kernel_output):
- for output in isolate_kunit_output(kernel_output):
- print(output)
+ test_case_output = re.compile('^kunit .*?: (.*)$')
+
+ test_module_success = re.compile('^kunit .*: all tests passed')
+ test_module_fail = re.compile('^kunit .*: one or more tests failed')
+
+ test_case_success = re.compile('^kunit (.*): (.*) passed')
+ test_case_fail = re.compile('^kunit (.*): (.*) failed')
+ test_case_crash = re.compile('^kunit (.*): (.*) crashed')
+
+ total_tests = set()
+ failed_tests = set()
+ crashed_tests = set()
+
+ def get_test_name(match):
+ return match.group(1) + ":" + match.group(2)
+
+ current_case_log = []
+ def end_one_test(match, log):
+ log.clear()
+ total_tests.add(get_test_name(match))
+
+ print_with_timestamp(DIVIDER)
+ for line in isolate_kunit_output(kernel_output):
+ # Ignore module output:
+ if (test_module_success.match(line) or
+ test_module_fail.match(line)):
+ print_with_timestamp(DIVIDER)
+ continue
+
+ match = re.match(test_case_success, line)
+ if match:
+ print_with_timestamp(green("[PASSED] ") +
+ get_test_name(match))
+ end_one_test(match, current_case_log)
+ continue
+
+ match = re.match(test_case_fail, line)
+ # Crashed tests will report as both failed and crashed. We only
+ # want to show and count it once.
+ if match and get_test_name(match) not in crashed_tests:
+ failed_tests.add(get_test_name(match))
+ print_with_timestamp(red("[FAILED] " +
+ get_test_name(match)))
+ print_log(map(yellow, current_case_log))
+ print_with_timestamp("")
+ end_one_test(match, current_case_log)
+ continue
+
+ match = re.match(test_case_crash, line)
+ if match:
+ crashed_tests.add(get_test_name(match))
+ print_with_timestamp(yellow("[CRASH] " +
+ get_test_name(match)))
+ print_log(current_case_log)
+ print_with_timestamp("")
+ end_one_test(match, current_case_log)
+ continue
+
+ # Strip off the `kunit module-name:` prefix
+ match = re.match(test_case_output, line)
+ if match:
+ current_case_log.append(match.group(1))
+ else:
+ current_case_log.append(line)
+
+ fmt = green if (len(failed_tests) + len(crashed_tests) == 0) else red
+ print_with_timestamp(
+ fmt("Testing complete. %d tests run. %d failed. %d crashed." %
+ (len(total_tests), len(failed_tests), len(crashed_tests))))
+
--
2.19.1.568.g152ad8e336-goog
next prev parent reply other threads:[~2018-10-24 2:44 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-10-23 23:57 [RFC v2 00/14] kunit: introduce KUnit, the Linux kernel unit testing framework Brendan Higgins
2018-10-23 23:57 ` [RFC v2 01/14] kunit: test: add KUnit test runner core Brendan Higgins
2018-11-02 18:44 ` Shuah Khan
2018-11-07 1:28 ` Brendan Higgins
2018-11-07 20:02 ` Shuah Khan
2018-10-23 23:57 ` [RFC v2 02/14] kunit: test: add test resource management API Brendan Higgins
2018-10-23 23:57 ` [RFC v2 03/14] kunit: test: add string_stream a std::stream like string builder Brendan Higgins
2018-10-23 23:57 ` [RFC v2 04/14] kunit: test: add test_stream a std::stream like logger Brendan Higgins
2018-10-23 23:57 ` [RFC v2 05/14] kunit: test: add the concept of expectations Brendan Higgins
2018-10-23 23:57 ` [RFC v2 06/14] arch: um: enable running kunit from User Mode Linux Brendan Higgins
2018-10-23 23:57 ` [RFC v2 07/14] kunit: test: add initial tests Brendan Higgins
2018-10-23 23:57 ` [RFC v2 08/14] arch: um: add shim to trap to allow installing a fault catcher for tests Brendan Higgins
2018-10-23 23:57 ` [RFC v2 09/14] kunit: test: add the concept of assertions Brendan Higgins
2018-10-23 23:57 ` [RFC v2 10/14] kunit: add Python libraries for handing KUnit config and kernel Brendan Higgins
2018-10-23 23:57 ` [RFC v2 11/14] kunit: add KUnit wrapper script and simple output parser Brendan Higgins
2018-10-23 23:57 ` Brendan Higgins [this message]
2018-10-23 23:57 ` [RFC v2 13/14] Documentation: kunit: add documentation for KUnit Brendan Higgins
2018-10-23 23:57 ` [RFC v2 14/14] MAINTAINERS: add entry for KUnit the unit testing framework Brendan Higgins
2018-10-24 9:14 ` [RFC v2 00/14] kunit: introduce KUnit, the Linux kernel " Daniel Vetter
2018-10-25 21:25 ` Brendan Higgins
2018-10-25 17:40 ` Shuah Khan
2018-11-02 18:23 ` Shuah Khan
2018-11-07 1:17 ` Brendan Higgins
2018-11-07 17:46 ` Frank Rowand
2018-11-13 10:10 ` Brendan Higgins
2018-11-24 5:15 ` Knut Omang
2018-11-27 1:41 ` Brendan Higgins
2018-11-28 19:54 ` Knut Omang
2018-11-28 20:50 ` shuah
2018-11-30 0:59 ` Luis Chamberlain
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20181023235750.103146-13-brendanhiggins@google.com \
--to=brendanhiggins@google.com \
--cc=Tim.Bird@sony.com \
--cc=brakmo@fb.com \
--cc=dan.j.williams@intel.com \
--cc=daniel@ffwll.ch \
--cc=dri-devel@lists.freedesktop.org \
--cc=felixguoxiuping@gmail.com \
--cc=gregkh@linuxfoundation.org \
--cc=jdike@addtoit.com \
--cc=joe@perches.com \
--cc=joel@jms.id.au \
--cc=julia.lawall@lip6.fr \
--cc=keescook@google.com \
--cc=khilman@baylibre.com \
--cc=kieran.bingham@ideasonboard.com \
--cc=kunit-dev@googlegroups.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=linux-nvdimm@lists.01.org \
--cc=linux-um@lists.infradead.org \
--cc=mcgrof@kernel.org \
--cc=mpe@ellerman.id.au \
--cc=richard@nod.at \
--cc=robh@kernel.org \
--cc=rostedt@goodmis.org \
--cc=shuah@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).