nvdimm.lists.linux.dev archive mirror
 help / color / mirror / Atom feed
From: Brendan Higgins <brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
To: gregkh-hQyY1W1yCW8ekmWlsbkhG0B+6BGkLq7r@public.gmane.org,
	keescook-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org,
	mcgrof-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org,
	shuah-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
Cc: brakmo-b10kYP2dOMg@public.gmane.org,
	jdike-OPE4K8JWMJJBDgjK7y7TUQ@public.gmane.org,
	dri-devel-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org,
	linux-kselftest-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
	robh-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org,
	linux-nvdimm-hn68Rpc1hR1g9hUCZPvPmw@public.gmane.org,
	richard-/L3Ra7n9ekc@public.gmane.org,
	kieran.bingham-ryLnwIuWjnjg/C1BVhZhaw@public.gmane.org,
	Felix Guo
	<felixguoxiuping-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>,
	joel-U3u1mxZcP9KHXe+LvDLADg@public.gmane.org,
	khilman-rdvid1DuHRBWk0Htik3J/w@public.gmane.org,
	Brendan Higgins
	<brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>,
	Tim.Bird-7U/KSKJipcs@public.gmane.org,
	linux-um-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org,
	rostedt-nx8X9YLhiw1AfugRpC6u6w@public.gmane.org,
	julia.lawall-L2FTfq7BK8M@public.gmane.org,
	kunit-dev-/JYPxA39Uh5TLH3MbocFFw@public.gmane.org,
	linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
	daniel-/w4YWyX8dFk@public.gmane.org,
	mpe-Gsx/Oe8HsFggBc27wqDAHg@public.gmane.org,
	joe-6d6DIl74uiNBDgjK7y7TUQ@public.gmane.org
Subject: [RFC v2 12/14] kunit.py: improve output from python wrapper
Date: Tue, 23 Oct 2018 16:57:48 -0700	[thread overview]
Message-ID: <20181023235750.103146-13-brendanhiggins@google.com> (raw)
In-Reply-To: <20181023235750.103146-1-brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>

- add colors to displayed output
- add timing and summary

Signed-off-by: Felix Guo <felixguoxiuping-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
Signed-off-by: Brendan Higgins <brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
---
 tools/testing/kunit/kunit.py        | 20 ++++++-
 tools/testing/kunit/kunit_parser.py | 93 ++++++++++++++++++++++++++++-
 2 files changed, 109 insertions(+), 4 deletions(-)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 1356be404996b..b36c7b7924567 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -6,6 +6,7 @@
 import argparse
 import sys
 import os
+import time
 
 import kunit_config
 import kunit_kernel
@@ -24,17 +25,32 @@ parser.add_argument('--timeout', help='maximum number of seconds to allow for '
 cli_args = parser.parse_args()
 linux = kunit_kernel.LinuxSourceTree()
 
+config_start = time.time()
 success = linux.build_reconfig()
+config_end = time.time()
 if not success:
 	quit()
 
-print('Building KUnit Kernel ...')
+kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
+
+build_start = time.time()
 success = linux.build_um_kernel()
+build_end = time.time()
 if not success:
 	quit()
 
-print('Starting KUnit Kernel ...')
+kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
+test_start = time.time()
+
 if cli_args.raw_output:
 	kunit_parser.raw_output(linux.run_kernel(timeout=cli_args.timeout))
 else:
 	kunit_parser.parse_run_tests(linux.run_kernel(timeout=cli_args.timeout))
+
+test_end = time.time()
+
+kunit_parser.print_with_timestamp((
+	"Elapsed time: %.3fs total, %.3fs configuring, %.3fs " +
+	"building, %.3fs running.\n") % (test_end - config_start,
+	config_end - config_start, build_end - build_start,
+	test_end - test_start))
diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py
index 1dff3adb73bd3..d9051e407d5a7 100644
--- a/tools/testing/kunit/kunit_parser.py
+++ b/tools/testing/kunit/kunit_parser.py
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import re
+from datetime import datetime
 
 kunit_start_re = re.compile('console .* enabled')
 kunit_end_re = re.compile('List of all partitions:')
@@ -19,6 +20,94 @@ def raw_output(kernel_output):
 	for line in kernel_output:
 		print(line)
 
+DIVIDER = "=" * 30
+
+RESET = '\033[0;0m'
+
+def red(text):
+	return '\033[1;31m' + text + RESET
+
+def yellow(text):
+	return '\033[1;33m' + text + RESET
+
+def green(text):
+	return '\033[1;32m' + text + RESET
+
+def print_with_timestamp(message):
+	print('[%s] %s' % (datetime.now().strftime('%H:%M:%S'), message))
+
+def print_log(log):
+	for m in log:
+		print_with_timestamp(m)
+
 def parse_run_tests(kernel_output):
-	for output in isolate_kunit_output(kernel_output):
-		print(output)
+	test_case_output = re.compile('^kunit .*?: (.*)$')
+
+	test_module_success = re.compile('^kunit .*: all tests passed')
+	test_module_fail = re.compile('^kunit .*: one or more tests failed')
+
+	test_case_success = re.compile('^kunit (.*): (.*) passed')
+	test_case_fail = re.compile('^kunit (.*): (.*) failed')
+	test_case_crash = re.compile('^kunit (.*): (.*) crashed')
+
+	total_tests = set()
+	failed_tests = set()
+	crashed_tests = set()
+
+	def get_test_name(match):
+		return match.group(1) + ":" + match.group(2)
+
+	current_case_log = []
+	def end_one_test(match, log):
+		log.clear()
+		total_tests.add(get_test_name(match))
+
+	print_with_timestamp(DIVIDER)
+	for line in isolate_kunit_output(kernel_output):
+		# Ignore module output:
+		if (test_module_success.match(line) or
+		    test_module_fail.match(line)):
+			print_with_timestamp(DIVIDER)
+			continue
+
+		match = re.match(test_case_success, line)
+		if match:
+			print_with_timestamp(green("[PASSED] ") +
+					     get_test_name(match))
+			end_one_test(match, current_case_log)
+			continue
+
+		match = re.match(test_case_fail, line)
+		# Crashed tests will report as both failed and crashed. We only
+		# want to show and count it once.
+		if match and get_test_name(match) not in crashed_tests:
+			failed_tests.add(get_test_name(match))
+			print_with_timestamp(red("[FAILED] " +
+						 get_test_name(match)))
+			print_log(map(yellow, current_case_log))
+			print_with_timestamp("")
+			end_one_test(match, current_case_log)
+			continue
+
+		match = re.match(test_case_crash, line)
+		if match:
+			crashed_tests.add(get_test_name(match))
+			print_with_timestamp(yellow("[CRASH] " +
+						    get_test_name(match)))
+			print_log(current_case_log)
+			print_with_timestamp("")
+			end_one_test(match, current_case_log)
+			continue
+
+		# Strip off the `kunit module-name:` prefix
+		match = re.match(test_case_output, line)
+		if match:
+			current_case_log.append(match.group(1))
+		else:
+			current_case_log.append(line)
+
+	fmt = green if (len(failed_tests) + len(crashed_tests) == 0) else red
+	print_with_timestamp(
+		fmt("Testing complete. %d tests run. %d failed. %d crashed." %
+		    (len(total_tests), len(failed_tests), len(crashed_tests))))
+
-- 
2.19.1.568.g152ad8e336-goog

  parent reply	other threads:[~2018-10-23 23:57 UTC|newest]

Thread overview: 30+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-10-23 23:57 [RFC v2 00/14] kunit: introduce KUnit, the Linux kernel unit testing framework Brendan Higgins
     [not found] ` <20181023235750.103146-1-brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
2018-10-23 23:57   ` [RFC v2 01/14] kunit: test: add KUnit test runner core Brendan Higgins
2018-11-02 18:44     ` Shuah Khan
     [not found]       ` <017b111f-d960-c1ef-46ae-eb0eb639fe5b-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
2018-11-07  1:28         ` Brendan Higgins
2018-11-07 20:02           ` Shuah Khan
2018-10-23 23:57   ` [RFC v2 02/14] kunit: test: add test resource management API Brendan Higgins
2018-10-23 23:57   ` [RFC v2 03/14] kunit: test: add string_stream a std::stream like string builder Brendan Higgins
2018-10-23 23:57   ` [RFC v2 04/14] kunit: test: add test_stream a std::stream like logger Brendan Higgins
2018-10-23 23:57   ` [RFC v2 05/14] kunit: test: add the concept of expectations Brendan Higgins
2018-10-23 23:57   ` [RFC v2 06/14] arch: um: enable running kunit from User Mode Linux Brendan Higgins
2018-10-23 23:57   ` [RFC v2 07/14] kunit: test: add initial tests Brendan Higgins
2018-10-23 23:57   ` [RFC v2 08/14] arch: um: add shim to trap to allow installing a fault catcher for tests Brendan Higgins
2018-10-23 23:57   ` [RFC v2 09/14] kunit: test: add the concept of assertions Brendan Higgins
2018-10-23 23:57   ` [RFC v2 10/14] kunit: add Python libraries for handing KUnit config and kernel Brendan Higgins
2018-10-23 23:57   ` [RFC v2 11/14] kunit: add KUnit wrapper script and simple output parser Brendan Higgins
2018-10-23 23:57   ` Brendan Higgins [this message]
2018-10-23 23:57   ` [RFC v2 13/14] Documentation: kunit: add documentation for KUnit Brendan Higgins
2018-10-23 23:57   ` [RFC v2 14/14] MAINTAINERS: add entry for KUnit the unit testing framework Brendan Higgins
2018-10-24  9:14 ` [RFC v2 00/14] kunit: introduce KUnit, the Linux kernel " Daniel Vetter
2018-10-25 21:25   ` Brendan Higgins
2018-10-25 17:40 ` Shuah Khan
2018-11-02 18:23 ` Shuah Khan
2018-11-07  1:17   ` Brendan Higgins
2018-11-07 17:46     ` Frank Rowand
     [not found]       ` <04f677b1-bc44-f004-cf2a-51b47baf0965-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
2018-11-13 10:10         ` Brendan Higgins
2018-11-24  5:15 ` Knut Omang
     [not found]   ` <1543036529.4680.655.camel-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org>
2018-11-27  1:41     ` Brendan Higgins
2018-11-28 19:54       ` Knut Omang
2018-11-28 20:50         ` shuah
2018-11-30  0:59           ` Luis Chamberlain

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20181023235750.103146-13-brendanhiggins@google.com \
    --to=brendanhiggins-hpiqsd4aklfqt0dzr+alfa@public.gmane.org \
    --cc=Tim.Bird-7U/KSKJipcs@public.gmane.org \
    --cc=brakmo-b10kYP2dOMg@public.gmane.org \
    --cc=daniel-/w4YWyX8dFk@public.gmane.org \
    --cc=dri-devel-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org \
    --cc=felixguoxiuping-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org \
    --cc=gregkh-hQyY1W1yCW8ekmWlsbkhG0B+6BGkLq7r@public.gmane.org \
    --cc=jdike-OPE4K8JWMJJBDgjK7y7TUQ@public.gmane.org \
    --cc=joe-6d6DIl74uiNBDgjK7y7TUQ@public.gmane.org \
    --cc=joel-U3u1mxZcP9KHXe+LvDLADg@public.gmane.org \
    --cc=julia.lawall-L2FTfq7BK8M@public.gmane.org \
    --cc=keescook-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org \
    --cc=khilman-rdvid1DuHRBWk0Htik3J/w@public.gmane.org \
    --cc=kieran.bingham-ryLnwIuWjnjg/C1BVhZhaw@public.gmane.org \
    --cc=kunit-dev-/JYPxA39Uh5TLH3MbocFFw@public.gmane.org \
    --cc=linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org \
    --cc=linux-kselftest-u79uwXL29TY76Z2rM5mHXA@public.gmane.org \
    --cc=linux-nvdimm-hn68Rpc1hR1g9hUCZPvPmw@public.gmane.org \
    --cc=linux-um-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org \
    --cc=mcgrof-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org \
    --cc=mpe-Gsx/Oe8HsFggBc27wqDAHg@public.gmane.org \
    --cc=richard-/L3Ra7n9ekc@public.gmane.org \
    --cc=robh-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org \
    --cc=rostedt-nx8X9YLhiw1AfugRpC6u6w@public.gmane.org \
    --cc=shuah-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org \
    --subject='Re: [RFC v2 12/14] kunit.py: improve output from python wrapper' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).