* [PATCH 1/2] kunit: tool: make --raw_output support only showing kunit output
@ 2021-07-29 23:55 Daniel Latypov
2021-07-29 23:55 ` [PATCH 2/2] kunit: add 'kunit.action' param to allow listing out tests Daniel Latypov
0 siblings, 1 reply; 2+ messages in thread
From: Daniel Latypov @ 2021-07-29 23:55 UTC (permalink / raw)
To: brendanhiggins, davidgow
Cc: linux-kernel, kunit-dev, linux-kselftest, skhan, Daniel Latypov
--raw_output is nice, but it would be nicer if could show only output
after KUnit tests ahve started.
So change the flag to allow specifying a string ('kunit').
Make it so `--raw_output` alone will default to `--raw_output=all` and
have the same original behavior.
Drop the small kunit_parser.raw_output() function since it feels wrong
to put it in "kunit_parser.py" when the point of it is to not parse
anything.
E.g.
$ ./tools/testing/kunit/kunit.py run --raw_output=kunit
...
[15:24:07] Starting KUnit Kernel ...
TAP version 14
1..1
# Subtest: example
1..3
# example_simple_test: initializing
ok 1 - example_simple_test
# example_skip_test: initializing
# example_skip_test: You should not see a line below.
ok 2 - example_skip_test # SKIP this test should be skipped
# example_mark_skipped_test: initializing
# example_mark_skipped_test: You should see a line below.
# example_mark_skipped_test: You should see this line.
ok 3 - example_mark_skipped_test # SKIP this test should be skipped
ok 1 - example
[15:24:10] Elapsed time: 6.487s total, 0.001s configuring, 3.510s building, 0.000s running
Signed-off-by: Daniel Latypov <dlatypov@google.com>
---
Documentation/dev-tools/kunit/kunit-tool.rst | 9 ++++++---
tools/testing/kunit/kunit.py | 20 +++++++++++++++-----
tools/testing/kunit/kunit_parser.py | 4 ----
tools/testing/kunit/kunit_tool_test.py | 9 +++++++++
4 files changed, 30 insertions(+), 12 deletions(-)
diff --git a/Documentation/dev-tools/kunit/kunit-tool.rst b/Documentation/dev-tools/kunit/kunit-tool.rst
index c7ff9afe407a..ae52e0f489f9 100644
--- a/Documentation/dev-tools/kunit/kunit-tool.rst
+++ b/Documentation/dev-tools/kunit/kunit-tool.rst
@@ -114,9 +114,12 @@ results in TAP format, you can pass the ``--raw_output`` argument.
./tools/testing/kunit/kunit.py run --raw_output
-.. note::
- The raw output from test runs may contain other, non-KUnit kernel log
- lines.
+The raw output from test runs may contain other, non-KUnit kernel log
+lines. You can see just KUnit output with ``--raw_output=kunit``:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --raw_output=kunit
If you have KUnit results in their raw TAP format, you can parse them and print
the human-readable summary with the ``parse`` command for kunit_tool. This
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 7174377c2172..5a931456e718 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -16,6 +16,7 @@ assert sys.version_info >= (3, 7), "Python version is too old"
from collections import namedtuple
from enum import Enum, auto
+from typing import Iterable
import kunit_config
import kunit_json
@@ -114,7 +115,16 @@ def parse_tests(request: KunitParseRequest) -> KunitResult:
'Tests not Parsed.')
if request.raw_output:
- kunit_parser.raw_output(request.input_data)
+ output: Iterable[str] = request.input_data
+ if request.raw_output == 'all':
+ pass
+ elif request.raw_output == 'kunit':
+ output = kunit_parser.extract_tap_lines(output)
+ else:
+ print(f'Unknown --raw_output option "{request.raw_output}"', file=sys.stderr)
+ for line in output:
+ print(line.rstrip())
+
else:
test_result = kunit_parser.parse_run_tests(request.input_data)
parse_end = time.time()
@@ -135,7 +145,6 @@ def parse_tests(request: KunitParseRequest) -> KunitResult:
return KunitResult(KunitStatus.SUCCESS, test_result,
parse_end - parse_start)
-
def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
run_start = time.time()
@@ -181,7 +190,7 @@ def add_common_opts(parser) -> None:
parser.add_argument('--build_dir',
help='As in the make command, it specifies the build '
'directory.',
- type=str, default='.kunit', metavar='build_dir')
+ type=str, default='.kunit', metavar='build_dir')
parser.add_argument('--make_options',
help='X=Y make option, can be repeated.',
action='append')
@@ -246,8 +255,9 @@ def add_exec_opts(parser) -> None:
action='append')
def add_parse_opts(parser) -> None:
- parser.add_argument('--raw_output', help='don\'t format output from kernel',
- action='store_true')
+ parser.add_argument('--raw_output', help='If set don\'t format output from kernel. '
+ 'If set to --raw_output=kunit, filters to just KUnit output.',
+ type=str, nargs='?', const='all', default=None)
parser.add_argument('--json',
nargs='?',
help='Stores test results in a JSON, and either '
diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py
index b88db3f51dc5..84938fefbac0 100644
--- a/tools/testing/kunit/kunit_parser.py
+++ b/tools/testing/kunit/kunit_parser.py
@@ -106,10 +106,6 @@ def extract_tap_lines(kernel_output: Iterable[str]) -> LineStream:
yield line_num, line[prefix_len:]
return LineStream(lines=isolate_kunit_output(kernel_output))
-def raw_output(kernel_output) -> None:
- for line in kernel_output:
- print(line.rstrip())
-
DIVIDER = '=' * 60
RESET = '\033[0;0m'
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 628ab00f74bc..619c4554cbff 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -399,6 +399,15 @@ class KUnitMainTest(unittest.TestCase):
self.assertNotEqual(call, mock.call(StrContains('Testing complete.')))
self.assertNotEqual(call, mock.call(StrContains(' 0 tests run')))
+ def test_run_raw_output_kunit(self):
+ self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
+ kunit.main(['run', '--raw_output=kunit'], self.linux_source_mock)
+ self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
+ self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)
+ for call in self.print_mock.call_args_list:
+ self.assertNotEqual(call, mock.call(StrContains('Testing complete.')))
+ self.assertNotEqual(call, mock.call(StrContains(' 0 tests run')))
+
def test_exec_timeout(self):
timeout = 3453
kunit.main(['exec', '--timeout', str(timeout)], self.linux_source_mock)
base-commit: f684616e08e9cd9db3cd53fe2e068dfe02481657
--
2.32.0.554.ge1b32706d8-goog
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [PATCH 2/2] kunit: add 'kunit.action' param to allow listing out tests
2021-07-29 23:55 [PATCH 1/2] kunit: tool: make --raw_output support only showing kunit output Daniel Latypov
@ 2021-07-29 23:55 ` Daniel Latypov
0 siblings, 0 replies; 2+ messages in thread
From: Daniel Latypov @ 2021-07-29 23:55 UTC (permalink / raw)
To: brendanhiggins, davidgow
Cc: linux-kernel, kunit-dev, linux-kselftest, skhan, Daniel Latypov
Context:
It's difficult to map a given .kunitconfig => set of enabled tests.
Having a standard, easy way of getting the list could be useful in a
number of ways. For example, if we also extended kunit.filter_glob to
allow filtering on tests, this would allow users to run tests cases one
by one if they wanted to debug hermeticity issues.
This patch:
* adds a kunit.action module param with one valid non-null value, "list"
* for the "list" action, it simply prints out "<suite>.<test>"
* does not itself introduce kunit.py changes to make use of this [1].
Note: kunit.filter_glob is respected for this and all future actions.
Note: we need a TAP header for kunit.py to isolate the KUnit output.
Tested:
$ ./tools/testing/kunit/kunit.py run --kernel_arg=kunit.action=list --raw_output=kunit
...
TAP version 14
1..1
example.example_simple_test
example.example_skip_test
example.example_mark_skipped_test
reboot: System halted
[1] The interface for this can work in a few ways. We could add a
--list_tests flag or a new subcommand. But this change is enough to
allow people to split each suite into its own invocation, e.g. via a
short script like:
#!/bin/bash
cd $(git rev-parse --show-toplevel)
for suite in $(
./tools/testing/kunit/kunit.py run --kernel_args=kunit.action=list --raw_output=kunit |
sed -n '/^TAP version/,$p' | grep -P -o '^[a-z][a-z0-9_-]+\.' | tr -d '.' | sort -u);
do
./tools/testing/kunit/kunit.py run "${suite}"
done
Signed-off-by: Daniel Latypov <dlatypov@google.com>
---
lib/kunit/executor.c | 46 +++++++++++++++++++++++++++++++++++++++-----
1 file changed, 41 insertions(+), 5 deletions(-)
diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
index acd1de436f59..77d99ee5ed64 100644
--- a/lib/kunit/executor.c
+++ b/lib/kunit/executor.c
@@ -15,9 +15,16 @@ extern struct kunit_suite * const * const __kunit_suites_end[];
#if IS_BUILTIN(CONFIG_KUNIT)
static char *filter_glob_param;
+static char *action_param;
+
module_param_named(filter_glob, filter_glob_param, charp, 0);
MODULE_PARM_DESC(filter_glob,
- "Filter which KUnit test suites run at boot-time, e.g. list*");
+ "Filter which KUnit test suites run at boot-time, e.g. list*");
+module_param_named(action, action_param, charp, 0);
+MODULE_PARM_DESC(action,
+ "Changes KUnit executor behavior, valid values are:\n"
+ "<none>: run the tests like normal\n"
+ "'list' to list test names instead of running them.\n");
static char *kunit_shutdown;
core_param(kunit_shutdown, kunit_shutdown, charp, 0644);
@@ -109,6 +116,33 @@ static void kunit_print_tap_header(struct suite_set *suite_set)
pr_info("1..%d\n", num_of_suites);
}
+static void kunit_exec_run_tests(struct suite_set *suite_set)
+{
+ struct kunit_suite * const * const *suites;
+
+ kunit_print_tap_header(suite_set);
+
+ for (suites = suite_set->start; suites < suite_set->end; suites++)
+ __kunit_test_suites_init(*suites);
+}
+
+static void kunit_exec_list_tests(struct suite_set *suite_set)
+{
+ unsigned int i;
+ struct kunit_suite * const * const *suites;
+ struct kunit_case *test_case;
+
+ /* Hack: print a tap header so kunit.py can find the start of KUnit output. */
+ kunit_print_tap_header(suite_set);
+
+ for (suites = suite_set->start; suites < suite_set->end; suites++)
+ for (i = 0; (*suites)[i] != NULL; i++) {
+ kunit_suite_for_each_test_case((*suites)[i], test_case) {
+ pr_info("%s.%s\n", (*suites)[i]->name, test_case->name);
+ }
+ }
+}
+
int kunit_run_all_tests(void)
{
struct kunit_suite * const * const *suites;
@@ -120,10 +154,12 @@ int kunit_run_all_tests(void)
if (filter_glob_param)
suite_set = kunit_filter_suites(&suite_set, filter_glob_param);
- kunit_print_tap_header(&suite_set);
-
- for (suites = suite_set.start; suites < suite_set.end; suites++)
- __kunit_test_suites_init(*suites);
+ if (!action_param)
+ kunit_exec_run_tests(&suite_set);
+ else if (strcmp(action_param, "list") == 0)
+ kunit_exec_list_tests(&suite_set);
+ else
+ pr_err("kunit executor: unknown action '%s'\n", action_param);
if (filter_glob_param) { /* a copy was made of each array */
for (suites = suite_set.start; suites < suite_set.end; suites++)
--
2.32.0.554.ge1b32706d8-goog
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2021-07-29 23:55 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-07-29 23:55 [PATCH 1/2] kunit: tool: make --raw_output support only showing kunit output Daniel Latypov
2021-07-29 23:55 ` [PATCH 2/2] kunit: add 'kunit.action' param to allow listing out tests Daniel Latypov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).