[PATCH] oeqa: Fix subTest result reporting

* [PATCH] oeqa: Fix subTest result reporting
@ 2019-11-08 19:21 Joshua Watt
  2019-11-12  4:11 ` Mittal, Anuj
  0 siblings, 1 reply; 2+ messages in thread
From: Joshua Watt @ 2019-11-08 19:21 UTC (permalink / raw)
  To: openembedded-core

Fixes the way that subTest results are reported so that each subTest is
reported independently, with it's own status. The base test case is also
reported, but only if it actually had a status.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
 meta/lib/oeqa/core/runner.py                | 44 +++++++++++++++++----
 meta/lib/oeqa/core/utils/concurrencytest.py |  3 ++
 2 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index f656e1a9c5f..ee9db6f7951 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -40,6 +40,7 @@ class OETestResult(_TestResult):
         super(OETestResult, self).__init__(*args, **kwargs)
 
         self.successes = []
+        self.subtests = {}
         self.starttime = {}
         self.endtime = {}
         self.progressinfo = {}
@@ -145,6 +146,13 @@ class OETestResult(_TestResult):
                 else:
                     self.extraresults[k] = v
 
+    def addSubTest(self, test, subtest, err, details=None):
+        self.extractExtraResults(test, details=details)
+        self.subtests.setdefault(test, list()).append(subtest)
+        if err is None:
+            self.successes.append((subtest, None))
+        return super(OETestResult, self).addSubTest(test, subtest, err)
+
     def addError(self, test, *args, details = None):
         self.extractExtraResults(test, details = details)
         return super(OETestResult, self).addError(test, *args)
@@ -169,18 +177,18 @@ class OETestResult(_TestResult):
 
     def logDetails(self, json_file_dir=None, configuration=None, result_id=None,
             dump_streams=False):
-        self.tc.logger.info("RESULTS:")
-
-        result = self.extraresults
-        logs = {}
-        if hasattr(self.tc, "extraresults"):
-            result.update(self.tc.extraresults)
 
-        for case_name in self.tc._registry['cases']:
-            case = self.tc._registry['cases'][case_name]
+        def logCaseDetails(case, required):
+            nonlocal self
+            nonlocal result
+            nonlocal dump_streams
+            nonlocal logs
 
             (status, log) = self._getTestResultDetails(case)
 
+            if status == "UNKNOWN" and not required:
+                return
+
             t = ""
             if case.id() in self.starttime and case.id() in self.endtime:
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
@@ -197,6 +205,26 @@ class OETestResult(_TestResult):
                 report['stderr'] = stderr
             result[case.id()] = report
 
+        self.tc.logger.info("RESULTS:")
+
+        result = self.extraresults
+        logs = {}
+        if hasattr(self.tc, "extraresults"):
+            result.update(self.tc.extraresults)
+
+        for case_name in self.tc._registry['cases']:
+            case = self.tc._registry['cases'][case_name]
+
+            if case in self.subtests:
+                for subtest in self.subtests[case]:
+                    logCaseDetails(subtest, True)
+
+                # The base case may not have any reported details. If it comes
+                # up as unknown, don't report anything.
+                logCaseDetails(case, False)
+            else:
+                logCaseDetails(case, True)
+
         for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']:
             if i not in logs:
                 continue
diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py
index 0f7b3dcc113..ae68fd50902 100644
--- a/meta/lib/oeqa/core/utils/concurrencytest.py
+++ b/meta/lib/oeqa/core/utils/concurrencytest.py
@@ -81,6 +81,9 @@ class ProxyTestResult:
     def _addResult(self, method, test, *args, exception = False, **kwargs):
         return method(test, *args, **kwargs)
 
+    def addSubTest(self, test, subtest, outcome, **kwargs):
+        self._addResult(self.result.addSubTest, test, subtest, outcome, exception = (outcome is not None), **kwargs)
+
     def addError(self, test, err = None, **kwargs):
         self._addResult(self.result.addError, test, err, exception = True, **kwargs)
 
-- 
2.23.0



^ permalink raw reply related	[flat|nested] 2+ messages in thread