[PATCH 1/5] oeqa/core/runner: refactor for OEQA to write json testresult

[PATCH 1/5] oeqa/core/runner: refactor for OEQA to write json testresult

[Yeoh Ee Peng]

Refactor the original _getDetailsNotPassed method to return
testresult details (test status and log), which will be reused
by future OEQA code to write json testresult.

Take the opportunity to consolidate and simplify the logic used
to gather test status and log within the TestResult instance.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/core/runner.py | 70 ++++++++++++++++++--------------------------
 1 file changed, 29 insertions(+), 41 deletions(-)

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index eeb625b..56666ee 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -76,40 +76,43 @@ class OETestResult(_TestResult):
         else:
             msg = "%s - FAIL - Required tests failed" % component
         skipped = len(self.skipped)
-        if skipped: 
+        if skipped:
             msg += " (skipped=%d)" % skipped
         self.tc.logger.info(msg)
 
-    def _getDetailsNotPassed(self, case, type, desc):
-        found = False
+    def _getTestResultDetails(self, case):
+        result_types = ['failures', 'errors', 'skipped', 'expectedFailures', 'successes']
+        result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL', 'PASSED']
 
-        for (scase, msg) in getattr(self, type):
-            if case.id() == scase.id():
-                found = True
-                break
-            scase_str = str(scase.id())
-
-            # When fails at module or class level the class name is passed as string
-            # so figure out to see if match
-            m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
-            if m:
-                if case.__class__.__module__ == m.group('module_name'):
+        for idx, name in enumerate(result_types):
+            found = False
+            for (scase, msg) in getattr(self, result_types[idx]):
+                if case.id() == scase.id():
                     found = True
                     break
+                scase_str = str(scase.id())
 
-            m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
-            if m:
-                class_name = "%s.%s" % (case.__class__.__module__,
-                        case.__class__.__name__)
+                # When fails at module or class level the class name is passed as string
+                # so figure out to see if match
+                m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
+                if m:
+                    if case.__class__.__module__ == m.group('module_name'):
+                        found = True
+                        break
 
-                if class_name == m.group('class_name'):
-                    found = True
-                    break
+                m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
+                if m:
+                    class_name = "%s.%s" % (case.__class__.__module__,
+                            case.__class__.__name__)
+
+                    if class_name == m.group('class_name'):
+                        found = True
+                        break
 
-        if found:
-            return (found, msg)
+            if found:
+                return result_desc[idx], msg
 
-        return (found, None)
+        return 'UNKNOWN', None
 
     def addSuccess(self, test):
         #Added so we can keep track of successes too
@@ -121,17 +124,7 @@ class OETestResult(_TestResult):
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
-            result_types = ['failures', 'errors', 'skipped', 'expectedFailures', 'successes']
-            result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL', 'PASSED']
-
-            fail = False
-            desc = None
-            for idx, name in enumerate(result_types):
-                (fail, msg) = self._getDetailsNotPassed(case, result_types[idx],
-                        result_desc[idx])
-                if fail:
-                    desc = result_desc[idx]
-                    break
+            (status, log) = self._getTestResultDetails(case)
 
             oeid = -1
             if hasattr(case, 'decorators'):
@@ -143,12 +136,7 @@ class OETestResult(_TestResult):
             if case.id() in self.starttime and case.id() in self.endtime:
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
 
-            if fail:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(),
-                    oeid, desc, t))
-            else:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(),
-                    oeid, 'UNKNOWN', t))
+            self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t))
 
 class OEListTestsResult(object):
     def wasSuccessful(self):
-- 
2.7.4

[PATCH 2/5] oeqa/core/runner: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA need to output testresult into json file, where these json
testresult file will be stored in git repository by the future
test-case-management tools.

Both the testresult (eg. PASSED, FAILED, ERROR) and  the test log
(eg. message from unit test assertion) will be created for storing.

Also the library class inside this patch will be reused by the future
test-case-management tools to write json testresult for manual test
case executed.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/core/runner.py | 64 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index 56666ee..56e8526 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -6,6 +6,8 @@ import time
 import unittest
 import logging
 import re
+import json
+import pathlib
 
 from unittest import TextTestResult as _TestResult
 from unittest import TextTestRunner as _TestRunner
@@ -119,8 +121,9 @@ class OETestResult(_TestResult):
         self.successes.append((test, None))
         super(OETestResult, self).addSuccess(test)
 
-    def logDetails(self):
+    def logDetails(self, json_file_dir=''):
         self.tc.logger.info("RESULTS:")
+        results = {}
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
@@ -137,6 +140,12 @@ class OETestResult(_TestResult):
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
 
             self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t))
+            results[case.id()] = (status, log)
+
+        if len(json_file_dir) > 0:
+            tresultjsonhelper = OETestResultJSONHelper()
+            tresultjsonhelper.dump_testresult_file(results, json_file_dir)
+            tresultjsonhelper.dump_log_files(results, os.path.join(json_file_dir, 'logs'))
 
 class OEListTestsResult(object):
     def wasSuccessful(self):
@@ -249,3 +258,56 @@ class OETestRunner(_TestRunner):
             self._list_tests_module(suite)
 
         return OEListTestsResult()
+
+class OETestResultJSONHelper(object):
+
+    def get_testsuite_from_testcase(self, testcase):
+        testsuite = testcase[0:testcase.rfind(".")]
+        return testsuite
+
+    def get_testsuite_testcase_dictionary(self, testresults):
+        testsuite_testcase_dict = {}
+        for testcase in testresults.keys():
+            testsuite = self.get_testsuite_from_testcase(testcase)
+            if testsuite in testsuite_testcase_dict:
+                testsuite_testcase_dict[testsuite].append(testcase)
+            else:
+                testsuite_testcase_dict[testsuite] = [testcase]
+        return testsuite_testcase_dict
+
+    def _create_testcase_testresult_object(self, testcase_list, testresults):
+        testcase_dict = {}
+        for testcase in sorted(testcase_list):
+            testcase_dict[testcase] = {"testresult": testresults[testcase][0]}
+        return testcase_dict
+
+    def _create_json_testsuite_string(self, testresults):
+        testsuite_testcase = self.get_testsuite_testcase_dictionary(testresults)
+        testsuite_object = {'testsuite': {}}
+        testsuite_dict = testsuite_object['testsuite']
+        for testsuite in sorted(testsuite_testcase.keys()):
+            testsuite_dict[testsuite] = {'testcase': {}}
+            testsuite_dict[testsuite]['testcase'] = self._create_testcase_testresult_object(
+                                                        testsuite_testcase[testsuite],
+                                                        testresults)
+        return json.dumps(testsuite_object, sort_keys=True, indent=4)
+
+    def _write_file(self, write_dir, file_name, file_content):
+        file_path = os.path.join(write_dir, file_name)
+        with open(file_path, 'w') as the_file:
+            the_file.write(file_content)
+
+    def dump_testresult_file(self, testresults, write_dir):
+        if not os.path.exists(write_dir):
+            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)
+        json_testsuite = self._create_json_testsuite_string(testresults)
+        self._write_file(write_dir, 'testresults.json', json_testsuite)
+
+    def dump_log_files(self, testresults, write_dir):
+        if not os.path.exists(write_dir):
+            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)
+        for testcase in testresults.keys():
+            test_log = testresults[testcase][1]
+            if test_log is not None:
+                file_name = '%s.log' % testcase
+                self._write_file(write_dir, file_name, test_log)
-- 
2.7.4

[PATCH 3/5] oeqa/selftest/context: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA selftest need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/context.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index c78947e..7a72f17 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -73,7 +73,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
 
         parser.add_argument('--machine', required=False, choices=['random', 'all'],
                             help='Run tests on different machines (random/all).')
-        
+
         parser.set_defaults(func=self.run)
 
     def _get_available_machines(self):
@@ -99,8 +99,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         return cases_paths
 
     def _process_args(self, logger, args):
-        args.output_log = '%s-results-%s.log' % (self.name,
-                time.strftime("%Y%m%d%H%M%S"))
+        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
+        args.output_log = '%s-results-%s.log' % (self.name, args.test_start_time)
         args.test_data_file = None
         args.CASES_PATHS = None
 
@@ -220,7 +220,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         else:
             self._pre_run()
             rc = self.tc.runTests(**self.tc_kwargs['run'])
-            rc.logDetails()
+            json_result_dir = os.path.join(os.path.dirname(os.path.abspath(args.output_log)),
+                                           'json_testresults-%s' % args.test_start_time,
+                                           'oe-selftest')
+            rc.logDetails(json_result_dir)
             rc.logSummary(self.name)
 
         return rc
-- 
2.7.4

[PATCH 4/5] testimage.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA testimage need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testimage.bbclass | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 2642a72..8c88340 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -308,7 +308,13 @@ def testimage_main(d):
     # Show results (if we have them)
     if not results:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    results.logDetails()
+    json_result_dir = os.path.join(d.getVar("WORKDIR"),
+                                   'temp',
+                                   'json_testresults-%s' % os.getpid(),
+                                   'runtime',
+                                   machine,
+                                   d.getVar("IMAGE_BASENAME"))
+    results.logDetails(json_result_dir)
     results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
-- 
2.7.4

[PATCH 5/5] testsdk.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA sdk and sdkext need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testsdk.bbclass | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass
index d3f475d..c0f268e 100644
--- a/meta/classes/testsdk.bbclass
+++ b/meta/classes/testsdk.bbclass
@@ -81,7 +81,12 @@ def testsdk_main(d):
         component = "%s %s" % (pn, OESDKTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
 
-        result.logDetails()
+        json_result_dir = os.path.join(d.getVar("WORKDIR"),
+                                       'temp',
+                                       'json_testresults-%s' % os.getpid(),
+                                       'sdk',
+                                       d.getVar("IMAGE_BASENAME"))
+        result.logDetails(json_result_dir)
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():
@@ -185,7 +190,12 @@ def testsdkext_main(d):
         component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
 
-        result.logDetails()
+        json_result_dir = os.path.join(d.getVar("WORKDIR"),
+                                       'temp',
+                                       'json_testresults-%s' % os.getpid(),
+                                       'sdkext',
+                                       d.getVar("IMAGE_BASENAME"))
+        result.logDetails(json_result_dir)
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():
-- 
2.7.4

Re: [PATCH 1/5] oeqa/core/runner: refactor for OEQA to write json testresult

[Richard Purdie]

On Mon, 2018-10-15 at 15:24 +0800, Yeoh Ee Peng wrote:
> Refactor the original _getDetailsNotPassed method to return
> testresult details (test status and log), which will be reused
> by future OEQA code to write json testresult.
> 
> Take the opportunity to consolidate and simplify the logic used
> to gather test status and log within the TestResult instance.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/core/runner.py | 70 ++++++++++++++++++--------------------------
>  1 file changed, 29 insertions(+), 41 deletions(-)
> 
> diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
> index eeb625b..56666ee 100644
> --- a/meta/lib/oeqa/core/runner.py
> +++ b/meta/lib/oeqa/core/runner.py
> @@ -76,40 +76,43 @@ class OETestResult(_TestResult):
>          else:
>              msg = "%s - FAIL - Required tests failed" % component
>          skipped = len(self.skipped)
> -        if skipped: 
> +        if skipped:
>              msg += " (skipped=%d)" % skipped
>          self.tc.logger.info(msg)
>  
> -    def _getDetailsNotPassed(self, case, type, desc):
> -        found = False
> +    def _getTestResultDetails(self, case):
> +        result_types = ['failures', 'errors', 'skipped', 'expectedFailures', 'successes']
> +        result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL', 'PASSED']

This patch is much better thanks! 

It could still do with one more minor tweak. You can use a dict above
to simplfy this code and avoid using indexes which is not very
pythonic, for example


result_types = {'failures' : 'FAILED', 'errors' : 'ERROR'}

then

for rtype in result_types:
    [...]
    return result_types[rtype], msg


The rest of the patch looks like a nice simplification now though!

Cheers,

Richard

Re: [PATCH 1/5] oeqa/core/runner: refactor for OEQA to write json testresult

[Yeoh, Ee Peng]

Hi Richard,

Thank you very much for the sharing! I will make the require enhancement asap.

Thanks,
Yeoh Ee Peng 

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 15, 2018 4:25 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 1/5] oeqa/core/runner: refactor for OEQA to write json testresult

On Mon, 2018-10-15 at 15:24 +0800, Yeoh Ee Peng wrote:
> Refactor the original _getDetailsNotPassed method to return testresult 
> details (test status and log), which will be reused by future OEQA 
> code to write json testresult.
> 
> Take the opportunity to consolidate and simplify the logic used to 
> gather test status and log within the TestResult instance.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/core/runner.py | 70 
> ++++++++++++++++++--------------------------
>  1 file changed, 29 insertions(+), 41 deletions(-)
> 
> diff --git a/meta/lib/oeqa/core/runner.py 
> b/meta/lib/oeqa/core/runner.py index eeb625b..56666ee 100644
> --- a/meta/lib/oeqa/core/runner.py
> +++ b/meta/lib/oeqa/core/runner.py
> @@ -76,40 +76,43 @@ class OETestResult(_TestResult):
>          else:
>              msg = "%s - FAIL - Required tests failed" % component
>          skipped = len(self.skipped)
> -        if skipped: 
> +        if skipped:
>              msg += " (skipped=%d)" % skipped
>          self.tc.logger.info(msg)
>  
> -    def _getDetailsNotPassed(self, case, type, desc):
> -        found = False
> +    def _getTestResultDetails(self, case):
> +        result_types = ['failures', 'errors', 'skipped', 'expectedFailures', 'successes']
> +        result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL', 
> + 'PASSED']

This patch is much better thanks! 

It could still do with one more minor tweak. You can use a dict above to simplfy this code and avoid using indexes which is not very pythonic, for example


result_types = {'failures' : 'FAILED', 'errors' : 'ERROR'}

then

for rtype in result_types:
    [...]
    return result_types[rtype], msg


The rest of the patch looks like a nice simplification now though!

Cheers,

Richard

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[Richard Purdie]

On Mon, 2018-10-15 at 15:24 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult,
> OEQA need to output testresult into json file, where these json
> testresult file will be stored in git repository by the future
> test-case-management tools.
> 
> Both the testresult (eg. PASSED, FAILED, ERROR) and  the test log
> (eg. message from unit test assertion) will be created for storing.
> 
> Also the library class inside this patch will be reused by the future
> test-case-management tools to write json testresult for manual test
> case executed.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/core/runner.py | 64 +++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 63 insertions(+), 1 deletion(-)
> 
> diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
> index 56666ee..56e8526 100644
> --- a/meta/lib/oeqa/core/runner.py
> +++ b/meta/lib/oeqa/core/runner.py
> @@ -6,6 +6,8 @@ import time
>  import unittest
>  import logging
>  import re
> +import json
> +import pathlib
>  
>  from unittest import TextTestResult as _TestResult
>  from unittest import TextTestRunner as _TestRunner
> @@ -119,8 +121,9 @@ class OETestResult(_TestResult):
>          self.successes.append((test, None))
>          super(OETestResult, self).addSuccess(test)
>  
> -    def logDetails(self):
> +    def logDetails(self, json_file_dir=''):

json_file_dir=None

>          self.tc.logger.info("RESULTS:")
> +        results = {}
>          for case_name in self.tc._registry['cases']:
>              case = self.tc._registry['cases'][case_name]
>  
> @@ -137,6 +140,12 @@ class OETestResult(_TestResult):
>                  t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
>  
>              self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t))
> +            results[case.id()] = (status, log)
> +
> +        if len(json_file_dir) > 0:

if json_file_dir:

> +            tresultjsonhelper = OETestResultJSONHelper()
> +            tresultjsonhelper.dump_testresult_file(results, json_file_dir)
> +            tresultjsonhelper.dump_log_files(results, os.path.join(json_file_dir, 'logs'))
>  
>  class OEListTestsResult(object):
>      def wasSuccessful(self):
> @@ -249,3 +258,56 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    def get_testsuite_from_testcase(self, testcase):
> +        testsuite = testcase[0:testcase.rfind(".")]
> +        return testsuite
> +
> +    def get_testsuite_testcase_dictionary(self, testresults):
> +        testsuite_testcase_dict = {}
> +        for testcase in testresults.keys():
> +            testsuite = self.get_testsuite_from_testcase(testcase)
> +            if testsuite in testsuite_testcase_dict:
> +                testsuite_testcase_dict[testsuite].append(testcase)
> +            else:
> +                testsuite_testcase_dict[testsuite] = [testcase]
> +        return testsuite_testcase_dict
> +
> +    def _create_testcase_testresult_object(self, testcase_list, testresults):
> +        testcase_dict = {}
> +        for testcase in sorted(testcase_list):
> +            testcase_dict[testcase] = {"testresult": testresults[testcase][0]}
> +        return testcase_dict
> +
> +    def _create_json_testsuite_string(self, testresults):
> +        testsuite_testcase = self.get_testsuite_testcase_dictionary(testresults)
> +        testsuite_object = {'testsuite': {}}
> +        testsuite_dict = testsuite_object['testsuite']
> +        for testsuite in sorted(testsuite_testcase.keys()):
> +            testsuite_dict[testsuite] = {'testcase': {}}
> +            testsuite_dict[testsuite]['testcase'] = self._create_testcase_testresult_object(
> +                                                        testsuite_testcase[testsuite],
> +                                                        testresults)
> +        return json.dumps(testsuite_object, sort_keys=True, indent=4)

This patch itself is better but I want to make sure we're using the
right data format for this results file. Can you describe the format
you're planning to use for the results?

I think what we may need to do is:

a) Combine the results and log data into a single file

b) Stop trying to split things into testsuites here and do that during
the results processing. Here, we should just be writing the test case
names as they are in the system.

c) Don't rely on the filename to pass information. For example,
logDetails() needs to take a parameter which says whether these are
"runtime", "selftest", "manual" or whatever and that information also
needs to be stored in the json file.

I think if you describe the results format and/or provide a sample
results entry in json format, this will become clearer.



> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, testresults, write_dir):
> +        if not os.path.exists(write_dir):
> +            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)

The above can be moved into _write_file() rather than being duplicated.
Also, can't we use bb.utils.mkdirhier() like the rest of the code? That
can be called unconditionally as it tests internally if it needs to
create the directory.

> +        json_testsuite = self._create_json_testsuite_string(testresults)
> +        self._write_file(write_dir, 'testresults.json', json_testsuite)
> +
> +    def dump_log_files(self, testresults, write_dir):
> +        if not os.path.exists(write_dir):
> +            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)
> +        for testcase in testresults.keys():
> +            test_log = testresults[testcase][1]
> +            if test_log is not None:

Just "if test_log:" should be fine.

> +                file_name = '%s.log' % testcase
> +                self._write_file(write_dir, file_name, test_log)


Cheers,

Richard

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

[-- Attachment #1: Type: text/plain, Size: 7300 bytes --]

Hi Richard,

Previous test result data format contain the hierarchy format (eg. testsuite and testcase) for better human reading, as discussed and agreed that the human readability was not needed, thus I will removed the hierarchy format with testsuite and testcase. 

New data format that I am thinking now. 
Attached sample data format
Two section :
1. Test environments: storing all key-value pair information related to test environments
2. Test cases: storing test result and log
Please let me know your inputs and thoughts. 

Thanks,
Yeoh Ee Peng 

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 15, 2018 4:54 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 2/5] oeqa/core/runner: write testresult to json files

On Mon, 2018-10-15 at 15:24 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult, OEQA 
> need to output testresult into json file, where these json testresult 
> file will be stored in git repository by the future 
> test-case-management tools.
> 
> Both the testresult (eg. PASSED, FAILED, ERROR) and  the test log (eg. 
> message from unit test assertion) will be created for storing.
> 
> Also the library class inside this patch will be reused by the future 
> test-case-management tools to write json testresult for manual test 
> case executed.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/core/runner.py | 64 
> +++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 63 insertions(+), 1 deletion(-)
> 
> diff --git a/meta/lib/oeqa/core/runner.py 
> b/meta/lib/oeqa/core/runner.py index 56666ee..56e8526 100644
> --- a/meta/lib/oeqa/core/runner.py
> +++ b/meta/lib/oeqa/core/runner.py
> @@ -6,6 +6,8 @@ import time
>  import unittest
>  import logging
>  import re
> +import json
> +import pathlib
>  
>  from unittest import TextTestResult as _TestResult  from unittest 
> import TextTestRunner as _TestRunner @@ -119,8 +121,9 @@ class 
> OETestResult(_TestResult):
>          self.successes.append((test, None))
>          super(OETestResult, self).addSuccess(test)
>  
> -    def logDetails(self):
> +    def logDetails(self, json_file_dir=''):

json_file_dir=None

>          self.tc.logger.info("RESULTS:")
> +        results = {}
>          for case_name in self.tc._registry['cases']:
>              case = self.tc._registry['cases'][case_name]
>  
> @@ -137,6 +140,12 @@ class OETestResult(_TestResult):
>                  t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
>  
>              self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % 
> (case.id(), oeid, status, t))
> +            results[case.id()] = (status, log)
> +
> +        if len(json_file_dir) > 0:

if json_file_dir:

> +            tresultjsonhelper = OETestResultJSONHelper()
> +            tresultjsonhelper.dump_testresult_file(results, json_file_dir)
> +            tresultjsonhelper.dump_log_files(results, 
> + os.path.join(json_file_dir, 'logs'))
>  
>  class OEListTestsResult(object):
>      def wasSuccessful(self):
> @@ -249,3 +258,56 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    def get_testsuite_from_testcase(self, testcase):
> +        testsuite = testcase[0:testcase.rfind(".")]
> +        return testsuite
> +
> +    def get_testsuite_testcase_dictionary(self, testresults):
> +        testsuite_testcase_dict = {}
> +        for testcase in testresults.keys():
> +            testsuite = self.get_testsuite_from_testcase(testcase)
> +            if testsuite in testsuite_testcase_dict:
> +                testsuite_testcase_dict[testsuite].append(testcase)
> +            else:
> +                testsuite_testcase_dict[testsuite] = [testcase]
> +        return testsuite_testcase_dict
> +
> +    def _create_testcase_testresult_object(self, testcase_list, testresults):
> +        testcase_dict = {}
> +        for testcase in sorted(testcase_list):
> +            testcase_dict[testcase] = {"testresult": testresults[testcase][0]}
> +        return testcase_dict
> +
> +    def _create_json_testsuite_string(self, testresults):
> +        testsuite_testcase = self.get_testsuite_testcase_dictionary(testresults)
> +        testsuite_object = {'testsuite': {}}
> +        testsuite_dict = testsuite_object['testsuite']
> +        for testsuite in sorted(testsuite_testcase.keys()):
> +            testsuite_dict[testsuite] = {'testcase': {}}
> +            testsuite_dict[testsuite]['testcase'] = self._create_testcase_testresult_object(
> +                                                        testsuite_testcase[testsuite],
> +                                                        testresults)
> +        return json.dumps(testsuite_object, sort_keys=True, indent=4)

This patch itself is better but I want to make sure we're using the right data format for this results file. Can you describe the format you're planning to use for the results?

I think what we may need to do is:

a) Combine the results and log data into a single file

b) Stop trying to split things into testsuites here and do that during the results processing. Here, we should just be writing the test case names as they are in the system.

c) Don't rely on the filename to pass information. For example,
logDetails() needs to take a parameter which says whether these are "runtime", "selftest", "manual" or whatever and that information also needs to be stored in the json file.

I think if you describe the results format and/or provide a sample results entry in json format, this will become clearer.



> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, testresults, write_dir):
> +        if not os.path.exists(write_dir):
> +            pathlib.Path(write_dir).mkdir(parents=True, 
> + exist_ok=True)

The above can be moved into _write_file() rather than being duplicated.
Also, can't we use bb.utils.mkdirhier() like the rest of the code? That can be called unconditionally as it tests internally if it needs to create the directory.

> +        json_testsuite = self._create_json_testsuite_string(testresults)
> +        self._write_file(write_dir, 'testresults.json', 
> + json_testsuite)
> +
> +    def dump_log_files(self, testresults, write_dir):
> +        if not os.path.exists(write_dir):
> +            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)
> +        for testcase in testresults.keys():
> +            test_log = testresults[testcase][1]
> +            if test_log is not None:

Just "if test_log:" should be fine.

> +                file_name = '%s.log' % testcase
> +                self._write_file(write_dir, file_name, test_log)


Cheers,

Richard


[-- Attachment #2: testresults.json --]
[-- Type: application/octet-stream, Size: 1446 bytes --]

{
	"test_environments": {
		"TEST_TYPE" : "auto_runtime",
		"MACHINE" : "qemux86",
		"IMAGE_BASENAME" : "core-image-sato-sdk",
	}
	"testcase": {
		"bblayers.BitbakeLayers.test_bitbakelayers_add_remove": {
			"testresult": "PASSED",
			"logs: ""
		},
		"bblayers.BitbakeLayers.test_bitbakelayers_createlayer": {
			"testresult": "PASSED",
			"logs: ""
		},
		"bblayers.BitbakeLayers.test_bitbakelayers_flatten": {
			"testresult": "PASSED",
			"logs: ""
		},
		"bblayers.BitbakeLayers.test_bitbakelayers_showappends": {
			"testresult": "PASSED",
			"logs: ""
		},
		"bblayers.BitbakeLayers.test_bitbakelayers_showcrossdepends": {
			"testresult": "PASSED",
			"logs: ""
		},
		"bblayers.BitbakeLayers.test_bitbakelayers_showlayers": {
			"testresult": "PASSED",
			"logs: ""
		},
		"bblayers.BitbakeLayers.test_bitbakelayers_showoverlayed": {
			"testresult": "PASSED", 
			"logs: ""
		},
		"bblayers.BitbakeLayers.test_bitbakelayers_showrecipes": {
			"testresult": "FAILED",
			"logs: "File "/data/eyeoh7/tmp/poky_qa/meta/lib/oeqa/core/decorator/__init__.py", line 32, in wrapped_f
    return func(*args, **kwargs)
  File "/data/eyeoh7/tmp/poky_qa/meta/lib/oeqa/selftest/cases/bblayers.py", line 45, in test_bitbakelayers_flatten
    self.assertTrue(os.path.isfile(bb_file+1), msg = "Cannot find xcursor-transparent-theme_0.1.1.bb in the test_bitbakelayers_flatten local dir.")
TypeError: Can't convert 'int' object to str implicitly"
		}
	}

}

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

Hi Richard,

Thank you very much for your great feedbacks and inputs!
I had completed the refactor and resubmitted the patches as below. 
Please let me know if any more feedbacks and inputs!

Thanks,
Yeoh Ee Peng 

[1] http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156835.html
[2] http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156836.html
[3] http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156837.html
[4] http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156838.html
[5] http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156839.html

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 15, 2018 4:54 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 2/5] oeqa/core/runner: write testresult to json files

On Mon, 2018-10-15 at 15:24 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult, OEQA 
> need to output testresult into json file, where these json testresult 
> file will be stored in git repository by the future 
> test-case-management tools.
> 
> Both the testresult (eg. PASSED, FAILED, ERROR) and  the test log (eg. 
> message from unit test assertion) will be created for storing.
> 
> Also the library class inside this patch will be reused by the future 
> test-case-management tools to write json testresult for manual test 
> case executed.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/core/runner.py | 64 
> +++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 63 insertions(+), 1 deletion(-)
> 
> diff --git a/meta/lib/oeqa/core/runner.py 
> b/meta/lib/oeqa/core/runner.py index 56666ee..56e8526 100644
> --- a/meta/lib/oeqa/core/runner.py
> +++ b/meta/lib/oeqa/core/runner.py
> @@ -6,6 +6,8 @@ import time
>  import unittest
>  import logging
>  import re
> +import json
> +import pathlib
>  
>  from unittest import TextTestResult as _TestResult  from unittest 
> import TextTestRunner as _TestRunner @@ -119,8 +121,9 @@ class 
> OETestResult(_TestResult):
>          self.successes.append((test, None))
>          super(OETestResult, self).addSuccess(test)
>  
> -    def logDetails(self):
> +    def logDetails(self, json_file_dir=''):

json_file_dir=None

>          self.tc.logger.info("RESULTS:")
> +        results = {}
>          for case_name in self.tc._registry['cases']:
>              case = self.tc._registry['cases'][case_name]
>  
> @@ -137,6 +140,12 @@ class OETestResult(_TestResult):
>                  t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
>  
>              self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % 
> (case.id(), oeid, status, t))
> +            results[case.id()] = (status, log)
> +
> +        if len(json_file_dir) > 0:

if json_file_dir:

> +            tresultjsonhelper = OETestResultJSONHelper()
> +            tresultjsonhelper.dump_testresult_file(results, json_file_dir)
> +            tresultjsonhelper.dump_log_files(results, 
> + os.path.join(json_file_dir, 'logs'))
>  
>  class OEListTestsResult(object):
>      def wasSuccessful(self):
> @@ -249,3 +258,56 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    def get_testsuite_from_testcase(self, testcase):
> +        testsuite = testcase[0:testcase.rfind(".")]
> +        return testsuite
> +
> +    def get_testsuite_testcase_dictionary(self, testresults):
> +        testsuite_testcase_dict = {}
> +        for testcase in testresults.keys():
> +            testsuite = self.get_testsuite_from_testcase(testcase)
> +            if testsuite in testsuite_testcase_dict:
> +                testsuite_testcase_dict[testsuite].append(testcase)
> +            else:
> +                testsuite_testcase_dict[testsuite] = [testcase]
> +        return testsuite_testcase_dict
> +
> +    def _create_testcase_testresult_object(self, testcase_list, testresults):
> +        testcase_dict = {}
> +        for testcase in sorted(testcase_list):
> +            testcase_dict[testcase] = {"testresult": testresults[testcase][0]}
> +        return testcase_dict
> +
> +    def _create_json_testsuite_string(self, testresults):
> +        testsuite_testcase = self.get_testsuite_testcase_dictionary(testresults)
> +        testsuite_object = {'testsuite': {}}
> +        testsuite_dict = testsuite_object['testsuite']
> +        for testsuite in sorted(testsuite_testcase.keys()):
> +            testsuite_dict[testsuite] = {'testcase': {}}
> +            testsuite_dict[testsuite]['testcase'] = self._create_testcase_testresult_object(
> +                                                        testsuite_testcase[testsuite],
> +                                                        testresults)
> +        return json.dumps(testsuite_object, sort_keys=True, indent=4)

This patch itself is better but I want to make sure we're using the right data format for this results file. Can you describe the format you're planning to use for the results?

I think what we may need to do is:

a) Combine the results and log data into a single file

b) Stop trying to split things into testsuites here and do that during the results processing. Here, we should just be writing the test case names as they are in the system.

c) Don't rely on the filename to pass information. For example,
logDetails() needs to take a parameter which says whether these are "runtime", "selftest", "manual" or whatever and that information also needs to be stored in the json file.

I think if you describe the results format and/or provide a sample results entry in json format, this will become clearer.



> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, testresults, write_dir):
> +        if not os.path.exists(write_dir):
> +            pathlib.Path(write_dir).mkdir(parents=True, 
> + exist_ok=True)

The above can be moved into _write_file() rather than being duplicated.
Also, can't we use bb.utils.mkdirhier() like the rest of the code? That can be called unconditionally as it tests internally if it needs to create the directory.

> +        json_testsuite = self._create_json_testsuite_string(testresults)
> +        self._write_file(write_dir, 'testresults.json', 
> + json_testsuite)
> +
> +    def dump_log_files(self, testresults, write_dir):
> +        if not os.path.exists(write_dir):
> +            pathlib.Path(write_dir).mkdir(parents=True, exist_ok=True)
> +        for testcase in testresults.keys():
> +            test_log = testresults[testcase][1]
> +            if test_log is not None:

Just "if test_log:" should be fine.

> +                file_name = '%s.log' % testcase
> +                self._write_file(write_dir, file_name, test_log)


Cheers,

Richard

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[richard.purdie]

On Thu, 2018-10-18 at 09:47 +0000, Yeoh, Ee Peng wrote:
> Thank you very much for your great feedbacks and inputs!
> I had completed the refactor and resubmitted the patches as below. 
> Please let me know if any more feedbacks and inputs!

Thanks for the revised series, its looking very much improved! I've
queued 1/5 onto the autobuilder and its tested ok so that will likely
merge.

I still have a few concerns about the data format for 2/5.

@@ -249,3 +256,24 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    def _create_json_testresult_string(self, test_results, test_environments):
> +        testcase_dict = {}
> +        for testcase in sorted(test_results):
> +            testcase_dict[testcase] = {"testresult": test_results[testcase][0], "log": test_results[testcase][1]}
> +        testresult_object = {'testenvironment': test_environments,
> +                             'testcase': testcase_dict}

I'd like to be able to store more than one set of results in a single
json file. In order to make that work, the above format will need
tweaking slightly. Something like:

testresult_object = {resultid : {'configuration': test_environments,
                                  'results': testcase_dict}}

where resultid would be something like "imagetest-qemux86-core-image-sato-YYMMDDHHMMSS" 
and passed in to the function by the call site. It'd be similar to the
other log file the tests generate. We could also perhaps use pid
instead of a timestamp in there. A timestamp would be good in the
configuration section.

Also, we need to record which revision of the configured layers this
test result was with. The get_layers_branch_rev() function in
base.bbclass should give an idea of how to get this information using
existing functions.

> +        return json.dumps(testresult_object, sort_keys=True, indent=4)
> +
> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, test_results, test_environments, write_dir):
> +        if not os.path.exists(write_dir):

No need for this if test here since mkdirhier handles that.

> +            bb.utils.mkdirhier(write_dir)
> +        json_testresult = self._create_json_testresult_string(test_results, test_environments)
> +        self._write_file(write_dir, 'testresults.json', json_testresult)

As mentioned above, I'd really like if if we could put all the test
results into one json file as this would save the autobuilder having to
combine them all. That should be straight forward to do but we need to
be careful about races.

This means locking a lockfile alongside the results file which we can
do with bb.utils.lockfile() call, then writing the results, then
calling unlockfile().

With regard to the other patches, for selftest we do need to log which
MACHINE/DISTRO was used.

DISTRO and IMAGE_PKGTYPE also needs adding to testimage and SDKMACHINE
and IMAGE_PKGTYPE to testsdk.

Cheers,

Richard

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

Hi Richard,

Thank you for your inputs! 
I am now working on the new features that you requested. 

Best regards,
Yeoh Ee Peng 

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Friday, October 19, 2018 5:56 AM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 2/5] oeqa/core/runner: write testresult to json files

On Thu, 2018-10-18 at 09:47 +0000, Yeoh, Ee Peng wrote:
> Thank you very much for your great feedbacks and inputs!
> I had completed the refactor and resubmitted the patches as below. 
> Please let me know if any more feedbacks and inputs!

Thanks for the revised series, its looking very much improved! I've queued 1/5 onto the autobuilder and its tested ok so that will likely merge.

I still have a few concerns about the data format for 2/5.

@@ -249,3 +256,24 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    def _create_json_testresult_string(self, test_results, test_environments):
> +        testcase_dict = {}
> +        for testcase in sorted(test_results):
> +            testcase_dict[testcase] = {"testresult": test_results[testcase][0], "log": test_results[testcase][1]}
> +        testresult_object = {'testenvironment': test_environments,
> +                             'testcase': testcase_dict}

I'd like to be able to store more than one set of results in a single json file. In order to make that work, the above format will need tweaking slightly. Something like:

testresult_object = {resultid : {'configuration': test_environments,
                                  'results': testcase_dict}}

where resultid would be something like "imagetest-qemux86-core-image-sato-YYMMDDHHMMSS" 
and passed in to the function by the call site. It'd be similar to the other log file the tests generate. We could also perhaps use pid instead of a timestamp in there. A timestamp would be good in the configuration section.

Also, we need to record which revision of the configured layers this test result was with. The get_layers_branch_rev() function in base.bbclass should give an idea of how to get this information using existing functions.

> +        return json.dumps(testresult_object, sort_keys=True, 
> + indent=4)
> +
> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, test_results, test_environments, write_dir):
> +        if not os.path.exists(write_dir):

No need for this if test here since mkdirhier handles that.

> +            bb.utils.mkdirhier(write_dir)
> +        json_testresult = self._create_json_testresult_string(test_results, test_environments)
> +        self._write_file(write_dir, 'testresults.json', 
> + json_testresult)

As mentioned above, I'd really like if if we could put all the test results into one json file as this would save the autobuilder having to combine them all. That should be straight forward to do but we need to be careful about races.

This means locking a lockfile alongside the results file which we can do with bb.utils.lockfile() call, then writing the results, then calling unlockfile().

With regard to the other patches, for selftest we do need to log which MACHINE/DISTRO was used.

DISTRO and IMAGE_PKGTYPE also needs adding to testimage and SDKMACHINE and IMAGE_PKGTYPE to testsdk.

Cheers,

Richard

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

Hi Richard,

I had done making the changes to match your inputs and suggestions.
Below was the patches resubmitted. 

Please let me know if you have any more inputs or questions.

Best regards,
Yeoh Ee Peng 

http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156926.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156927.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156928.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156929.html


-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Friday, October 19, 2018 5:56 AM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 2/5] oeqa/core/runner: write testresult to json files

On Thu, 2018-10-18 at 09:47 +0000, Yeoh, Ee Peng wrote:
> Thank you very much for your great feedbacks and inputs!
> I had completed the refactor and resubmitted the patches as below. 
> Please let me know if any more feedbacks and inputs!

Thanks for the revised series, its looking very much improved! I've queued 1/5 onto the autobuilder and its tested ok so that will likely merge.

I still have a few concerns about the data format for 2/5.

@@ -249,3 +256,24 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    def _create_json_testresult_string(self, test_results, test_environments):
> +        testcase_dict = {}
> +        for testcase in sorted(test_results):
> +            testcase_dict[testcase] = {"testresult": test_results[testcase][0], "log": test_results[testcase][1]}
> +        testresult_object = {'testenvironment': test_environments,
> +                             'testcase': testcase_dict}

I'd like to be able to store more than one set of results in a single json file. In order to make that work, the above format will need tweaking slightly. Something like:

testresult_object = {resultid : {'configuration': test_environments,
                                  'results': testcase_dict}}

where resultid would be something like "imagetest-qemux86-core-image-sato-YYMMDDHHMMSS" 
and passed in to the function by the call site. It'd be similar to the other log file the tests generate. We could also perhaps use pid instead of a timestamp in there. A timestamp would be good in the configuration section.

Also, we need to record which revision of the configured layers this test result was with. The get_layers_branch_rev() function in base.bbclass should give an idea of how to get this information using existing functions.

> +        return json.dumps(testresult_object, sort_keys=True, 
> + indent=4)
> +
> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, test_results, test_environments, write_dir):
> +        if not os.path.exists(write_dir):

No need for this if test here since mkdirhier handles that.

> +            bb.utils.mkdirhier(write_dir)
> +        json_testresult = self._create_json_testresult_string(test_results, test_environments)
> +        self._write_file(write_dir, 'testresults.json', 
> + json_testresult)

As mentioned above, I'd really like if if we could put all the test results into one json file as this would save the autobuilder having to combine them all. That should be straight forward to do but we need to be careful about races.

This means locking a lockfile alongside the results file which we can do with bb.utils.lockfile() call, then writing the results, then calling unlockfile().

With regard to the other patches, for selftest we do need to log which MACHINE/DISTRO was used.

DISTRO and IMAGE_PKGTYPE also needs adding to testimage and SDKMACHINE and IMAGE_PKGTYPE to testsdk.

Cheers,

Richard

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

Hi Richard,

Please ignore the two patches below for testimage.bbclass and testsdk.bbclass, initially testing successfully when running these patch separately but I am finding one issue with method namespace collision after testing both patches together.  I am in progress of resolving it. Sorry for the inconvenience.

http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156928.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156929.html

Best regards,
Yeoh Ee Peng 

-----Original Message-----
From: Yeoh, Ee Peng 
Sent: Monday, October 22, 2018 3:23 PM
To: 'richard.purdie@linuxfoundation.org' <richard.purdie@linuxfoundation.org>; openembedded-core@lists.openembedded.org
Subject: RE: [OE-core] [PATCH 2/5] oeqa/core/runner: write testresult to json files

Hi Richard,

I had done making the changes to match your inputs and suggestions.
Below was the patches resubmitted. 

Please let me know if you have any more inputs or questions.

Best regards,
Yeoh Ee Peng 

http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156926.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156927.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156928.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156929.html


-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Friday, October 19, 2018 5:56 AM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 2/5] oeqa/core/runner: write testresult to json files

On Thu, 2018-10-18 at 09:47 +0000, Yeoh, Ee Peng wrote:
> Thank you very much for your great feedbacks and inputs!
> I had completed the refactor and resubmitted the patches as below. 
> Please let me know if any more feedbacks and inputs!

Thanks for the revised series, its looking very much improved! I've queued 1/5 onto the autobuilder and its tested ok so that will likely merge.

I still have a few concerns about the data format for 2/5.

@@ -249,3 +256,24 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    def _create_json_testresult_string(self, test_results, test_environments):
> +        testcase_dict = {}
> +        for testcase in sorted(test_results):
> +            testcase_dict[testcase] = {"testresult": test_results[testcase][0], "log": test_results[testcase][1]}
> +        testresult_object = {'testenvironment': test_environments,
> +                             'testcase': testcase_dict}

I'd like to be able to store more than one set of results in a single json file. In order to make that work, the above format will need tweaking slightly. Something like:

testresult_object = {resultid : {'configuration': test_environments,
                                  'results': testcase_dict}}

where resultid would be something like "imagetest-qemux86-core-image-sato-YYMMDDHHMMSS" 
and passed in to the function by the call site. It'd be similar to the other log file the tests generate. We could also perhaps use pid instead of a timestamp in there. A timestamp would be good in the configuration section.

Also, we need to record which revision of the configured layers this test result was with. The get_layers_branch_rev() function in base.bbclass should give an idea of how to get this information using existing functions.

> +        return json.dumps(testresult_object, sort_keys=True, 
> + indent=4)
> +
> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, test_results, test_environments, write_dir):
> +        if not os.path.exists(write_dir):

No need for this if test here since mkdirhier handles that.

> +            bb.utils.mkdirhier(write_dir)
> +        json_testresult = self._create_json_testresult_string(test_results, test_environments)
> +        self._write_file(write_dir, 'testresults.json', 
> + json_testresult)

As mentioned above, I'd really like if if we could put all the test results into one json file as this would save the autobuilder having to combine them all. That should be straight forward to do but we need to be careful about races.

This means locking a lockfile alongside the results file which we can do with bb.utils.lockfile() call, then writing the results, then calling unlockfile().

With regard to the other patches, for selftest we do need to log which MACHINE/DISTRO was used.

DISTRO and IMAGE_PKGTYPE also needs adding to testimage and SDKMACHINE and IMAGE_PKGTYPE to testsdk.

Cheers,

Richard

Re: [PATCH 2/5] oeqa/core/runner: write testresult to json files

[richard.purdie]

On Mon, 2018-10-22 at 08:17 +0000, Yeoh, Ee Peng wrote:
> Please ignore the two patches below for testimage.bbclass and
> testsdk.bbclass, initially testing successfully when running these
> patch separately but I am finding one issue with method namespace
> collision after testing both patches together.  I am in progress of
> resolving it. Sorry for the inconvenience.

I think if your code appends to the results file, things should work
better and we can remove the split level directory structure. Merging
the results file will help the autobuiler process the results.

Cheers,

Richard

[PATCH 2/5] oeqa/core/runner: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA need to output testresult into json file, where these json
testresult file will be stored in git repository by the future
test-case-management tools.

Both the testresult (eg. PASSED, FAILED, ERROR) and  the test log
(eg. message from unit test assertion) will be created for storing.

Also the library class inside this patch will be reused by the future
test-case-management tools to write json testresult for manual test
case executed.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/core/runner.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index f1dd080..e82acdf 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -6,6 +6,7 @@ import time
 import unittest
 import logging
 import re
+import json
 
 from unittest import TextTestResult as _TestResult
 from unittest import TextTestRunner as _TestRunner
@@ -119,8 +120,9 @@ class OETestResult(_TestResult):
         self.successes.append((test, None))
         super(OETestResult, self).addSuccess(test)
 
-    def logDetails(self):
+    def logDetails(self, json_file_dir=None, test_environments=None):
         self.tc.logger.info("RESULTS:")
+        results = {}
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
@@ -137,6 +139,11 @@ class OETestResult(_TestResult):
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
 
             self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t))
+            results[case.id()] = (status, log)
+
+        if json_file_dir:
+            tresultjsonhelper = OETestResultJSONHelper()
+            tresultjsonhelper.dump_testresult_file(results, test_environments, json_file_dir)
 
 class OEListTestsResult(object):
     def wasSuccessful(self):
@@ -249,3 +256,24 @@ class OETestRunner(_TestRunner):
             self._list_tests_module(suite)
 
         return OEListTestsResult()
+
+class OETestResultJSONHelper(object):
+
+    def _create_json_testresult_string(self, test_results, test_environments):
+        testcase_dict = {}
+        for testcase in sorted(test_results):
+            testcase_dict[testcase] = {"testresult": test_results[testcase][0], "log": test_results[testcase][1]}
+        testresult_object = {'testenvironment': test_environments,
+                             'testcase': testcase_dict}
+        return json.dumps(testresult_object, sort_keys=True, indent=4)
+
+    def _write_file(self, write_dir, file_name, file_content):
+        file_path = os.path.join(write_dir, file_name)
+        with open(file_path, 'w') as the_file:
+            the_file.write(file_content)
+
+    def dump_testresult_file(self, test_results, test_environments, write_dir):
+        if not os.path.exists(write_dir):
+            bb.utils.mkdirhier(write_dir)
+        json_testresult = self._create_json_testresult_string(test_results, test_environments)
+        self._write_file(write_dir, 'testresults.json', json_testresult)
-- 
2.7.4