[PATCH 1/4] oeqa/core/runner: write testresult to json files

[PATCH 1/4] oeqa/core/runner: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA need to output testresult into single json file, where json
testresult file will be stored in git repository by the future
test-case-management tools.

The json testresult file will store more than one set of results,
where each set of results was uniquely identified by the result_id.
The result_id would be like "runtime-qemux86-core-image-sato", where
it was a runtime test with target machine equal to qemux86 and running
on core-image-sato image. The json testresult file will only store
the latest testresult for a given result_id. The json testresult
file contains the configuration (eg. COMMIT, BRANCH, MACHINE, IMAGE),
result (eg. PASSED, FAILED, ERROR), test log, and result_id.

Based on the destination json testresult file directory provided,
it could have multiple instances of bitbake trying to write json
testresult to a single testresult file, using locking a lockfile
alongside the results file directory to prevent races.

Also the library class inside this patch will be reused by the future
test-case-management tools to write json testresult for manual test
case executed.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/core/runner.py | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index f1dd080..82463cf 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -6,6 +6,7 @@ import time
 import unittest
 import logging
 import re
+import json
 
 from unittest import TextTestResult as _TestResult
 from unittest import TextTestRunner as _TestRunner
@@ -119,8 +120,9 @@ class OETestResult(_TestResult):
         self.successes.append((test, None))
         super(OETestResult, self).addSuccess(test)
 
-    def logDetails(self):
+    def logDetails(self, json_file_dir=None, configuration=None, result_id=None):
         self.tc.logger.info("RESULTS:")
+        result = {}
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
@@ -137,6 +139,11 @@ class OETestResult(_TestResult):
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
 
             self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t))
+            result[case.id()] = {'status': status, 'log': log}
+
+        if json_file_dir:
+            tresultjsonhelper = OETestResultJSONHelper()
+            tresultjsonhelper.dump_testresult_file(result_id, result, configuration, json_file_dir)
 
 class OEListTestsResult(object):
     def wasSuccessful(self):
@@ -249,3 +256,34 @@ class OETestRunner(_TestRunner):
             self._list_tests_module(suite)
 
         return OEListTestsResult()
+
+class OETestResultJSONHelper(object):
+
+    testresult_filename = 'testresults.json'
+
+    def _get_testresults(self, write_dir):
+        testresults = {}
+        file = os.path.join(write_dir, self.testresult_filename)
+        if os.path.exists(file):
+            with open(file, "r") as f:
+                testresults = json.load(f)
+        return testresults
+
+    def _create_json_testresults_string(self, result_id, test_result, configuration, write_dir):
+        testresults = self._get_testresults(write_dir)
+        testresult = {'configuration': configuration,
+                      'result': test_result}
+        testresults[result_id] = testresult
+        return json.dumps(testresults, sort_keys=True, indent=4)
+
+    def _write_file(self, write_dir, file_name, file_content):
+        file_path = os.path.join(write_dir, file_name)
+        with open(file_path, 'w') as the_file:
+            the_file.write(file_content)
+
+    def dump_testresult_file(self, result_id, test_result, configuration, write_dir):
+        bb.utils.mkdirhier(write_dir)
+        lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock'))
+        json_testresults = self._create_json_testresults_string(result_id, test_result, configuration, write_dir)
+        self._write_file(write_dir, self.testresult_filename, json_testresults)
+        bb.utils.unlockfile(lf)
-- 
2.7.4

[PATCH 2/4] oeqa/selftest/context: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA selftest need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

To configure multiple instances of bitbake to write json testresult
to a single testresult file, user will define the variable
"OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/context.py | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index c78947e..e053183 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -99,8 +99,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         return cases_paths
 
     def _process_args(self, logger, args):
-        args.output_log = '%s-results-%s.log' % (self.name,
-                time.strftime("%Y%m%d%H%M%S"))
+        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
+        args.output_log = '%s-results-%s.log' % (self.name, args.test_start_time)
         args.test_data_file = None
         args.CASES_PATHS = None
 
@@ -204,6 +204,32 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable")
         runCmd("bitbake -e")
 
+    def _get_json_result_dir(self, args):
+        json_result_dir = os.path.join(os.path.dirname(os.path.abspath(args.output_log)),
+                                       'json_testresults-%s' % args.test_start_time,
+                                       'oe-selftest')
+        if "OEQA_JSON_RESULT_COMMON_DIR" in self.tc.td:
+            json_result_dir = self.tc.td["OEQA_JSON_RESULT_COMMON_DIR"]
+
+        return json_result_dir
+
+    def _get_configuration(self, args):
+        import platform
+        from oeqa.utils.metadata import metadata_from_bb
+
+        metadata = metadata_from_bb()
+        return {'TEST_TYPE': 'oeselftest',
+                'BRANCH': metadata['layers']['meta']['branch'],
+                'COMMIT': metadata['layers']['meta']['commit'],
+                'START_TIME': args.test_start_time,
+                'MACHINE': self.tc.td["MACHINE"],
+                'HOST_DISTRO': platform.linux_distribution(),
+                'HOST_NAME': metadata['hostname']}
+
+    def _get_result_id(self, configuration):
+        distro = '_'.join(configuration['HOST_DISTRO'])
+        return '%s-%s-%s' % (configuration['TEST_TYPE'], distro, configuration['MACHINE'])
+
     def _internal_run(self, logger, args):
         self.module_paths = self._get_cases_paths(
                 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
@@ -220,7 +246,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         else:
             self._pre_run()
             rc = self.tc.runTests(**self.tc_kwargs['run'])
-            rc.logDetails()
+            configuration = self._get_configuration(args)
+            rc.logDetails(self._get_json_result_dir(args),
+                          configuration,
+                          self._get_result_id(configuration))
             rc.logSummary(self.name)
 
         return rc
-- 
2.7.4

[PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA testimage need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

To configure multiple instances of bitbake to write json testresult
to a single testresult file, user will define the variable
"OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testimage.bbclass | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 2642a72..112ba71 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -2,7 +2,7 @@
 #
 # Released under the MIT license (see COPYING.MIT)
 
-
+inherit metadata_scm
 # testimage.bbclass enables testing of qemu images using python unittests.
 # Most of the tests are commands run on target image over ssh.
 # To use it add testimage to global inherit and call your target image with -c testimage
@@ -141,6 +141,33 @@ def testimage_sanity(d):
         bb.fatal('When TEST_TARGET is set to "simpleremote" '
                  'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
 
+def _get_configuration(d, test_type, pid, machine):
+    import platform
+    bb_core_dir = d.getVar("BBFILE_PATTERN_core=")
+    return {'TEST_TYPE': test_type,
+            'BRANCH': base_get_metadata_git_branch(bb_core_dir, None).strip(),
+            'COMMIT': base_get_metadata_git_revision(bb_core_dir, None),
+            'PROCESS_ID': pid,
+            'MACHINE': machine,
+            'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+            'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+            'HOST_DISTRO': platform.linux_distribution()}
+
+def _get_json_result_dir(d, configuration):
+    json_result_dir = os.path.join(d.getVar("WORKDIR"),
+                                   'temp',
+                                   'json_testresults-%s' % configuration['PROCESS_ID'],
+                                   configuration['TEST_TYPE'],
+                                   configuration['MACHINE'],
+                                   configuration['IMAGE_BASENAME'])
+    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
+    if oeqa_json_result_common_dir:
+        json_result_dir = oeqa_json_result_common_dir
+    return json_result_dir
+
+def _get_result_id(configuration):
+    return '%s-%s-%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['MACHINE'])
+
 def testimage_main(d):
     import os
     import json
@@ -308,7 +335,10 @@ def testimage_main(d):
     # Show results (if we have them)
     if not results:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    results.logDetails()
+    configuration = _get_configuration(d, 'runtime', os.getpid(), machine)
+    results.logDetails(_get_json_result_dir(d, configuration),
+                       configuration,
+                       _get_result_id(configuration))
     results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
-- 
2.7.4

[PATCH 4/4] testsdk.bbclass: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA sdk and sdkext need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

To configure multiple instances of bitbake to write json testresult
to a single testresult file, user will define the variable
"OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/classes/testsdk.bbclass | 40 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass
index d3f475d..c4a0f37 100644
--- a/meta/classes/testsdk.bbclass
+++ b/meta/classes/testsdk.bbclass
@@ -2,6 +2,7 @@
 #
 # Released under the MIT license (see COPYING.MIT)
 
+inherit metadata_scm
 # testsdk.bbclass enables testing for SDK and Extensible SDK
 #
 # To run SDK tests, run the commands:
@@ -14,6 +15,33 @@
 #
 # where "<image-name>" is an image like core-image-sato.
 
+def _get_configuration(d, test_type, pid):
+    import platform
+    bb_core_dir = d.getVar("BBFILE_PATTERN_core=")
+    return {'TEST_TYPE': test_type,
+            'BRANCH': base_get_metadata_git_branch(bb_core_dir, None).strip(),
+            'COMMIT': base_get_metadata_git_revision(bb_core_dir, None),
+            'PROCESS_ID': pid,
+            'SDK_MACHINE': d.getVar("SDKMACHINE"),
+            'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+            'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+            'HOST_DISTRO': platform.linux_distribution()}
+
+def _get_json_result_dir(d, configuration):
+    json_result_dir = os.path.join(d.getVar("WORKDIR"),
+                                   'temp',
+                                   'json_testresults-%s' % configuration['PROCESS_ID'],
+                                   configuration['TEST_TYPE'],
+                                   configuration['SDK_MACHINE'],
+                                   configuration['IMAGE_BASENAME'])
+    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
+    if oeqa_json_result_common_dir:
+        json_result_dir = oeqa_json_result_common_dir
+    return json_result_dir
+
+def _get_result_id(configuration):
+    return '%s-%s-%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['SDK_MACHINE'])
+
 def testsdk_main(d):
     import os
     import subprocess
@@ -80,8 +108,10 @@ def testsdk_main(d):
 
         component = "%s %s" % (pn, OESDKTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
-
-        result.logDetails()
+        configuration = _get_configuration(d, 'sdk', os.getpid())
+        result.logDetails(_get_json_result_dir(d, configuration),
+                           configuration,
+                           _get_result_id(configuration))
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():
@@ -184,8 +214,10 @@ def testsdkext_main(d):
 
         component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
-
-        result.logDetails()
+        configuration = _get_configuration(d, 'sdkext', os.getpid())
+        result.logDetails(_get_json_result_dir(d, configuration),
+                           configuration,
+                           _get_result_id(configuration))
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():
-- 
2.7.4

Re: [PATCH 1/4] oeqa/core/runner: write testresult to json files

[Richard Purdie]

Hi Ee Peng,

Thanks, this is looking good, there is still one small tweak needed
below.

On Mon, 2018-10-22 at 14:54 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult,
> OEQA need to output testresult into single json file, where json
> testresult file will be stored in git repository by the future
> test-case-management tools.
> 
> The json testresult file will store more than one set of results,
> where each set of results was uniquely identified by the result_id.
> The result_id would be like "runtime-qemux86-core-image-sato", where
> it was a runtime test with target machine equal to qemux86 and running
> on core-image-sato image. The json testresult file will only store
> the latest testresult for a given result_id. The json testresult
> file contains the configuration (eg. COMMIT, BRANCH, MACHINE, IMAGE),
> result (eg. PASSED, FAILED, ERROR), test log, and result_id.
> 
> Based on the destination json testresult file directory provided,
> it could have multiple instances of bitbake trying to write json
> testresult to a single testresult file, using locking a lockfile
> alongside the results file directory to prevent races.
> 
> Also the library class inside this patch will be reused by the future
> test-case-management tools to write json testresult for manual test
> case executed.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/core/runner.py | 40 +++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 39 insertions(+), 1 deletion(-)
> 
> diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
> index f1dd080..82463cf 100644
> --- a/meta/lib/oeqa/core/runner.py
> +++ b/meta/lib/oeqa/core/runner.py
> @@ -249,3 +256,34 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    testresult_filename = 'testresults.json'
> +
> +    def _get_testresults(self, write_dir):
> +        testresults = {}
> +        file = os.path.join(write_dir, self.testresult_filename)
> +        if os.path.exists(file):
> +            with open(file, "r") as f:
> +                testresults = json.load(f)
> +        return testresults
> +
> +    def _create_json_testresults_string(self, result_id, test_result, configuration, write_dir):
> +        testresults = self._get_testresults(write_dir)
> +        testresult = {'configuration': configuration,
> +                      'result': test_result}
> +        testresults[result_id] = testresult
> +        return json.dumps(testresults, sort_keys=True, indent=4)
> +
> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, result_id, test_result, configuration, write_dir):
> +        bb.utils.mkdirhier(write_dir)
> +        lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock'))
> +        json_testresults = self._create_json_testresults_string(result_id, test_result, configuration, write_dir)
> +        self._write_file(write_dir, self.testresult_filename, json_testresults)
> +        bb.utils.unlockfile(lf)

Before we write out the file we need to load in any existing data so we
effectively append to the data. I think if we do that this patch should
be ready to merge.

I did also wonder if we need a common configuration function rather
than duplicating the code into each of the test classes.

Cheers,

Richard

Re: [PATCH 3/4] testimage.bbclass: write testresult to json files

[Richard Purdie]

On Mon, 2018-10-22 at 14:54 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult,
> OEQA testimage need to output testresult into json files, where
> these json testresult files will be stored into git repository
> by the future test-case-management tools.
> 
> To configure multiple instances of bitbake to write json testresult
> to a single testresult file, user will define the variable
> "OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing
> json testresult.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/classes/testimage.bbclass | 34 ++++++++++++++++++++++++++++++++--
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
> index 2642a72..112ba71 100644
> --- a/meta/classes/testimage.bbclass
> +++ b/meta/classes/testimage.bbclass
> @@ -2,7 +2,7 @@
>  #
>  # Released under the MIT license (see COPYING.MIT)
>  
> -
> +inherit metadata_scm
>  # testimage.bbclass enables testing of qemu images using python unittests.
>  # Most of the tests are commands run on target image over ssh.
>  # To use it add testimage to global inherit and call your target image with -c testimage
> @@ -141,6 +141,33 @@ def testimage_sanity(d):
>          bb.fatal('When TEST_TARGET is set to "simpleremote" '
>                   'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
>  
> +def _get_configuration(d, test_type, pid, machine):
> +    import platform
> +    bb_core_dir = d.getVar("BBFILE_PATTERN_core=")
> +    return {'TEST_TYPE': test_type,
> +            'BRANCH': base_get_metadata_git_branch(bb_core_dir, None).strip(),
> +            'COMMIT': base_get_metadata_git_revision(bb_core_dir, None),

We need to record all the revisions/branches that are being used. Could
we just inject the contents of metadata['layers'] directly into the
json file here?

> +            'PROCESS_ID': pid,
> +            'MACHINE': machine,
> +            'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
> +            'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
> +            'HOST_DISTRO': platform.linux_distribution()}
> +
> +def _get_json_result_dir(d, configuration):
> +    json_result_dir = os.path.join(d.getVar("WORKDIR"),
> +                                   'temp',
> +                                   'json_testresults-%s' % configuration['PROCESS_ID'],
> +                                   configuration['TEST_TYPE'],
> +                                   configuration['MACHINE'],
> +                                   configuration['IMAGE_BASENAME'])
> +    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
> +    if oeqa_json_result_common_dir:
> +        json_result_dir = oeqa_json_result_common_dir

Do we need both codepaths here or can we just place things in
OEQA_JSON_RESULT_COMMON_DIR unconditionally here?

(we could rename it to OEQA_JSON_RESULT_DIR?)

Cheers,

Richard

Re: [PATCH 1/4] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

Hi Richard

Current codes does load existing testresult json file if it exist, then it will write the new testresult into it based on the result_id.
> +    def _get_testresults(self, write_dir):
> +        testresults = {}
> +        file = os.path.join(write_dir, self.testresult_filename)
> +        if os.path.exists(file):
> +            with open(file, "r") as f:
> +                testresults = json.load(f)
> +        return testresults

I did have the same thinking on if we can have a common function to manage configuration and result_id or let individual test classes to manage it, in the end, the thinking was configuration/result_id were really responsibility of each test classes, where the json helper class inside runner shall not have the knowledge or know-how on configuration/result_id. Thus the decision in the end was to make json helper class responsibility as simple as to consume the configuration, results, result_id information provided by the test classes. Hope this explain the reason behind current design. 

Thank you very much for your attention & knowledge sharing! 

Thanks,
Yeoh Ee Peng

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 22, 2018 4:32 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 1/4] oeqa/core/runner: write testresult to json files

Hi Ee Peng,

Thanks, this is looking good, there is still one small tweak needed below.

On Mon, 2018-10-22 at 14:54 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult, OEQA 
> need to output testresult into single json file, where json testresult 
> file will be stored in git repository by the future 
> test-case-management tools.
> 
> The json testresult file will store more than one set of results, 
> where each set of results was uniquely identified by the result_id.
> The result_id would be like "runtime-qemux86-core-image-sato", where 
> it was a runtime test with target machine equal to qemux86 and running 
> on core-image-sato image. The json testresult file will only store the 
> latest testresult for a given result_id. The json testresult file 
> contains the configuration (eg. COMMIT, BRANCH, MACHINE, IMAGE), 
> result (eg. PASSED, FAILED, ERROR), test log, and result_id.
> 
> Based on the destination json testresult file directory provided, it 
> could have multiple instances of bitbake trying to write json 
> testresult to a single testresult file, using locking a lockfile 
> alongside the results file directory to prevent races.
> 
> Also the library class inside this patch will be reused by the future 
> test-case-management tools to write json testresult for manual test 
> case executed.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/core/runner.py | 40 
> +++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 39 insertions(+), 1 deletion(-)
> 
> diff --git a/meta/lib/oeqa/core/runner.py 
> b/meta/lib/oeqa/core/runner.py index f1dd080..82463cf 100644
> --- a/meta/lib/oeqa/core/runner.py
> +++ b/meta/lib/oeqa/core/runner.py
> @@ -249,3 +256,34 @@ class OETestRunner(_TestRunner):
>              self._list_tests_module(suite)
>  
>          return OEListTestsResult()
> +
> +class OETestResultJSONHelper(object):
> +
> +    testresult_filename = 'testresults.json'
> +
> +    def _get_testresults(self, write_dir):
> +        testresults = {}
> +        file = os.path.join(write_dir, self.testresult_filename)
> +        if os.path.exists(file):
> +            with open(file, "r") as f:
> +                testresults = json.load(f)
> +        return testresults
> +
> +    def _create_json_testresults_string(self, result_id, test_result, configuration, write_dir):
> +        testresults = self._get_testresults(write_dir)
> +        testresult = {'configuration': configuration,
> +                      'result': test_result}
> +        testresults[result_id] = testresult
> +        return json.dumps(testresults, sort_keys=True, indent=4)
> +
> +    def _write_file(self, write_dir, file_name, file_content):
> +        file_path = os.path.join(write_dir, file_name)
> +        with open(file_path, 'w') as the_file:
> +            the_file.write(file_content)
> +
> +    def dump_testresult_file(self, result_id, test_result, configuration, write_dir):
> +        bb.utils.mkdirhier(write_dir)
> +        lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock'))
> +        json_testresults = self._create_json_testresults_string(result_id, test_result, configuration, write_dir)
> +        self._write_file(write_dir, self.testresult_filename, json_testresults)
> +        bb.utils.unlockfile(lf)

Before we write out the file we need to load in any existing data so we effectively append to the data. I think if we do that this patch should be ready to merge.

I did also wonder if we need a common configuration function rather than duplicating the code into each of the test classes.

Cheers,

Richard

Re: [PATCH 3/4] testimage.bbclass: write testresult to json files

[Yeoh, Ee Peng]

Richard,

Noted, let's us made the changes to record all the revisions/branches that are being used.

The reason to provide the default result_dir here was to prevent the case where no json testresult being write out when user does not provide the specific OEQA_JSON_RESULT_DIR, otherwise user might need to rerun test to get the json testresult.  Probably, it will be better that bitbake will prompt user to provide result_dir variable next round if it was not avaiable to write testresult to a specify location (eg. common location to write all testresult). Please let me know your inputs.
> +def _get_json_result_dir(d, configuration):
> +    json_result_dir = os.path.join(d.getVar("WORKDIR"),
> +                                   'temp',
> +                                   'json_testresults-%s' % configuration['PROCESS_ID'],
> +                                   configuration['TEST_TYPE'],
> +                                   configuration['MACHINE'],
> +                                   configuration['IMAGE_BASENAME'])
> +    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
> +    if oeqa_json_result_common_dir:
> +        json_result_dir = oeqa_json_result_common_dir

Best regards,
Yeoh Ee Peng 

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 22, 2018 4:38 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 3/4] testimage.bbclass: write testresult to json files

On Mon, 2018-10-22 at 14:54 +0800, Yeoh Ee Peng wrote:
> As part of the solution to replace Testopia to store testresult, OEQA 
> testimage need to output testresult into json files, where these json 
> testresult files will be stored into git repository by the future 
> test-case-management tools.
> 
> To configure multiple instances of bitbake to write json testresult to 
> a single testresult file, user will define the variable 
> "OEQA_JSON_RESULT_COMMON_DIR" with the common directory for writing 
> json testresult.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/classes/testimage.bbclass | 34 
> ++++++++++++++++++++++++++++++++--
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/testimage.bbclass 
> b/meta/classes/testimage.bbclass index 2642a72..112ba71 100644
> --- a/meta/classes/testimage.bbclass
> +++ b/meta/classes/testimage.bbclass
> @@ -2,7 +2,7 @@
>  #
>  # Released under the MIT license (see COPYING.MIT)
>  
> -
> +inherit metadata_scm
>  # testimage.bbclass enables testing of qemu images using python unittests.
>  # Most of the tests are commands run on target image over ssh.
>  # To use it add testimage to global inherit and call your target 
> image with -c testimage @@ -141,6 +141,33 @@ def testimage_sanity(d):
>          bb.fatal('When TEST_TARGET is set to "simpleremote" '
>                   'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
>  
> +def _get_configuration(d, test_type, pid, machine):
> +    import platform
> +    bb_core_dir = d.getVar("BBFILE_PATTERN_core=")
> +    return {'TEST_TYPE': test_type,
> +            'BRANCH': base_get_metadata_git_branch(bb_core_dir, None).strip(),
> +            'COMMIT': base_get_metadata_git_revision(bb_core_dir, 
> +None),

We need to record all the revisions/branches that are being used. Could we just inject the contents of metadata['layers'] directly into the json file here?

> +            'PROCESS_ID': pid,
> +            'MACHINE': machine,
> +            'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
> +            'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
> +            'HOST_DISTRO': platform.linux_distribution()}
> +
> +def _get_json_result_dir(d, configuration):
> +    json_result_dir = os.path.join(d.getVar("WORKDIR"),
> +                                   'temp',
> +                                   'json_testresults-%s' % configuration['PROCESS_ID'],
> +                                   configuration['TEST_TYPE'],
> +                                   configuration['MACHINE'],
> +                                   configuration['IMAGE_BASENAME'])
> +    oeqa_json_result_common_dir = d.getVar("OEQA_JSON_RESULT_COMMON_DIR")
> +    if oeqa_json_result_common_dir:
> +        json_result_dir = oeqa_json_result_common_dir

Do we need both codepaths here or can we just place things in OEQA_JSON_RESULT_COMMON_DIR unconditionally here?

(we could rename it to OEQA_JSON_RESULT_DIR?)

Cheers,

Richard

Re: [PATCH 1/4] oeqa/core/runner: write testresult to json files

[richard.purdie]

On Mon, 2018-10-22 at 08:59 +0000, Yeoh, Ee Peng wrote:
> Hi Richard
> 
> Current codes does load existing testresult json file if it exist,
> then it will write the new testresult into it based on the result_id.
> > +    def _get_testresults(self, write_dir):
> > +        testresults = {}
> > +        file = os.path.join(write_dir, self.testresult_filename)
> > +        if os.path.exists(file):
> > +            with open(file, "r") as f:
> > +                testresults = json.load(f)
> > +        return testresults

I managed to miss that function and call, sorry. That should be fine. I
think we may want to inline some of these functions to make things
clearer.

> I did have the same thinking on if we can have a common function to
> manage configuration and result_id or let individual test classes to
> manage it, in the end, the thinking was configuration/result_id were
> really responsibility of each test classes, where the json helper
> class inside runner shall not have the knowledge or know-how on
> configuration/result_id. Thus the decision in the end was to make
> json helper class responsibility as simple as to consume the
> configuration, results, result_id information provided by the test
> classes. Hope this explain the reason behind current design. 
> 
> Thank you very much for your attention & knowledge sharing! 

You could just do what amounts to getVar("LOGDIR") + "/oeqa" and have
that as the default test result location?

Cheers,

Richard

Re: [PATCH 1/4] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

Hi Richard,

You are right, the current codes and functions does not express clearly what it does, especially on getting existing testresult, let me refactor this to make it clearer. 

Yes, let me use getVar("LOGDIR") + "/oeqa" as default result_dir. Do you think we shall have OEQA_JSON_RESULT_DIR variable for user to define custom result_dir? 

Thanks,
Yeoh Ee Peng 

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 22, 2018 5:34 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 1/4] oeqa/core/runner: write testresult to json files

On Mon, 2018-10-22 at 08:59 +0000, Yeoh, Ee Peng wrote:
> Hi Richard
> 
> Current codes does load existing testresult json file if it exist, 
> then it will write the new testresult into it based on the result_id.
> > +    def _get_testresults(self, write_dir):
> > +        testresults = {}
> > +        file = os.path.join(write_dir, self.testresult_filename)
> > +        if os.path.exists(file):
> > +            with open(file, "r") as f:
> > +                testresults = json.load(f)
> > +        return testresults

I managed to miss that function and call, sorry. That should be fine. I think we may want to inline some of these functions to make things clearer.

> I did have the same thinking on if we can have a common function to 
> manage configuration and result_id or let individual test classes to 
> manage it, in the end, the thinking was configuration/result_id were 
> really responsibility of each test classes, where the json helper 
> class inside runner shall not have the knowledge or know-how on 
> configuration/result_id. Thus the decision in the end was to make json 
> helper class responsibility as simple as to consume the configuration, 
> results, result_id information provided by the test classes. Hope this 
> explain the reason behind current design.
> 
> Thank you very much for your attention & knowledge sharing! 

You could just do what amounts to getVar("LOGDIR") + "/oeqa" and have that as the default test result location?

Cheers,

Richard

Re: [PATCH 1/4] oeqa/core/runner: write testresult to json files

[Yeoh, Ee Peng]

Hi Richard,

I had refactor and incorporate the inputs your provided, submitted the patches for your review.
Thank you very much for your attention and sharing!

http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156945.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156946.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156947.html
http://lists.openembedded.org/pipermail/openembedded-core/2018-October/156948.html

Thanks,
Yeoh Ee Peng 

-----Original Message-----
From: richard.purdie@linuxfoundation.org [mailto:richard.purdie@linuxfoundation.org] 
Sent: Monday, October 22, 2018 5:34 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH 1/4] oeqa/core/runner: write testresult to json files

On Mon, 2018-10-22 at 08:59 +0000, Yeoh, Ee Peng wrote:
> Hi Richard
> 
> Current codes does load existing testresult json file if it exist, 
> then it will write the new testresult into it based on the result_id.
> > +    def _get_testresults(self, write_dir):
> > +        testresults = {}
> > +        file = os.path.join(write_dir, self.testresult_filename)
> > +        if os.path.exists(file):
> > +            with open(file, "r") as f:
> > +                testresults = json.load(f)
> > +        return testresults

I managed to miss that function and call, sorry. That should be fine. I think we may want to inline some of these functions to make things clearer.

> I did have the same thinking on if we can have a common function to 
> manage configuration and result_id or let individual test classes to 
> manage it, in the end, the thinking was configuration/result_id were 
> really responsibility of each test classes, where the json helper 
> class inside runner shall not have the knowledge or know-how on 
> configuration/result_id. Thus the decision in the end was to make json 
> helper class responsibility as simple as to consume the configuration, 
> results, result_id information provided by the test classes. Hope this 
> explain the reason behind current design.
> 
> Thank you very much for your attention & knowledge sharing! 

You could just do what amounts to getVar("LOGDIR") + "/oeqa" and have that as the default test result location?

Cheers,

Richard

[PATCH 2/4] oeqa/selftest/context: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA selftest need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

By default, json testresult file will be written to "oeqa"
directory under the oe-selftest log directory.

To configure multiple instances of bitbake to write json testresult
to a single testresult file at custom directory, user will define
the variable "OEQA_JSON_RESULT_DIR" with the custom directory for
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/context.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index c78947e..ee83a91 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -99,8 +99,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         return cases_paths
 
     def _process_args(self, logger, args):
-        args.output_log = '%s-results-%s.log' % (self.name,
-                time.strftime("%Y%m%d%H%M%S"))
+        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
+        args.output_log = '%s-results-%s.log' % (self.name, args.test_start_time)
         args.test_data_file = None
         args.CASES_PATHS = None
 
@@ -204,6 +204,31 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable")
         runCmd("bitbake -e")
 
+    def _get_json_result_dir(self, args):
+        json_result_dir = os.path.join(os.path.dirname(os.path.abspath(args.output_log)), 'oeqa')
+        if "OEQA_JSON_RESULT_DIR" in self.tc.td:
+            json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"]
+
+        return json_result_dir
+
+    def _get_configuration(self, args):
+        import platform
+        from oeqa.utils.metadata import metadata_from_bb
+        metadata = metadata_from_bb()
+        configuration = {'TEST_TYPE': 'oeselftest',
+                        'START_TIME': args.test_start_time,
+                        'MACHINE': self.tc.td["MACHINE"],
+                        'HOST_DISTRO': ('-'.join(platform.linux_distribution())).replace(' ', '-'),
+                        'HOST_NAME': metadata['hostname']}
+        layers = metadata['layers']
+        for l in layers:
+            configuration['%s_BRANCH_REV' % os.path.basename(l)] = '%s:%s' % (metadata['layers'][l]['branch'],
+                                                                              metadata['layers'][l]['commit'])
+        return configuration
+
+    def _get_result_id(self, configuration):
+        return '%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'])
+
     def _internal_run(self, logger, args):
         self.module_paths = self._get_cases_paths(
                 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
@@ -220,7 +245,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         else:
             self._pre_run()
             rc = self.tc.runTests(**self.tc_kwargs['run'])
-            rc.logDetails()
+            configuration = self._get_configuration(args)
+            rc.logDetails(self._get_json_result_dir(args),
+                          configuration,
+                          self._get_result_id(configuration))
             rc.logSummary(self.name)
 
         return rc
-- 
2.7.4

[PATCH 2/4] oeqa/selftest/context: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA selftest need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

To configure multiple instances of bitbake to write json testresult
to a single testresult file at custom direcotry, user will define
the variable "OEQA_JSON_RESULT_DIR" with the custom directory for writing
json testresult.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/context.py | 36 +++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index c78947e..59d4b59 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -99,8 +99,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         return cases_paths
 
     def _process_args(self, logger, args):
-        args.output_log = '%s-results-%s.log' % (self.name,
-                time.strftime("%Y%m%d%H%M%S"))
+        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
+        args.output_log = '%s-results-%s.log' % (self.name, args.test_start_time)
         args.test_data_file = None
         args.CASES_PATHS = None
 
@@ -204,6 +204,33 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable")
         runCmd("bitbake -e")
 
+    def _get_json_result_dir(self, args):
+        json_result_dir = os.path.join(os.path.dirname(os.path.abspath(args.output_log)), 'oeqa')
+        if "OEQA_JSON_RESULT_DIR" in self.tc.td:
+            json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"]
+
+        return json_result_dir
+
+    def _get_configuration(self, args):
+        import platform
+        from oeqa.utils.metadata import metadata_from_bb
+        metadata = metadata_from_bb()
+        configuration = {'TEST_TYPE': 'oeselftest',
+                        'START_TIME': args.test_start_time,
+                        'MACHINE': self.tc.td["MACHINE"],
+                        'HOST_DISTRO': platform.linux_distribution(),
+                        'HOST_NAME': metadata['hostname']}
+        layers = metadata['layers']
+        for l in layers:
+            configuration['%s_BRANCH_REV' % os.path.basename(l)] = '%s:%s' % (
+                                                                    metadata['layers'][l]['branch'],
+                                                                    metadata['layers'][l]['commit'])
+        return configuration
+
+    def _get_result_id(self, configuration):
+        distro = '_'.join(configuration['HOST_DISTRO'])
+        return '%s-%s-%s' % (configuration['TEST_TYPE'], distro, configuration['MACHINE'])
+
     def _internal_run(self, logger, args):
         self.module_paths = self._get_cases_paths(
                 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
@@ -220,7 +247,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         else:
             self._pre_run()
             rc = self.tc.runTests(**self.tc_kwargs['run'])
-            rc.logDetails()
+            configuration = self._get_configuration(args)
+            rc.logDetails(self._get_json_result_dir(args),
+                          configuration,
+                          self._get_result_id(configuration))
             rc.logSummary(self.name)
 
         return rc
-- 
2.7.4

[PATCH 2/4] oeqa/selftest/context: write testresult to json files

[Yeoh Ee Peng]

As part of the solution to replace Testopia to store testresult,
OEQA selftest need to output testresult into json files, where
these json testresult files will be stored into git repository
by the future test-case-management tools.

By default, oe-selftest will write json testresult into files.
To disable this, provide '-s' argument to oe-selftest execution.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/context.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index c78947e..61b4afb 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -73,6 +73,9 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
 
         parser.add_argument('--machine', required=False, choices=['random', 'all'],
                             help='Run tests on different machines (random/all).')
+
+        parser.add_argument('-s', '--skip-export-json', action='store_true',
+                            help='Skip the output test result in json format to files.')
         
         parser.set_defaults(func=self.run)
 
@@ -99,8 +102,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         return cases_paths
 
     def _process_args(self, logger, args):
-        args.output_log = '%s-results-%s.log' % (self.name,
-                time.strftime("%Y%m%d%H%M%S"))
+        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
+        args.output_log = '%s-results-%s.log' % (self.name, args.test_start_time)
         args.test_data_file = None
         args.CASES_PATHS = None
 
@@ -222,6 +225,11 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
             rc = self.tc.runTests(**self.tc_kwargs['run'])
             rc.logDetails()
             rc.logSummary(self.name)
+            if not args.skip_export_json:
+                json_result_dir = os.path.join(os.path.dirname(os.path.abspath(args.output_log)),
+                                               'json_testresults-%s' % args.test_start_time,
+                                               'oe-selftest')
+                rc.logDetailsInJson(json_result_dir)
 
         return rc
 
-- 
2.7.4