* resulttool: Tweaks on top of Ee Peng's base patchset
@ 2019-02-17 16:12 Richard Purdie
0 siblings, 0 replies; only message in thread
From: Richard Purdie @ 2019-02-17 16:12 UTC (permalink / raw)
To: Yeoh Ee Peng, Patches and discussions about the oe-core layer
A summary of the tweaks I think we need to make on top of the base
patchset. See the separate email for a discussion of the reasoning
behind this and a list of the work remaining.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
scripts/lib/resulttool/manualexecution.py | 8 +-
scripts/lib/resulttool/merge.py | 54 ++---
scripts/lib/resulttool/regression.py | 247 ++++++++-------------
scripts/lib/resulttool/report.py | 86 +++----
scripts/lib/resulttool/resultsutils.py | 67 ------
scripts/lib/resulttool/resultutils.py | 104 +++++++++
scripts/lib/resulttool/store.py | 109 +++------
.../resulttool/template/test_report_full_text.txt | 31 +--
8 files changed, 294 insertions(+), 412 deletions(-)
delete mode 100644 scripts/lib/resulttool/resultsutils.py
create mode 100644 scripts/lib/resulttool/resultutils.py
diff --git a/scripts/lib/resulttool/manualexecution.py b/scripts/lib/resulttool/manualexecution.py
index 64ec581a9f..ecdc4e7a7a 100755
--- a/scripts/lib/resulttool/manualexecution.py
+++ b/scripts/lib/resulttool/manualexecution.py
@@ -18,7 +18,11 @@ import sys
import datetime
import re
from oeqa.core.runner import OETestResultJSONHelper
-from resulttool.resultsutils import load_json_file
+
+def load_json_file(file):
+ with open(file, "r") as f:
+ return json.load(f)
+
class ManualTestRunner(object):
def __init__(self):
@@ -134,4 +138,4 @@ def register_commands(subparsers):
description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/',
group='manualexecution')
parser_build.set_defaults(func=manualexecution)
- parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.')
\ No newline at end of file
+ parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.')
diff --git a/scripts/lib/resulttool/merge.py b/scripts/lib/resulttool/merge.py
index 1d9cfafd41..3a91bd8eac 100644
--- a/scripts/lib/resulttool/merge.py
+++ b/scripts/lib/resulttool/merge.py
@@ -1,4 +1,4 @@
-# test result tool - merge multiple testresults.json files
+# resulttool - merge multiple testresults.json files
#
# Copyright (c) 2019, Intel Corporation.
#
@@ -11,45 +11,23 @@
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
-from resulttool.resultsutils import load_json_file, get_dict_value, dump_json_data
import os
import json
-
-class ResultsMerge(object):
-
- def get_test_results(self, logger, file, result_id):
- results = load_json_file(file)
- if result_id:
- result = get_dict_value(logger, results, result_id)
- if result:
- return {result_id: result}
- return result
- return results
-
- def merge_results(self, base_results, target_results):
- for k in target_results:
- base_results[k] = target_results[k]
- return base_results
-
- def _get_write_dir(self):
- basepath = os.environ['BUILDDIR']
- return basepath + '/tmp/'
-
- def dump_merged_results(self, results, output_dir):
- file_output_dir = output_dir if output_dir else self._get_write_dir()
- dump_json_data(file_output_dir, 'testresults.json', results)
- print('Successfully merged results to: %s' % os.path.join(file_output_dir, 'testresults.json'))
-
- def run(self, logger, base_result_file, target_result_file, target_result_id, output_dir):
- base_results = self.get_test_results(logger, base_result_file, '')
- target_results = self.get_test_results(logger, target_result_file, target_result_id)
- if base_results and target_results:
- merged_results = self.merge_results(base_results, target_results)
- self.dump_merged_results(merged_results, output_dir)
+import resulttool.resultutils as resultutils
def merge(args, logger):
- merge = ResultsMerge()
- merge.run(logger, args.base_result_file, args.target_result_file, args.target_result_id, args.output_dir)
+ # FIXME Add -t support for args.target_result_id
+
+ if os.path.isdir(args.target_result_file):
+ results = resultutils.load_resultsdata(args.target_result_file, configmap=resultutils.store_map)
+ resultutils.append_resultsdata(results, args.base_result_file, configmap=resultutils.store_map)
+ resultutils.save_resultsdata(results, args.target_result_file)
+ else:
+ results = resultutils.load_resultsdata(args.base_result_file, configmap=resultutils.flatten_map)
+ if os.path.exists(args.target_result_file):
+ resultutils.append_resultsdata(results, args.target_result_file, configmap=resultutils.flatten_map)
+ resultutils.save_resultsdata(results, os.path.dirname(args.target_result_file), fn=os.path.basename(args.target_result_file))
+
return 0
def register_commands(subparsers):
@@ -66,6 +44,4 @@ def register_commands(subparsers):
parser_build.add_argument('-t', '--target-result-id', default='',
help='(optional) default merge all result sets available from target to base '
'unless specific target result id was provided')
- parser_build.add_argument('-o', '--output-dir', default='',
- help='(optional) default write merged results to <poky>/build/tmp/ unless specific '
- 'output directory was provided')
+
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index bee3fb011a..1f19c398f9 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -1,6 +1,7 @@
-# test result tool - regression analysis
+# resulttool - regression analysis
#
# Copyright (c) 2019, Intel Corporation.
+# Copyright (c) 2019, Linux Foundation
#
# This program is free software; you can redistribute it and/or modify it
# under the terms and conditions of the GNU General Public License,
@@ -11,171 +12,104 @@
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
-from resulttool.resultsutils import load_json_file, get_dict_value, pop_dict_element
+from resulttool.resultutils import checkout_git_dir
+import resulttool.resultutils as resultutils
import json
-class ResultsRegressionSelector(object):
-
- def get_results_unique_configurations(self, logger, results):
- unique_configurations_map = {"oeselftest": ['TEST_TYPE', 'HOST_DISTRO', 'MACHINE'],
- "runtime": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE'],
- "sdk": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'],
- "sdkext": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE']}
- results_unique_configs = {}
- for k in results:
- result = results[k]
- result_configs = get_dict_value(logger, result, 'configuration')
- result_test_type = get_dict_value(logger, result_configs, 'TEST_TYPE')
- unique_configuration_keys = get_dict_value(logger, unique_configurations_map, result_test_type)
- result_unique_config = {}
- for ck in unique_configuration_keys:
- config_value = get_dict_value(logger, result_configs, ck)
- if config_value:
- result_unique_config[ck] = config_value
- results_unique_configs[k] = result_unique_config
- return results_unique_configs
-
- def get_regression_base_target_pair(self, logger, base_results, target_results):
- base_configs = self.get_results_unique_configurations(logger, base_results)
- logger.debug('Retrieved base configuration: config=%s' % base_configs)
- target_configs = self.get_results_unique_configurations(logger, target_results)
- logger.debug('Retrieved target configuration: config=%s' % target_configs)
- regression_pair = {}
- for bk in base_configs:
- base_config = base_configs[bk]
- for tk in target_configs:
- target_config = target_configs[tk]
- if base_config == target_config:
- if bk in regression_pair:
- regression_pair[bk].append(tk)
- else:
- regression_pair[bk] = [tk]
- return regression_pair
-
- def run_regression_with_regression_pairing(self, logger, regression_pair, base_results, target_results):
- regression = ResultsRegression()
- for base in regression_pair:
- for target in regression_pair[base]:
- print('Getting regression for base=%s target=%s' % (base, target))
- regression.run(logger, base_results[base], target_results[target])
-
-class ResultsRegression(object):
-
- def print_regression_result(self, result):
- if result:
- print('============================Start Regression============================')
- print('Only print regression if base status not equal target')
- print('<test case> : <base status> -> <target status>')
- print('========================================================================')
- for k in result:
- print(k, ':', result[k]['base'], '->', result[k]['target'])
- print('==============================End Regression==============================')
-
- def get_regression_result(self, logger, base_result, target_result):
- base_result = get_dict_value(logger, base_result, 'result')
- target_result = get_dict_value(logger, target_result, 'result')
- result = {}
- if base_result and target_result:
- logger.debug('Getting regression result')
- for k in base_result:
- base_testcase = base_result[k]
- base_status = get_dict_value(logger, base_testcase, 'status')
- if base_status:
- target_testcase = get_dict_value(logger, target_result, k)
- target_status = get_dict_value(logger, target_testcase, 'status')
- if base_status != target_status:
- result[k] = {'base': base_status, 'target': target_status}
- else:
- logger.error('Failed to retrieved base test case status: %s' % k)
- return result
-
- def run(self, logger, base_result, target_result):
- if base_result and target_result:
- result = self.get_regression_result(logger, base_result, target_result)
- logger.debug('Retrieved regression result =%s' % result)
- self.print_regression_result(result)
- else:
- logger.error('Input data objects must not be empty (base_result=%s, target_result=%s)' %
- (base_result, target_result))
-
-def get_results_from_directory(logger, source_dir):
- from resulttool.merge import ResultsMerge
- from resulttool.resultsutils import get_directory_files
- result_files = get_directory_files(source_dir, ['.git'], 'testresults.json')
- base_results = {}
- for file in result_files:
- merge = ResultsMerge()
- results = merge.get_test_results(logger, file, '')
- base_results = merge.merge_results(base_results, results)
- return base_results
-
-def remove_testcases_to_optimize_regression_runtime(logger, results):
- test_case_removal = ['ptestresult.rawlogs', 'ptestresult.sections']
- for r in test_case_removal:
- for k in results:
- result = get_dict_value(logger, results[k], 'result')
- pop_dict_element(logger, result, r)
-
-def regression_file(args, logger):
- base_results = load_json_file(args.base_result_file)
- print('Successfully loaded base test results from: %s' % args.base_result_file)
- target_results = load_json_file(args.target_result_file)
- print('Successfully loaded target test results from: %s' % args.target_result_file)
- remove_testcases_to_optimize_regression_runtime(logger, base_results)
- remove_testcases_to_optimize_regression_runtime(logger, target_results)
- if args.base_result_id and args.target_result_id:
- base_result = get_dict_value(logger, base_results, base_result_id)
- print('Getting base test result with result_id=%s' % base_result_id)
- target_result = get_dict_value(logger, target_results, target_result_id)
- print('Getting target test result with result_id=%s' % target_result_id)
- regression = ResultsRegression()
- regression.run(logger, base_result, target_result)
+def compare_result(logger, base_name, target_name, base_result, target_result):
+ base_result = base_result.get('result')
+ target_result = target_result.get('result')
+ result = {}
+ if base_result and target_result:
+ for k in base_result:
+ base_testcase = base_result[k]
+ base_status = base_testcase.get('status')
+ if base_status:
+ target_testcase = target_result.get(k, {})
+ target_status = target_testcase.get('status')
+ if base_status != target_status:
+ result[k] = {'base': base_status, 'target': target_status}
+ else:
+ logger.error('Failed to retrieved base test case status: %s' % k)
+ if result:
+ resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
+ for k in result:
+ resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
else:
- regression = ResultsRegressionSelector()
- regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results)
- logger.debug('Retrieved regression pair=%s' % regression_pair)
- regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results)
- return 0
+ resultstring = "Match: %s\n %s" % (base_name, target_name)
+ return result, resultstring
+
+def get_results(logger, source):
+ return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)
+
+def regression(args, logger):
+ base_results = get_results(logger, args.base_result)
+ target_results = get_results(logger, args.target_result)
+
+ regression_common(args, logger, base_results, target_results)
+
+def regression_common(args, logger, base_results, target_results):
+ if args.base_result_id:
+ base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
+ if args.target_result_id:
+ target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
+
+ matches = []
+ regressions = []
+ notfound = []
+
+ for a in base_results:
+ if a in target_results:
+ base = list(base_results[a].keys())
+ target = list(target_results[a].keys())
+ # We may have multiple base/targets which are for different configurations. Start by
+ # removing any pairs which match
+ for c in base.copy():
+ for b in target.copy():
+ res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
+ if not res:
+ matches.append(resstr)
+ base.remove(c)
+ target.remove(b)
+ break
+ # Should only now see regressions, we may not be able to match multiple pairs directly
+ for c in base:
+ for b in target:
+ res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
+ if res:
+ regressions.append(resstr)
+ else:
+ notfound.append("%s not found in target" % a)
+ print("\n".join(matches))
+ print("\n".join(regressions))
+ print("\n".join(notfound))
-def regression_directory(args, logger):
- base_results = get_results_from_directory(logger, args.base_result_directory)
- target_results = get_results_from_directory(logger, args.target_result_directory)
- remove_testcases_to_optimize_regression_runtime(logger, base_results)
- remove_testcases_to_optimize_regression_runtime(logger, target_results)
- regression = ResultsRegressionSelector()
- regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results)
- logger.debug('Retrieved regression pair=%s' % regression_pair)
- regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results)
return 0
def regression_git(args, logger):
- from resulttool.resultsutils import checkout_git_dir
base_results = {}
target_results = {}
if checkout_git_dir(args.source_dir, args.base_git_branch):
- base_results = get_results_from_directory(logger, args.source_dir)
+ base_results = get_results(logger, args.source_dir)
if checkout_git_dir(args.source_dir, args.target_git_branch):
- target_results = get_results_from_directory(logger, args.source_dir)
- if base_results and target_results:
- remove_testcases_to_optimize_regression_runtime(logger, base_results)
- remove_testcases_to_optimize_regression_runtime(logger, target_results)
- regression = ResultsRegressionSelector()
- regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results)
- logger.debug('Retrieved regression pair=%s' % regression_pair)
- regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results)
+ target_results = get_results(logger, args.source_dir)
+
+ regression_common(args, logger, base_result, target_result)
return 0
def register_commands(subparsers):
"""Register subcommands from this plugin"""
- parser_build = subparsers.add_parser('regression-file', help='regression file analysis',
+
+ parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
description='regression analysis comparing base result set to target '
'result set',
group='analysis')
- parser_build.set_defaults(func=regression_file)
- parser_build.add_argument('base_result_file',
- help='base result file provide the base result set')
- parser_build.add_argument('target_result_file',
- help='target result file provide the target result set for comparison with base result')
+ parser_build.set_defaults(func=regression)
+ parser_build.add_argument('base_result',
+ help='base result file/directory provide the files for base result set')
+ parser_build.add_argument('target_result',
+ help='target result file/directory provide the files for target result set for comparison with '
+ 'base result')
parser_build.add_argument('-b', '--base-result-id', default='',
help='(optional) default select regression based on configurations unless base result '
'id was provided')
@@ -183,16 +117,6 @@ def register_commands(subparsers):
help='(optional) default select regression based on configurations unless target result '
'id was provided')
- parser_build = subparsers.add_parser('regression-dir', help='regression directory analysis',
- description='regression analysis comparing base result set to target '
- 'result set',
- group='analysis')
- parser_build.set_defaults(func=regression_directory)
- parser_build.add_argument('base_result_directory',
- help='base result directory provide the files for base result set')
- parser_build.add_argument('target_result_directory',
- help='target result file provide the files for target result set for comparison with '
- 'base result')
parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
description='regression analysis comparing base result set to target '
@@ -206,3 +130,10 @@ def register_commands(subparsers):
parser_build.add_argument('target_git_branch',
help='target git branch that provide the files for target result set for comparison with '
'base result')
+ parser_build.add_argument('-b', '--base-result-id', default='',
+ help='(optional) default select regression based on configurations unless base result '
+ 'id was provided')
+ parser_build.add_argument('-t', '--target-result-id', default='',
+ help='(optional) default select regression based on configurations unless target result '
+ 'id was provided')
+
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py
index ab5de1f3a7..bdb49ce4be 100644
--- a/scripts/lib/resulttool/report.py
+++ b/scripts/lib/resulttool/report.py
@@ -1,6 +1,7 @@
# test result tool - report text based test results
#
# Copyright (c) 2019, Intel Corporation.
+# Copyright (c) 2019, Linux Foundation
#
# This program is free software; you can redistribute it and/or modify it
# under the terms and conditions of the GNU General Public License,
@@ -14,7 +15,8 @@
import os
import glob
import json
-from resulttool.resultsutils import checkout_git_dir, load_json_file, get_dict_value, get_directory_files
+from resulttool.resultutils import checkout_git_dir
+import resulttool.resultutils as resultutils
class ResultsTextReport(object):
@@ -23,9 +25,9 @@ class ResultsTextReport(object):
result_types = {'passed': ['PASSED', 'passed'],
'failed': ['FAILED', 'failed', 'ERROR', 'error', 'UNKNOWN'],
'skipped': ['SKIPPED', 'skipped']}
- result = get_dict_value(logger, testresult, 'result')
+ result = testresult.get('result', [])
for k in result:
- test_status = get_dict_value(logger, result[k], 'status')
+ test_status = result[k].get('status', [])
for tk in result_types:
if test_status in result_types[tk]:
test_count_report[tk] += 1
@@ -33,66 +35,48 @@ class ResultsTextReport(object):
test_count_report['failed_testcases'].append(k)
return test_count_report
- def get_test_result_percentage(self, test_result_count):
- total_tested = test_result_count['passed'] + test_result_count['failed'] + test_result_count['skipped']
- test_percent_report = {'passed': 0, 'failed': 0, 'skipped': 0}
- for k in test_percent_report:
- test_percent_report[k] = format(test_result_count[k] / total_tested * 100, '.2f')
- return test_percent_report
-
- def add_test_configurations(self, test_report, source_dir, file, result_id):
- test_report['file_dir'] = self._get_short_file_dir(source_dir, file)
- test_report['result_id'] = result_id
- test_report['test_file_dir_result_id'] = '%s_%s' % (test_report['file_dir'], test_report['result_id'])
-
- def _get_short_file_dir(self, source_dir, file):
- file_dir = os.path.dirname(file)
- source_dir = source_dir[:-1] if source_dir[-1] == '/' else source_dir
- if file_dir == source_dir:
- return 'None'
- return file_dir.replace(source_dir, '')
-
- def get_max_string_len(self, test_result_list, key, default_max_len):
- max_len = default_max_len
- for test_result in test_result_list:
- value_len = len(test_result[key])
- if value_len > max_len:
- max_len = value_len
- return max_len
-
- def print_test_report(self, template_file_name, test_count_reports, test_percent_reports,
- max_len_dir, max_len_result_id):
+ def print_test_report(self, template_file_name, test_count_reports):
from jinja2 import Environment, FileSystemLoader
script_path = os.path.dirname(os.path.realpath(__file__))
file_loader = FileSystemLoader(script_path + '/template')
env = Environment(loader=file_loader, trim_blocks=True)
template = env.get_template(template_file_name)
- output = template.render(test_count_reports=test_count_reports,
- test_percent_reports=test_percent_reports,
- max_len_dir=max_len_dir,
- max_len_result_id=max_len_result_id)
- print('Printing text-based test report:')
+ havefailed = False
+ reportvalues = []
+ cols = ['passed', 'failed', 'skipped']
+ maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' :0 }
+ for line in test_count_reports:
+ total_tested = line['passed'] + line['failed'] + line['skipped']
+ vals = {}
+ vals['result_id'] = line['result_id']
+ vals['testseries'] = line['testseries']
+ vals['sort'] = line['testseries'] + "_" + line['result_id']
+ for k in cols:
+ vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f'))
+ for k in maxlen:
+ if len(vals[k]) > maxlen[k]:
+ maxlen[k] = len(vals[k])
+ reportvalues.append(vals)
+ if line['failed_testcases']:
+ havefailed = True
+ output = template.render(reportvalues=reportvalues,
+ havefailed=havefailed,
+ maxlen=maxlen)
print(output)
def view_test_report(self, logger, source_dir, git_branch):
if git_branch:
checkout_git_dir(source_dir, git_branch)
test_count_reports = []
- test_percent_reports = []
- for file in get_directory_files(source_dir, ['.git'], 'testresults.json'):
- logger.debug('Computing result for test result file: %s' % file)
- testresults = load_json_file(file)
- for k in testresults:
- test_count_report = self.get_aggregated_test_result(logger, testresults[k])
- test_percent_report = self.get_test_result_percentage(test_count_report)
- self.add_test_configurations(test_count_report, source_dir, file, k)
- self.add_test_configurations(test_percent_report, source_dir, file, k)
+ testresults = resultutils.load_resultsdata(source_dir)
+ for testsuite in testresults:
+ for resultid in testresults[testsuite]:
+ result = testresults[testsuite][resultid]
+ test_count_report = self.get_aggregated_test_result(logger, result)
+ test_count_report['testseries'] = result['configuration']['TESTSERIES']
+ test_count_report['result_id'] = resultid
test_count_reports.append(test_count_report)
- test_percent_reports.append(test_percent_report)
- max_len_dir = self.get_max_string_len(test_count_reports, 'file_dir', len('file_dir'))
- max_len_result_id = self.get_max_string_len(test_count_reports, 'result_id', len('result_id'))
- self.print_test_report('test_report_full_text.txt', test_count_reports, test_percent_reports,
- max_len_dir, max_len_result_id)
+ self.print_test_report('test_report_full_text.txt', test_count_reports)
def report(args, logger):
report = ResultsTextReport()
diff --git a/scripts/lib/resulttool/resultsutils.py b/scripts/lib/resulttool/resultsutils.py
deleted file mode 100644
index 368786922c..0000000000
--- a/scripts/lib/resulttool/resultsutils.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# test result tool - utilities
-#
-# Copyright (c) 2019, Intel Corporation.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms and conditions of the GNU General Public License,
-# version 2, as published by the Free Software Foundation.
-#
-# This program is distributed in the hope it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-# more details.
-#
-import os
-import json
-import scriptpath
-scriptpath.add_oe_lib_path()
-from oeqa.utils.git import GitRepo, GitError
-
-def load_json_file(file):
- with open(file, "r") as f:
- return json.load(f)
-
-def dump_json_data(write_dir, file_name, json_data):
- file_content = json.dumps(json_data, sort_keys=True, indent=4)
- file_path = os.path.join(write_dir, file_name)
- with open(file_path, 'w') as the_file:
- the_file.write(file_content)
-
-def get_dict_value(logger, dict, key):
- try:
- return dict[key]
- except KeyError:
- if logger:
- logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key))
- return None
- except TypeError:
- if logger:
- logger.debug('Faced TypeError exception: dict=%s: key=%s' % (dict, key))
- return None
-
-def pop_dict_element(logger, dict, key):
- try:
- dict.pop(key)
- except KeyError:
- if logger:
- logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key))
- except AttributeError:
- if logger:
- logger.debug('Faced AttributeError exception: dict=%s: key=%s' % (dict, key))
-
-def checkout_git_dir(git_dir, git_branch):
- try:
- repo = GitRepo(git_dir, is_topdir=True)
- repo.run_cmd('checkout %s' % git_branch)
- return True
- except GitError:
- return False
-
-def get_directory_files(source_dir, excludes, file):
- files_in_dir = []
- for root, dirs, files in os.walk(source_dir, topdown=True):
- [dirs.remove(d) for d in list(dirs) if d in excludes]
- for name in files:
- if name == file:
- files_in_dir.append(os.path.join(root, name))
- return files_in_dir
\ No newline at end of file
diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py
new file mode 100644
index 0000000000..54f8dd1bfa
--- /dev/null
+++ b/scripts/lib/resulttool/resultutils.py
@@ -0,0 +1,104 @@
+# resulttool - common library/utility functions
+#
+# Copyright (c) 2019, Intel Corporation.
+# Copyright (c) 2019, Linux Foundation
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+import os
+import json
+import scriptpath
+scriptpath.add_oe_lib_path()
+from oeqa.utils.git import GitRepo, GitError
+
+def checkout_git_dir(git_dir, git_branch):
+ try:
+ repo = GitRepo(git_dir, is_topdir=True)
+ repo.run_cmd('checkout %s' % git_branch)
+ return True
+ except GitError:
+ return False
+
+flatten_map = {
+ "oeselftest": [],
+ "runtime": [],
+ "sdk": [],
+ "sdkext": []
+}
+regression_map = {
+ "oeselftest": ['TEST_TYPE', 'MACHINE'],
+ "runtime": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'IMAGE_PKGTYPE', 'DISTRO'],
+ "sdk": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'],
+ "sdkext": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE']
+}
+store_map = {
+ "oeselftest": ['TEST_TYPE'],
+ "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'],
+ "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'],
+ "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME']
+}
+
+#
+# Load the json file and append the results data into the provided results dict
+#
+def append_resultsdata(results, f, configmap=store_map):
+ with open(f, "r") as filedata:
+ data = json.load(filedata)
+ for res in data:
+ if "configuration" not in data[res] or "result" not in data[res]:
+ raise ValueError("Test results data without configuration or result section?")
+ if "TESTSERIES" not in data[res]["configuration"]:
+ data[res]["configuration"]["TESTSERIES"] = os.path.basename(os.path.dirname(f))
+ testtype = data[res]["configuration"].get("TEST_TYPE")
+ if testtype not in configmap:
+ raise ValueError("Unknown test type %s" % testtype)
+ configvars = configmap[testtype]
+ testpath = "/".join(data[res]["configuration"].get(i) for i in configmap[testtype])
+ if testpath not in results:
+ results[testpath] = {}
+ for i in ['ptestresult.rawlogs', 'ptestresult.sections']:
+ if i in data[res]['result']:
+ del data[res]['result'][i]
+ results[testpath][res] = data[res]
+
+#
+# Walk a directory and find/load results data
+# or load directly from a file
+#
+def load_resultsdata(source, configmap=store_map):
+ results = {}
+ if os.path.isfile(source):
+ append_resultsdata(results, source, configmap)
+ return results
+ for root, dirs, files in os.walk(source):
+ for name in files:
+ f = os.path.join(root, name)
+ if name == "testresults.json":
+ append_resultsdata(results, f, configmap)
+ return results
+
+def filter_resultsdata(results, resultid):
+ newresults = {}
+ for r in results:
+ for i in results[r]:
+ if i == resultsid:
+ newresults[r] = {}
+ newresults[r][i] = results[r][i]
+ return newresults
+
+def save_resultsdata(results, destdir, fn="testresults.json"):
+ for res in results:
+ if res:
+ dst = destdir + "/" + res + "/" + fn
+ else:
+ dst = destdir + "/" + fn
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ with open(dst, 'w') as f:
+ f.write(json.dumps(results[res], sort_keys=True, indent=4))
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py
index 2c6fd8492c..236bfdf64d 100644
--- a/scripts/lib/resulttool/store.py
+++ b/scripts/lib/resulttool/store.py
@@ -1,6 +1,7 @@
-# test result tool - store test results
+# resulttool - store test results
#
# Copyright (c) 2019, Intel Corporation.
+# Copyright (c) 2019, Linux Foundation
#
# This program is free software; you can redistribute it and/or modify it
# under the terms and conditions of the GNU General Public License,
@@ -11,87 +12,44 @@
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
-import datetime
import tempfile
import os
import subprocess
+import json
+import shutil
import scriptpath
scriptpath.add_bitbake_lib_path()
scriptpath.add_oe_lib_path()
-from resulttool.resultsutils import checkout_git_dir
-try:
- import bb
-except ImportError:
- pass
+import resulttool.resultutils as resultutils
-class ResultsGitStore(object):
- def _get_output_dir(self):
- basepath = os.environ['BUILDDIR']
- return basepath + '/testresults_%s/' % datetime.datetime.now().strftime("%Y%m%d%H%M%S")
-
- def _create_temporary_workspace_dir(self):
- return tempfile.mkdtemp(prefix='testresults.')
-
- def _remove_temporary_workspace_dir(self, workspace_dir):
- return subprocess.run(["rm", "-rf", workspace_dir])
-
- def _oe_copy_files(self, source_dir, destination_dir):
- from oe.path import copytree
- copytree(source_dir, destination_dir)
-
- def _copy_files(self, source_dir, destination_dir, copy_ignore=None):
- from shutil import copytree
- copytree(source_dir, destination_dir, ignore=copy_ignore)
-
- def _store_files_to_git(self, logger, file_dir, git_dir, git_branch, commit_msg_subject, commit_msg_body):
- logger.debug('Storing test result into git repository (%s) and branch (%s)'
- % (git_dir, git_branch))
- return subprocess.run(["oe-git-archive",
- file_dir,
- "-g", git_dir,
- "-b", git_branch,
- "--commit-msg-subject", commit_msg_subject,
- "--commit-msg-body", commit_msg_body])
-
- def store_to_existing(self, logger, source_dir, git_dir, git_branch):
- logger.debug('Storing files to existing git repository and branch')
- from shutil import ignore_patterns
- dest_dir = self._create_temporary_workspace_dir()
- dest_top_dir = os.path.join(dest_dir, 'top_dir')
- self._copy_files(git_dir, dest_top_dir, copy_ignore=ignore_patterns('.git'))
- self._oe_copy_files(source_dir, dest_top_dir)
- self._store_files_to_git(logger, dest_top_dir, git_dir, git_branch,
- 'Store as existing git and branch', 'Store as existing git repository and branch')
- self._remove_temporary_workspace_dir(dest_dir)
- return git_dir
-
- def store_to_existing_with_new_branch(self, logger, source_dir, git_dir, git_branch):
- logger.debug('Storing files to existing git repository with new branch')
- self._store_files_to_git(logger, source_dir, git_dir, git_branch,
- 'Store as existing git with new branch',
- 'Store as existing git repository with new branch')
- return git_dir
-
- def store_to_new(self, logger, source_dir, git_branch):
- logger.debug('Storing files to new git repository')
- output_dir = self._get_output_dir()
- self._store_files_to_git(logger, source_dir, output_dir, git_branch,
- 'Store as new', 'Store as new git repository')
- return output_dir
+def store(args, logger):
+ tempdir = tempfile.mkdtemp(prefix='testresults.')
+ try:
+ results = {}
+ logger.info('Reading files from %s' % args.source_dir)
+ for root, dirs, files in os.walk(args.source_dir):
+ for name in files:
+ f = os.path.join(root, name)
+ if name == "testresults.json":
+ resultutils.append_resultsdata(results, f)
+ else:
+ dst = f.replace(args.source_dir, tempdir + "/")
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ shutil.copyfile(f, dst)
+ resultutils.save_resultsdata(results, tempdir)
- def store(self, logger, source_dir, git_dir, git_branch):
- if git_dir:
- if checkout_git_dir(git_dir, git_branch):
- self.store_to_existing(logger, source_dir, git_dir, git_branch)
- else:
- self.store_to_existing_with_new_branch(logger, source_dir, git_dir, git_branch)
- else:
- self.store_to_new(logger, source_dir, git_branch)
+ logger.info('Storing test result into git repository %s' % args.git_dir)
+ subprocess.check_call(["oe-git-archive",
+ tempdir,
+ "-g", args.git_dir,
+ "-b", "{branch}",
+ "--tag-name", "{branch}/{commit_count}-g{commit}/{tag_number}",
+ "--commit-msg-subject", "Results of {branch}:{commit}",
+ "--commit-msg-body", "branch: {branch}\ncommit: {commit}"])
+ finally:
+ subprocess.check_call(["rm", "-rf", tempdir])
-def store(args, logger):
- gitstore = ResultsGitStore()
- gitstore.store(logger, args.source_dir, args.git_dir, args.git_branch)
return 0
def register_commands(subparsers):
@@ -104,7 +62,6 @@ def register_commands(subparsers):
parser_build.set_defaults(func=store)
parser_build.add_argument('source_dir',
help='source directory that contain the test result files and directories to be stored')
- parser_build.add_argument('git_branch', help='git branch used for store')
- parser_build.add_argument('-d', '--git-dir', default='',
- help='(optional) default store to new <top_dir>/<build>/<testresults_datetime> '
- 'directory unless provided with existing git repository as destination')
+ parser_build.add_argument('git_dir',
+ help='the location of the git repository to store the results in')
+
diff --git a/scripts/lib/resulttool/template/test_report_full_text.txt b/scripts/lib/resulttool/template/test_report_full_text.txt
index bc4874ba4b..1c7484dc0e 100644
--- a/scripts/lib/resulttool/template/test_report_full_text.txt
+++ b/scripts/lib/resulttool/template/test_report_full_text.txt
@@ -1,35 +1,28 @@
==============================================================================================================
-Test Report (Count of passed, failed, skipped group by file_dir, result_id)
+Test Result Status Summary (Counts/Percentages sorted by testseries, ID)
==============================================================================================================
--------------------------------------------------------------------------------------------------------------
-{{ 'file_dir'.ljust(max_len_dir) }} | {{ 'result_id'.ljust(max_len_result_id) }} | {{ 'passed'.ljust(10) }} | {{ 'failed'.ljust(10) }} | {{ 'skipped'.ljust(10) }}
+{{ 'Test Series'.ljust(maxlen['testseries']) }} | {{ 'ID'.ljust(maxlen['result_id']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }}
--------------------------------------------------------------------------------------------------------------
-{% for report in test_count_reports |sort(attribute='test_file_dir_result_id') %}
-{{ report.file_dir.ljust(max_len_dir) }} | {{ report.result_id.ljust(max_len_result_id) }} | {{ (report.passed|string).ljust(10) }} | {{ (report.failed|string).ljust(10) }} | {{ (report.skipped|string).ljust(10) }}
+{% for report in reportvalues |sort(attribute='sort') %}
+{{ report.testseries.ljust(maxlen['testseries']) }} | {{ report.result_id.ljust(maxlen['result_id']) }} | {{ (report.passed|string).ljust(maxlen['passed']) }} | {{ (report.failed|string).ljust(maxlen['failed']) }} | {{ (report.skipped|string).ljust(maxlen['skipped']) }}
{% endfor %}
--------------------------------------------------------------------------------------------------------------
==============================================================================================================
-Test Report (Percent of passed, failed, skipped group by file_dir, result_id)
+Failed test cases (sorted by testseries, ID)
==============================================================================================================
+{% if havefailed %}
--------------------------------------------------------------------------------------------------------------
-{{ 'file_dir'.ljust(max_len_dir) }} | {{ 'result_id'.ljust(max_len_result_id) }} | {{ 'passed_%'.ljust(10) }} | {{ 'failed_%'.ljust(10) }} | {{ 'skipped_%'.ljust(10) }}
---------------------------------------------------------------------------------------------------------------
-{% for report in test_percent_reports |sort(attribute='test_file_dir_result_id') %}
-{{ report.file_dir.ljust(max_len_dir) }} | {{ report.result_id.ljust(max_len_result_id) }} | {{ (report.passed|string).ljust(10) }} | {{ (report.failed|string).ljust(10) }} | {{ (report.skipped|string).ljust(10) }}
-{% endfor %}
---------------------------------------------------------------------------------------------------------------
-
-==============================================================================================================
-Test Report (Failed test cases group by file_dir, result_id)
-==============================================================================================================
---------------------------------------------------------------------------------------------------------------
-{% for report in test_count_reports |sort(attribute='test_file_dir_result_id') %}
+{% for report in reportvalues |sort(attribute='sort') %}
{% if report.failed_testcases %}
-file_dir | result_id : {{ report.file_dir }} | {{ report.result_id }}
+testseries | result_id : {{ report.testseries }} | {{ report.result_id }}
{% for testcase in report.failed_testcases %}
{{ testcase }}
{% endfor %}
{% endif %}
{% endfor %}
---------------------------------------------------------------------------------------------------------------
\ No newline at end of file
+--------------------------------------------------------------------------------------------------------------
+{% else %}
+There were no test failures
+{% endif %}
--
cgit v1.2.1
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2019-02-17 16:12 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-02-17 16:12 resulttool: Tweaks on top of Ee Peng's base patchset Richard Purdie
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.