* [PATCH 01/28] oeqa.utils.commands: Introduce get_bb_vars()
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 02/28] oeqa.utils.commands: use get_bb_vars() in get_bb_var() Markus Lehtonen
` (26 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
A new function for getting values of multiple bitbake variables at the
same time.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/utils/commands.py | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 18fe39e..0297e53 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -141,6 +141,39 @@ def get_bb_env(target=None, postconfig=None):
else:
return bitbake("-e", postconfig=postconfig).output
+def get_bb_vars(variables=None, target=None, postconfig=None):
+ """Get values of multiple bitbake variables"""
+ bbenv = get_bb_env(target, postconfig=postconfig)
+
+ var_re = re.compile(r'^(export )?(?P<var>[a-zA-Z]\w+)="(?P<value>.*)"$')
+ unset_re = re.compile(r'^unset (?P<var>[a-zA-Z]\w+)$')
+ lastline = None
+ values = {}
+ for line in bbenv.splitlines():
+ match = var_re.match(line)
+ val = None
+ if match:
+ val = match.group('value')
+ else:
+ match = unset_re.match(line)
+ if match:
+ # Handle [unexport] variables
+ if lastline.startswith('# "'):
+ val = lastline.split('"')[1]
+ if val:
+ var = match.group('var')
+ if variables is None:
+ values[var] = val
+ else:
+ if var in variables:
+ values[var] = val
+ variables.remove(var)
+ # Stop after all required variables have been found
+ if not variables:
+ break
+ lastline = line
+ return values
+
def get_bb_var(var, target=None, postconfig=None):
val = None
bbenv = get_bb_env(target, postconfig=postconfig)
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 02/28] oeqa.utils.commands: use get_bb_vars() in get_bb_var()
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
2016-06-24 10:37 ` [PATCH 01/28] oeqa.utils.commands: Introduce get_bb_vars() Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 03/28] oeqa.utils.commands: runCmd: gracefully handle empty output Markus Lehtonen
` (25 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Get rid of duplicate code.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/utils/commands.py | 20 +++++---------------
1 file changed, 5 insertions(+), 15 deletions(-)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 0297e53..2e513be 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -172,24 +172,14 @@ def get_bb_vars(variables=None, target=None, postconfig=None):
if not variables:
break
lastline = line
+ if variables:
+ # Fill in missing values
+ for var in variables:
+ values[var] = None
return values
def get_bb_var(var, target=None, postconfig=None):
- val = None
- bbenv = get_bb_env(target, postconfig=postconfig)
- lastline = None
- for line in bbenv.splitlines():
- if re.search("^(export )?%s=" % var, line):
- val = line.split('=', 1)[1]
- val = val.strip('\"')
- break
- elif re.match("unset %s$" % var, line):
- # Handle [unexport] variables
- if lastline.startswith('# "'):
- val = lastline.split('\"')[1]
- break
- lastline = line
- return val
+ return get_bb_vars([var], target, postconfig)[var]
def get_test_layer():
layers = get_bb_var("BBLAYERS").split()
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 03/28] oeqa.utils.commands: runCmd: gracefully handle empty output
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
2016-06-24 10:37 ` [PATCH 01/28] oeqa.utils.commands: Introduce get_bb_vars() Markus Lehtonen
2016-06-24 10:37 ` [PATCH 02/28] oeqa.utils.commands: use get_bb_vars() in get_bb_var() Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 04/28] oeqa.utils.commands: runCmd: return stderr output, too Markus Lehtonen
` (24 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Don't crash if the caller redirects the command output e.g. into a file,
causing the "output" seen by runCmd to be None.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/utils/commands.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 2e513be..171c64f 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -78,14 +78,15 @@ class Command(object):
self.process.kill()
self.thread.join()
- self.output = self.output.decode("utf-8").rstrip()
+ if self.output:
+ self.output = self.output.decode("utf-8").rstrip()
self.status = self.process.poll()
self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status))
# logging the complete output is insane
# bitbake -e output is really big
# and makes the log file useless
- if self.status:
+ if self.status and self.output:
lout = "\n".join(self.output.splitlines()[-20:])
self.log.debug("Last 20 lines:\n%s" % lout)
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 04/28] oeqa.utils.commands: runCmd: return stderr output, too
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (2 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 03/28] oeqa.utils.commands: runCmd: gracefully handle empty output Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 05/28] scripts: introduce oe-build-perf-test Markus Lehtonen
` (23 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Useful if one wants to separate stdout and stderr.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/utils/commands.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 171c64f..17341aa 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -104,6 +104,7 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **opti
result.command = command
result.status = cmd.status
result.output = cmd.output
+ result.error = cmd.error
result.pid = cmd.process.pid
if result.status and not ignore_status:
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 05/28] scripts: introduce oe-build-perf-test
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (3 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 04/28] oeqa.utils.commands: runCmd: return stderr output, too Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 06/28] oe-build-perf-test: add pre-run sanity check Markus Lehtonen
` (22 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Initial wireframe for re-writing build-perf-test.sh in Python.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/oe-build-perf-test | 51 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100755 scripts/oe-build-perf-test
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
new file mode 100755
index 0000000..66477eb
--- /dev/null
+++ b/scripts/oe-build-perf-test
@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+#
+# Build performance test script
+#
+# Copyright (c) 2016, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+"""Build performance test script"""
+import argparse
+import logging
+import sys
+
+
+# Set-up logging
+LOG_FORMAT = '[%(asctime)s] %(levelname)s: %(message)s'
+logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+log = logging.getLogger()
+
+
+def parse_args(argv):
+ """Parse command line arguments"""
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+ parser.add_argument('-D', '--debug', action='store_true',
+ help='Enable debug level logging')
+
+ return parser.parse_args(argv)
+
+
+def main(argv=None):
+ """Script entry point"""
+ args = parse_args(argv)
+
+ if args.debug:
+ log.setLevel(logging.DEBUG)
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
+
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 06/28] oe-build-perf-test: add pre-run sanity check
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (4 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 05/28] scripts: introduce oe-build-perf-test Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 07/28] oe-build-perf-test: introduce oeqa.buildperf module Markus Lehtonen
` (21 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
The script will be required to be run in an initialized bitbake build
environment.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/oe-build-perf-test | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index 66477eb..9fb4310 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -16,8 +16,14 @@
"""Build performance test script"""
import argparse
import logging
+import os
import sys
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
+import scriptpath
+scriptpath.add_oe_lib_path()
+from oeqa.utils.commands import runCmd
+
# Set-up logging
LOG_FORMAT = '[%(asctime)s] %(levelname)s: %(message)s'
@@ -25,6 +31,25 @@ logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
log = logging.getLogger()
+def pre_run_sanity_check():
+ """Sanity check of build environment"""
+ build_dir = os.environ.get("BUILDDIR")
+ if not build_dir:
+ log.error("BUILDDIR not set. Please run the build environmnent setup "
+ "script.")
+ return False
+ if os.getcwd() != build_dir:
+ log.error("Please run this script under BUILDDIR (%s)", build_dir)
+ return False
+
+ ret = runCmd('which bitbake', ignore_status=True)
+ if ret.status:
+ log.error("bitbake command not found")
+ return False
+
+ return True
+
+
def parse_args(argv):
"""Parse command line arguments"""
parser = argparse.ArgumentParser(
@@ -43,6 +68,9 @@ def main(argv=None):
if args.debug:
log.setLevel(logging.DEBUG)
+ if not pre_run_sanity_check():
+ return 1
+
return 0
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 07/28] oe-build-perf-test: introduce oeqa.buildperf module
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (5 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 06/28] oe-build-perf-test: add pre-run sanity check Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 08/28] oeqa.buildperf: functionality to drop kernel caches Markus Lehtonen
` (20 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Wireframe of a new Python module for containing build performance tests
and utility functions.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/__init__.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
create mode 100644 meta/lib/oeqa/buildperf/__init__.py
diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py
new file mode 100644
index 0000000..c953559
--- /dev/null
+++ b/meta/lib/oeqa/buildperf/__init__.py
@@ -0,0 +1,12 @@
+# Copyright (c) 2016, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+"""Build performance tests"""
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 08/28] oeqa.buildperf: functionality to drop kernel caches
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (6 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 07/28] oe-build-perf-test: introduce oeqa.buildperf module Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 09/28] oeqa.buildperf: add BuildPerfTest class Markus Lehtonen
` (19 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Add a new utility class for dropping Linux kernel caches. It uses sudo
and tee to write to the drop_caches file. Checking if the user has the
permissions to drop caches (without a password) is done by trying to
writing an invalid value to the drop_caches file. This way, we will find
if writing (with tee) is possible but not really dropping caches, yet.
User can avoid giving the password by adding something like:
<user> ALL = NOPASSWD: /usr/bin/tee /proc/sys/vm/drop_caches
to the system sudoers file.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/__init__.py | 1 +
meta/lib/oeqa/buildperf/base.py | 46 +++++++++++++++++++++++++++++++++++++
scripts/oe-build-perf-test | 4 ++++
3 files changed, 51 insertions(+)
create mode 100644 meta/lib/oeqa/buildperf/base.py
diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py
index c953559..bab122e 100644
--- a/meta/lib/oeqa/buildperf/__init__.py
+++ b/meta/lib/oeqa/buildperf/__init__.py
@@ -10,3 +10,4 @@
# more details.
#
"""Build performance tests"""
+from .base import KernelDropCaches
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
new file mode 100644
index 0000000..3cbdfa7
--- /dev/null
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -0,0 +1,46 @@
+# Copyright (c) 2016, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+"""Build performance test base classes and functionality"""
+from oeqa.utils.commands import runCmd
+
+
+class KernelDropCaches(object):
+ """Container of the functions for dropping kernel caches"""
+ sudo_passwd = None
+
+ @classmethod
+ def check(cls):
+ """Check permssions for dropping kernel caches"""
+ from getpass import getpass
+ from locale import getdefaultlocale
+ cmd = ['sudo', '-k', '-n', 'tee', '/proc/sys/vm/drop_caches']
+ ret = runCmd(cmd, ignore_status=True, data=b'0')
+ if ret.output.startswith('sudo:'):
+ pass_str = getpass(
+ "\nThe script requires sudo access to drop caches between "
+ "builds (echo 3 > /proc/sys/vm/drop_caches).\n"
+ "Please enter your sudo password: ")
+ cls.sudo_passwd = bytes(pass_str, getdefaultlocale()[1])
+
+ @classmethod
+ def drop(cls):
+ """Drop kernel caches"""
+ cmd = ['sudo', '-k']
+ if cls.sudo_passwd:
+ cmd.append('-S')
+ input_data = cls.sudo_passwd + b'\n'
+ else:
+ cmd.append('-n')
+ input_data = b''
+ cmd += ['tee', '/proc/sys/vm/drop_caches']
+ input_data += b'3'
+ runCmd(cmd, data=input_data)
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index 9fb4310..9589dee 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -22,6 +22,7 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
import scriptpath
scriptpath.add_oe_lib_path()
+from oeqa.buildperf import KernelDropCaches
from oeqa.utils.commands import runCmd
@@ -71,6 +72,9 @@ def main(argv=None):
if not pre_run_sanity_check():
return 1
+ # Check our capability to drop caches and ask pass if needed
+ KernelDropCaches.check()
+
return 0
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 09/28] oeqa.buildperf: add BuildPerfTest class
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (7 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 08/28] oeqa.buildperf: functionality to drop kernel caches Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-27 12:12 ` Joshua G Lock
2016-06-24 10:37 ` [PATCH 10/28] oeqa.buildperf: method for measuring system resource usage Markus Lehtonen
` (18 subsequent siblings)
27 siblings, 1 reply; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
The new class will be used as an abstract base class for build
performance tests. This implementation contains some common
functionality used in multiple tests, "copied" from the
build-perf-test.sh shell script.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/__init__.py | 2 +-
meta/lib/oeqa/buildperf/base.py | 78 ++++++++++++++++++++++++++++++++++++-
2 files changed, 78 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py
index bab122e..3bee5b9 100644
--- a/meta/lib/oeqa/buildperf/__init__.py
+++ b/meta/lib/oeqa/buildperf/__init__.py
@@ -10,4 +10,4 @@
# more details.
#
"""Build performance tests"""
-from .base import KernelDropCaches
+from .base import BuildPerfTest, KernelDropCaches
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index 3cbdfa7..3ef0384 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -10,7 +10,16 @@
# more details.
#
"""Build performance test base classes and functionality"""
-from oeqa.utils.commands import runCmd
+import logging
+import os
+import shutil
+import time
+from datetime import datetime
+
+from oeqa.utils.commands import runCmd, get_bb_vars
+
+# Get logger for this module
+log = logging.getLogger('build-perf')
class KernelDropCaches(object):
@@ -44,3 +53,70 @@ class KernelDropCaches(object):
cmd += ['tee', '/proc/sys/vm/drop_caches']
input_data += b'3'
runCmd(cmd, data=input_data)
+
+
+class BuildPerfTest(object):
+ """Base class for build performance tests"""
+ name = None
+ description = None
+
+ def __init__(self, out_dir):
+ self.out_dir = out_dir
+ self.results = {'name':self.name,
+ 'description': self.description,
+ 'status': 'NOTRUN',
+ 'start_time': None,
+ 'elapsed_time': None,
+ 'measurements': []}
+ if not os.path.exists(self.out_dir):
+ os.makedirs(self.out_dir)
+ if not self.name:
+ self.name = self.__class__.__name__
+ self.bb_vars = get_bb_vars()
+
+ def run(self):
+ """Run test"""
+ self.results['status'] = 'FAILED'
+ self.results['start_time'] = datetime.now()
+ self._run()
+ self.results['elapsed_time'] = (datetime.now() -
+ self.results['start_time'])
+ # Test is regarded as completed if it doesn't raise an exception
+ self.results['status'] = 'COMPLETED'
+
+ def _run(self):
+ """Actual test payload"""
+ raise NotImplementedError
+
+ @staticmethod
+ def force_rm(path):
+ """Equivalent of 'rm -rf'"""
+ if os.path.isfile(path) or os.path.islink(path):
+ os.unlink(path)
+ elif os.path.isdir(path):
+ shutil.rmtree(path)
+
+ def rm_tmp(self):
+ """Cleanup temporary/intermediate files and directories"""
+ log.debug("Removing temporary and cache files")
+ for name in ['bitbake.lock', 'conf/sanity_info',
+ self.bb_vars['TMPDIR']]:
+ self.force_rm(name)
+
+ def rm_sstate(self):
+ """Remove sstate directory"""
+ log.debug("Removing sstate-cache")
+ self.force_rm(self.bb_vars['SSTATE_DIR'])
+
+ def rm_cache(self):
+ """Drop bitbake caches"""
+ self.force_rm(self.bb_vars['PERSISTENT_DIR'])
+
+ @staticmethod
+ def sync():
+ """Sync and drop kernel caches"""
+ log.debug("Syncing and dropping kernel caches""")
+ KernelDropCaches.drop()
+ os.sync()
+ # Wait a bit for all the dirty blocks to be written onto disk
+ time.sleep(3)
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* Re: [PATCH 09/28] oeqa.buildperf: add BuildPerfTest class
2016-06-24 10:37 ` [PATCH 09/28] oeqa.buildperf: add BuildPerfTest class Markus Lehtonen
@ 2016-06-27 12:12 ` Joshua G Lock
2016-06-30 14:06 ` Markus Lehtonen
0 siblings, 1 reply; 31+ messages in thread
From: Joshua G Lock @ 2016-06-27 12:12 UTC (permalink / raw)
To: Markus Lehtonen, openembedded-core
On Fri, 2016-06-24 at 13:37 +0300, Markus Lehtonen wrote:
> The new class will be used as an abstract base class for build
> performance tests. This implementation contains some common
> functionality used in multiple tests, "copied" from the
> build-perf-test.sh shell script.
>
> Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
> ---
> meta/lib/oeqa/buildperf/__init__.py | 2 +-
> meta/lib/oeqa/buildperf/base.py | 78
> ++++++++++++++++++++++++++++++++++++-
> 2 files changed, 78 insertions(+), 2 deletions(-)
>
> diff --git a/meta/lib/oeqa/buildperf/__init__.py
> b/meta/lib/oeqa/buildperf/__init__.py
> index bab122e..3bee5b9 100644
> --- a/meta/lib/oeqa/buildperf/__init__.py
> +++ b/meta/lib/oeqa/buildperf/__init__.py
> @@ -10,4 +10,4 @@
> # more details.
> #
> """Build performance tests"""
> -from .base import KernelDropCaches
> +from .base import BuildPerfTest, KernelDropCaches
> diff --git a/meta/lib/oeqa/buildperf/base.py
> b/meta/lib/oeqa/buildperf/base.py
> index 3cbdfa7..3ef0384 100644
> --- a/meta/lib/oeqa/buildperf/base.py
> +++ b/meta/lib/oeqa/buildperf/base.py
> @@ -10,7 +10,16 @@
> # more details.
> #
> """Build performance test base classes and functionality"""
> -from oeqa.utils.commands import runCmd
> +import logging
> +import os
> +import shutil
> +import time
> +from datetime import datetime
> +
> +from oeqa.utils.commands import runCmd, get_bb_vars
> +
> +# Get logger for this module
> +log = logging.getLogger('build-perf')
>
>
> class KernelDropCaches(object):
> @@ -44,3 +53,70 @@ class KernelDropCaches(object):
> cmd += ['tee', '/proc/sys/vm/drop_caches']
> input_data += b'3'
> runCmd(cmd, data=input_data)
> +
> +
> +class BuildPerfTest(object):
> + """Base class for build performance tests"""
> + name = None
> + description = None
> +
> + def __init__(self, out_dir):
> + self.out_dir = out_dir
> + self.results = {'name':self.name,
> + 'description': self.description,
> + 'status': 'NOTRUN',
> + 'start_time': None,
> + 'elapsed_time': None,
> + 'measurements': []}
> + if not os.path.exists(self.out_dir):
> + os.makedirs(self.out_dir)
> + if not self.name:
> + self.name = self.__class__.__name__
> + self.bb_vars = get_bb_vars()
> +
> + def run(self):
> + """Run test"""
> + self.results['status'] = 'FAILED'
> + self.results['start_time'] = datetime.now()
> + self._run()
> + self.results['elapsed_time'] = (datetime.now() -
> + self.results['start_time'])
> + # Test is regarded as completed if it doesn't raise an
> exception
> + self.results['status'] = 'COMPLETED'
> +
> + def _run(self):
> + """Actual test payload"""
> + raise NotImplementedError
> +
> + @staticmethod
> + def force_rm(path):
> + """Equivalent of 'rm -rf'"""
we have oe.path.remove() which does similar.
> + if os.path.isfile(path) or os.path.islink(path):
> + os.unlink(path)
> + elif os.path.isdir(path):
> + shutil.rmtree(path)
> +
> + def rm_tmp(self):
> + """Cleanup temporary/intermediate files and directories"""
> + log.debug("Removing temporary and cache files")
> + for name in ['bitbake.lock', 'conf/sanity_info',
> + self.bb_vars['TMPDIR']]:
> + self.force_rm(name)
> +
> + def rm_sstate(self):
> + """Remove sstate directory"""
> + log.debug("Removing sstate-cache")
> + self.force_rm(self.bb_vars['SSTATE_DIR'])
> +
> + def rm_cache(self):
> + """Drop bitbake caches"""
> + self.force_rm(self.bb_vars['PERSISTENT_DIR'])
> +
> + @staticmethod
> + def sync():
> + """Sync and drop kernel caches"""
> + log.debug("Syncing and dropping kernel caches""")
> + KernelDropCaches.drop()
> + os.sync()
> + # Wait a bit for all the dirty blocks to be written onto
> disk
> + time.sleep(3)
> --
> 2.6.6
>
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [PATCH 09/28] oeqa.buildperf: add BuildPerfTest class
2016-06-27 12:12 ` Joshua G Lock
@ 2016-06-30 14:06 ` Markus Lehtonen
0 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-30 14:06 UTC (permalink / raw)
To: Joshua G Lock, openembedded-core
On Mon, 2016-06-27 at 13:12 +0100, Joshua G Lock wrote:
> On Fri, 2016-06-24 at 13:37 +0300, Markus Lehtonen wrote:
[...SNIP...]
> > +
> > + @staticmethod
> > + def force_rm(path):
> > + """Equivalent of 'rm -rf'"""
>
> we have oe.path.remove() which does similar.
Thanks! I knew we had that but I just didn't remember/find it. Fixed in
the new version of my patchset.
- Markus
^ permalink raw reply [flat|nested] 31+ messages in thread
* [PATCH 10/28] oeqa.buildperf: method for measuring system resource usage
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (8 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 09/28] oeqa.buildperf: add BuildPerfTest class Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 11/28] oeqa.buildperf: add method to log shell commands Markus Lehtonen
` (17 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Extend BuildPerfTest class with a new method for measuring the system
resource usage of a shell command to BuildPerfTest class. For now,
easurement of the elapsed time is done with the Gnu time utility,
similarly to the build-perf-test.sh shell script. And, it currently only
records the elapsed (wall clock).
The measured values (currently, only the elapsed time) is actually a
dictionary, making it possible to extend it with additional resource
values, e.g. cpu time or i/o usage, in the future. In addition to the
actual values of the measurement each record contains a 'name' and
'legend' where name is supposed to function as a common key or id over
test runs, making comparison and trending easier, for example. Legend is
supposed to be a short human readable description.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/base.py | 64 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 62 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index 3ef0384..d608061 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -10,11 +10,14 @@
# more details.
#
"""Build performance test base classes and functionality"""
+import glob
import logging
import os
+import re
import shutil
+import tempfile
import time
-from datetime import datetime
+from datetime import datetime, timedelta
from oeqa.utils.commands import runCmd, get_bb_vars
@@ -55,8 +58,24 @@ class KernelDropCaches(object):
runCmd(cmd, data=input_data)
+def time_cmd(cmd, **kwargs):
+ """TIme a command"""
+ with tempfile.NamedTemporaryFile(mode='w+') as tmpf:
+ timecmd = ['/usr/bin/time', '-v', '-o', tmpf.name]
+ if isinstance(cmd, str):
+ timecmd = ' '.join(timecmd) + ' '
+ timecmd += cmd
+ # TODO: 'ignore_status' could/should be removed when globalres.log is
+ # deprecated. The function would just raise an exception, instead
+ ret = runCmd(timecmd, ignore_status=True, **kwargs)
+ timedata = tmpf.file.read()
+ return ret, timedata
+
+
class BuildPerfTest(object):
"""Base class for build performance tests"""
+ SYSRES = 'sysres'
+
name = None
description = None
@@ -73,6 +92,10 @@ class BuildPerfTest(object):
if not self.name:
self.name = self.__class__.__name__
self.bb_vars = get_bb_vars()
+ # TODO: remove the _failed flag when globalres.log is ditched as all
+ # failures should raise an exception
+ self._failed = False
+ self.cmd_log = os.path.join(self.out_dir, 'commands.log')
def run(self):
"""Run test"""
@@ -82,12 +105,49 @@ class BuildPerfTest(object):
self.results['elapsed_time'] = (datetime.now() -
self.results['start_time'])
# Test is regarded as completed if it doesn't raise an exception
- self.results['status'] = 'COMPLETED'
+ if not self._failed:
+ self.results['status'] = 'COMPLETED'
def _run(self):
"""Actual test payload"""
raise NotImplementedError
+ def measure_cmd_resources(self, cmd, name, legend):
+ """Measure system resource usage of a command"""
+ def str_time_to_timedelta(strtime):
+ """Convert time strig from the time utility to timedelta"""
+ split = strtime.split(':')
+ hours = int(split[0]) if len(split) > 2 else 0
+ mins = int(split[-2])
+ secs, frac = split[-1].split('.')
+ secs = int(secs)
+ microsecs = int(float('0.' + frac) * pow(10, 6))
+ return timedelta(0, hours*3600 + mins*60 + secs, microsecs)
+
+ cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd)
+ log.info("Timing command: %s", cmd_str)
+ with open(self.cmd_log, 'a') as fobj:
+ ret, timedata = time_cmd(cmd, stdout=fobj)
+ if ret.status:
+ log.error("Time will be reported as 0. Command failed: %s",
+ ret.status)
+ etime = timedelta(0)
+ self._failed = True
+ else:
+ match = re.search(r'.*wall clock.*: (?P<etime>.*)\n', timedata)
+ etime = str_time_to_timedelta(match.group('etime'))
+
+ measurement = {'type': self.SYSRES,
+ 'name': name,
+ 'legend': legend}
+ measurement['values'] = {'elapsed_time': etime}
+ self.results['measurements'].append(measurement)
+ nlogs = len(glob.glob(self.out_dir + '/results.log*'))
+ results_log = os.path.join(self.out_dir,
+ 'results.log.{}'.format(nlogs + 1))
+ with open(results_log, 'w') as fobj:
+ fobj.write(timedata)
+
@staticmethod
def force_rm(path):
"""Equivalent of 'rm -rf'"""
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 11/28] oeqa.buildperf: add method to log shell commands
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (9 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 10/28] oeqa.buildperf: method for measuring system resource usage Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 12/28] oeqa.buildperf: add method for measuring file disk usage Markus Lehtonen
` (16 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Add new methods to BuildPerfTest class for running a shell
command and logging its output.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/base.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index d608061..230a7e7 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -112,6 +112,11 @@ class BuildPerfTest(object):
"""Actual test payload"""
raise NotImplementedError
+ def log_cmd_output(self, cmd):
+ """Run a command and log it's output"""
+ with open(self.cmd_log, 'a') as fobj:
+ runCmd(cmd, stdout=fobj)
+
def measure_cmd_resources(self, cmd, name, legend):
"""Measure system resource usage of a command"""
def str_time_to_timedelta(strtime):
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 12/28] oeqa.buildperf: add method for measuring file disk usage
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (10 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 11/28] oeqa.buildperf: add method to log shell commands Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 13/28] oeqa.buildperf: add method for saving buildstats Markus Lehtonen
` (15 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Add a new method to BuildPerfTest class for measuring the disk usage of
a file of directory.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/base.py | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index 230a7e7..e29e9d1 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -75,6 +75,7 @@ def time_cmd(cmd, **kwargs):
class BuildPerfTest(object):
"""Base class for build performance tests"""
SYSRES = 'sysres'
+ DISKUSAGE = 'diskusage'
name = None
description = None
@@ -153,6 +154,24 @@ class BuildPerfTest(object):
with open(results_log, 'w') as fobj:
fobj.write(timedata)
+ def measure_disk_usage(self, path, name, legend):
+ """Estimate disk usage of a file or directory"""
+ # TODO: 'ignore_status' could/should be removed when globalres.log is
+ # deprecated. The function would just raise an exception, instead
+ ret = runCmd(['du', '-s', path], ignore_status=True)
+ if ret.status:
+ log.error("du failed, disk usage will be reported as 0")
+ size = 0
+ self._failed = True
+ else:
+ size = int(ret.output.split()[0])
+ log.debug("Size of %s path is %s", path, size)
+ measurement = {'type': self.DISKUSAGE,
+ 'name': name,
+ 'legend': legend}
+ measurement['values'] = {'size': size}
+ self.results['measurements'].append(measurement)
+
@staticmethod
def force_rm(path):
"""Equivalent of 'rm -rf'"""
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 13/28] oeqa.buildperf: add method for saving buildstats
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (11 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 12/28] oeqa.buildperf: add method for measuring file disk usage Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 14/28] oeqa.buildperf: implement BuildPerfTestRunner class Markus Lehtonen
` (14 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/base.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index e29e9d1..c54b70c 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -172,6 +172,11 @@ class BuildPerfTest(object):
measurement['values'] = {'size': size}
self.results['measurements'].append(measurement)
+ def save_buildstats(self):
+ """Save buildstats"""
+ shutil.move(self.bb_vars['BUILDSTATS_BASE'],
+ os.path.join(self.out_dir, 'buildstats-' + self.name))
+
@staticmethod
def force_rm(path):
"""Equivalent of 'rm -rf'"""
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 14/28] oeqa.buildperf: implement BuildPerfTestRunner class
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (12 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 13/28] oeqa.buildperf: add method for saving buildstats Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 15/28] oeqa.buildperf: add test Test1P1 Markus Lehtonen
` (13 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
The new class is responsible for actually running the tests and
processing their results. This commit also adds a decorator function for
adding new tests. No automatic test discovery, at least yet.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/__init__.py | 3 ++-
meta/lib/oeqa/buildperf/base.py | 47 +++++++++++++++++++++++++++++++++++++
scripts/oe-build-perf-test | 10 ++++++--
3 files changed, 57 insertions(+), 3 deletions(-)
diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py
index 3bee5b9..d6065e8 100644
--- a/meta/lib/oeqa/buildperf/__init__.py
+++ b/meta/lib/oeqa/buildperf/__init__.py
@@ -10,4 +10,5 @@
# more details.
#
"""Build performance tests"""
-from .base import BuildPerfTest, KernelDropCaches
+from .base import (build_perf_test, BuildPerfTest, BuildPerfTestRunner,
+ KernelDropCaches)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index c54b70c..1bfcb71 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -15,8 +15,10 @@ import logging
import os
import re
import shutil
+import socket
import tempfile
import time
+import traceback
from datetime import datetime, timedelta
from oeqa.utils.commands import runCmd, get_bb_vars
@@ -72,6 +74,51 @@ def time_cmd(cmd, **kwargs):
return ret, timedata
+class BuildPerfTestRunner(object):
+ """Runner class for executing the individual tests"""
+ # List of test cases to run
+ test_run_queue = []
+
+ def __init__(self, out_dir):
+ self.results = {}
+ self.out_dir = os.path.abspath(out_dir)
+ if not os.path.exists(self.out_dir):
+ os.makedirs(self.out_dir)
+
+
+ def run_tests(self):
+ """Method that actually runs the tests"""
+ self.results['schema_version'] = 1
+ self.results['tester_host'] = socket.gethostname()
+ start_time = datetime.utcnow()
+ self.results['start_time'] = start_time
+ self.results['tests'] = {}
+
+ for test_class in self.test_run_queue:
+ log.info("Executing test %s: %s", test_class.name,
+ test_class.description)
+
+ test = test_class(self.out_dir)
+ try:
+ test.run()
+ except Exception:
+ # Catch all exceptions. This way e.g buggy tests won't scrap
+ # the whole test run
+ sep = '-' * 5 + ' TRACEBACK ' + '-' * 60 + '\n'
+ tb_msg = sep + traceback.format_exc() + sep
+ log.error("Test execution failed with:\n" + tb_msg)
+ self.results['tests'][test.name] = test.results
+
+ self.results['elapsed_time'] = datetime.utcnow() - start_time
+ return 0
+
+
+def perf_test_case(obj):
+ """Decorator for adding test classes"""
+ BuildPerfTestRunner.test_run_queue.append(obj)
+ return obj
+
+
class BuildPerfTest(object):
"""Base class for build performance tests"""
SYSRES = 'sysres'
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index 9589dee..0a9fc9e 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -18,11 +18,12 @@ import argparse
import logging
import os
import sys
+from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
import scriptpath
scriptpath.add_oe_lib_path()
-from oeqa.buildperf import KernelDropCaches
+from oeqa.buildperf import BuildPerfTestRunner, KernelDropCaches
from oeqa.utils.commands import runCmd
@@ -75,7 +76,12 @@ def main(argv=None):
# Check our capability to drop caches and ask pass if needed
KernelDropCaches.check()
- return 0
+ # Run actual tests
+ out_dir = 'results-{}'.format(datetime.now().strftime('%Y%m%d%H%M%S'))
+ runner = BuildPerfTestRunner(out_dir)
+ ret = runner.run_tests()
+
+ return ret
if __name__ == '__main__':
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 15/28] oeqa.buildperf: add test Test1P1
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (13 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 14/28] oeqa.buildperf: implement BuildPerfTestRunner class Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 16/28] oeqa.buildperf: add test Test1P2 Markus Lehtonen
` (12 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Re-implement "test1_p1" from build-perf-test.sh which measures
'bitbake core-image-sato'.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/__init__.py | 3 ++-
meta/lib/oeqa/buildperf/basic_tests.py | 31 +++++++++++++++++++++++++++++++
2 files changed, 33 insertions(+), 1 deletion(-)
create mode 100644 meta/lib/oeqa/buildperf/basic_tests.py
diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py
index d6065e8..ad5b37c 100644
--- a/meta/lib/oeqa/buildperf/__init__.py
+++ b/meta/lib/oeqa/buildperf/__init__.py
@@ -10,5 +10,6 @@
# more details.
#
"""Build performance tests"""
-from .base import (build_perf_test, BuildPerfTest, BuildPerfTestRunner,
+from .base import (perf_test_case, BuildPerfTest, BuildPerfTestRunner,
KernelDropCaches)
+from .basic_tests import *
diff --git a/meta/lib/oeqa/buildperf/basic_tests.py b/meta/lib/oeqa/buildperf/basic_tests.py
new file mode 100644
index 0000000..95bb8d8
--- /dev/null
+++ b/meta/lib/oeqa/buildperf/basic_tests.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2016, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+"""Basic set of build performance tests"""
+from . import BuildPerfTest, perf_test_case
+
+
+@perf_test_case
+class Test1P1(BuildPerfTest):
+ name = "test1"
+ build_target = 'core-image-sato'
+ description = "Measure wall clock of bitbake {} and size of tmp dir".format(build_target)
+
+ def _run(self):
+ self.log_cmd_output("bitbake {} -c fetchall".format(self.build_target))
+ self.rm_tmp()
+ self.rm_sstate()
+ self.rm_cache()
+ self.sync()
+ self.measure_cmd_resources(['bitbake', self.build_target], 'build',
+ 'bitbake ' + self.build_target)
+ self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir')
+ self.save_buildstats()
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 16/28] oeqa.buildperf: add test Test1P2
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (14 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 15/28] oeqa.buildperf: add test Test1P1 Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 17/28] oeqa.buildperf: add test Test1P3 Markus Lehtonen
` (11 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Re-implement "test1_p2" from build-perf-test.sh which measures
'bitbake virtual/kernel'.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/basic_tests.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/basic_tests.py b/meta/lib/oeqa/buildperf/basic_tests.py
index 95bb8d8..80bdb3f 100644
--- a/meta/lib/oeqa/buildperf/basic_tests.py
+++ b/meta/lib/oeqa/buildperf/basic_tests.py
@@ -29,3 +29,17 @@ class Test1P1(BuildPerfTest):
'bitbake ' + self.build_target)
self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir')
self.save_buildstats()
+
+
+@perf_test_case
+class Test1P2(BuildPerfTest):
+ name = "test12"
+ build_target = 'virtual/kernel'
+ description = "Measure bitbake {}".format(build_target)
+
+ def _run(self):
+ self.log_cmd_output("bitbake {} -c cleansstate".format(
+ self.build_target))
+ self.sync()
+ self.measure_cmd_resources(['bitbake', self.build_target], 'build',
+ 'bitbake ' + self.build_target)
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 17/28] oeqa.buildperf: add test Test1P3
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (15 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 16/28] oeqa.buildperf: add test Test1P2 Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 18/28] oeqa.buildperf: add test Test2 Markus Lehtonen
` (10 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Re-implement "test1_p3" from build-perf-test.sh which measures
'bitbake core-image-sato' with rm_work enabled.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/basic_tests.py | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/basic_tests.py b/meta/lib/oeqa/buildperf/basic_tests.py
index 80bdb3f..a3c1e82 100644
--- a/meta/lib/oeqa/buildperf/basic_tests.py
+++ b/meta/lib/oeqa/buildperf/basic_tests.py
@@ -10,6 +10,8 @@
# more details.
#
"""Basic set of build performance tests"""
+import os
+
from . import BuildPerfTest, perf_test_case
@@ -43,3 +45,27 @@ class Test1P2(BuildPerfTest):
self.sync()
self.measure_cmd_resources(['bitbake', self.build_target], 'build',
'bitbake ' + self.build_target)
+
+
+@perf_test_case
+class Test1P3(BuildPerfTest):
+ name = "test13"
+ build_target = 'core-image-sato'
+ description = "Build {} with rm_work enabled".format(build_target)
+
+ def _run(self):
+ postfile = os.path.join(self.out_dir, 'postfile.conf')
+ with open(postfile, 'w') as fobj:
+ fobj.write('INHERIT += "rm_work"\n')
+ try:
+ self.rm_tmp()
+ self.rm_sstate()
+ self.rm_cache()
+ self.sync()
+ cmd = ['bitbake', '-R', postfile, self.build_target]
+ self.measure_cmd_resources(cmd, 'build',
+ 'bitbake' + self.build_target)
+ self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir')
+ finally:
+ os.unlink(postfile)
+ self.save_buildstats()
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 18/28] oeqa.buildperf: add test Test2
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (16 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 17/28] oeqa.buildperf: add test Test1P3 Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 19/28] oeqa.buildperf: add test Test3 Markus Lehtonen
` (9 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Re-implement "test2" from build-perf-test.sh which measures
'bitbake core-image-sato -c rootfs'.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/basic_tests.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/basic_tests.py b/meta/lib/oeqa/buildperf/basic_tests.py
index a3c1e82..9012a2d 100644
--- a/meta/lib/oeqa/buildperf/basic_tests.py
+++ b/meta/lib/oeqa/buildperf/basic_tests.py
@@ -69,3 +69,17 @@ class Test1P3(BuildPerfTest):
finally:
os.unlink(postfile)
self.save_buildstats()
+
+
+@perf_test_case
+class Test2(BuildPerfTest):
+ name = "test2"
+ build_target = 'core-image-sato'
+ description = "Measure bitbake {} -c rootfs with sstate".format(build_target)
+
+ def _run(self):
+ self.rm_tmp()
+ self.rm_cache()
+ self.sync()
+ cmd = ['bitbake', self.build_target, '-c', 'rootfs']
+ self.measure_cmd_resources(cmd, 'do_rootfs', 'bitbake do_rootfs')
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 19/28] oeqa.buildperf: add test Test3
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (17 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 18/28] oeqa.buildperf: add test Test2 Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 20/28] oeqa.buildperf: add test Test4 Markus Lehtonen
` (8 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Re-implement "test3" from build-perf-test.sh which measures
bitbake parsing time.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/basic_tests.py | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/basic_tests.py b/meta/lib/oeqa/buildperf/basic_tests.py
index 9012a2d..4ebbb77 100644
--- a/meta/lib/oeqa/buildperf/basic_tests.py
+++ b/meta/lib/oeqa/buildperf/basic_tests.py
@@ -83,3 +83,23 @@ class Test2(BuildPerfTest):
self.sync()
cmd = ['bitbake', self.build_target, '-c', 'rootfs']
self.measure_cmd_resources(cmd, 'do_rootfs', 'bitbake do_rootfs')
+
+
+@perf_test_case
+class Test3(BuildPerfTest):
+ name = "test3"
+ description = "Parsing time metrics (bitbake -p)"
+
+ def _run(self):
+ # Drop all caches and parse
+ self.rm_cache()
+ self.force_rm(os.path.join(self.bb_vars['TMPDIR'], 'cache'))
+ self.measure_cmd_resources(['bitbake', '-p'], 'parse_1',
+ 'bitbake -p (no caches)')
+ # Drop tmp/cache
+ self.force_rm(os.path.join(self.bb_vars['TMPDIR'], 'cache'))
+ self.measure_cmd_resources(['bitbake', '-p'], 'parse_2',
+ 'bitbake -p (no tmp/cache)')
+ # Parse with fully cached data
+ self.measure_cmd_resources(['bitbake', '-p'], 'parse_3',
+ 'bitbake -p (cached)')
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 20/28] oeqa.buildperf: add test Test4
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (18 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 19/28] oeqa.buildperf: add test Test3 Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 21/28] oeqa.buildperf: archive build/conf into test results Markus Lehtonen
` (7 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Re-implement "test4" from build-perf-test.sh which measures eSDK metrics.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/basic_tests.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/basic_tests.py b/meta/lib/oeqa/buildperf/basic_tests.py
index 4ebbb77..ada5aba 100644
--- a/meta/lib/oeqa/buildperf/basic_tests.py
+++ b/meta/lib/oeqa/buildperf/basic_tests.py
@@ -11,8 +11,10 @@
#
"""Basic set of build performance tests"""
import os
+import shutil
from . import BuildPerfTest, perf_test_case
+from oeqa.utils.commands import get_bb_vars
@perf_test_case
@@ -103,3 +105,29 @@ class Test3(BuildPerfTest):
# Parse with fully cached data
self.measure_cmd_resources(['bitbake', '-p'], 'parse_3',
'bitbake -p (cached)')
+
+
+@perf_test_case
+class Test4(BuildPerfTest):
+ name = "test4"
+ build_target = 'core-image-sato'
+ description = "eSDK metrics"
+
+ def _run(self):
+ self.log_cmd_output("bitbake {} -c do_populate_sdk_ext".format(
+ self.build_target))
+ self.bb_vars = get_bb_vars(None, self.build_target)
+ tmp_dir = self.bb_vars['TMPDIR']
+ installer = os.path.join(
+ self.bb_vars['SDK_DEPLOY'],
+ self.bb_vars['TOOLCHAINEXT_OUTPUTNAME'] + '.sh')
+ # Measure installer size
+ self.measure_disk_usage(installer, 'installer_bin', 'eSDK installer')
+ # Measure deployment time and deployed size
+ deploy_dir = os.path.join(tmp_dir, 'esdk-deploy')
+ if os.path.exists(deploy_dir):
+ shutil.rmtree(deploy_dir)
+ self.sync()
+ self.measure_cmd_resources([installer, '-y', '-d', deploy_dir],
+ 'deploy', 'eSDK deploy')
+ self.measure_disk_usage(deploy_dir, 'deploy_dir', 'deploy dir')
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 21/28] oeqa.buildperf: archive build/conf into test results
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (19 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 20/28] oeqa.buildperf: add test Test4 Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 22/28] oe-build-perf-test: enable logging into file Markus Lehtonen
` (6 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Have the build/conf directory as part of test results.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/base.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index 1bfcb71..c0318a1 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -94,6 +94,7 @@ class BuildPerfTestRunner(object):
self.results['start_time'] = start_time
self.results['tests'] = {}
+ self.archive_build_conf()
for test_class in self.test_run_queue:
log.info("Executing test %s: %s", test_class.name,
test_class.description)
@@ -112,6 +113,13 @@ class BuildPerfTestRunner(object):
self.results['elapsed_time'] = datetime.utcnow() - start_time
return 0
+ def archive_build_conf(self):
+ """Archive build/conf to test results"""
+ src_dir = os.path.join(os.environ['BUILDDIR'], 'conf')
+ tgt_dir = os.path.join(self.out_dir, 'build', 'conf')
+ os.makedirs(os.path.dirname(tgt_dir))
+ shutil.copytree(src_dir, tgt_dir)
+
def perf_test_case(obj):
"""Decorator for adding test classes"""
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 22/28] oe-build-perf-test: enable logging into file
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (20 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 21/28] oeqa.buildperf: archive build/conf into test results Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 23/28] oeqa.utils: add git module Markus Lehtonen
` (5 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/oe-build-perf-test | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index 0a9fc9e..ca90f69 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -52,6 +52,17 @@ def pre_run_sanity_check():
return True
+def setup_file_logging(log_file):
+ """Setup loggin to file"""
+ log_dir = os.path.dirname(log_file)
+ if not os.path.exists(log_dir):
+ os.makedirs(log_dir)
+ formatter = logging.Formatter(LOG_FORMAT)
+ handler = logging.FileHandler(log_file)
+ handler.setFormatter(formatter)
+ log.addHandler(handler)
+
+
def parse_args(argv):
"""Parse command line arguments"""
parser = argparse.ArgumentParser(
@@ -76,8 +87,11 @@ def main(argv=None):
# Check our capability to drop caches and ask pass if needed
KernelDropCaches.check()
- # Run actual tests
+ # Set-up log file
out_dir = 'results-{}'.format(datetime.now().strftime('%Y%m%d%H%M%S'))
+ setup_file_logging(os.path.join(out_dir, 'output.log'))
+
+ # Run actual tests
runner = BuildPerfTestRunner(out_dir)
ret = runner.run_tests()
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 23/28] oeqa.utils: add git module
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (21 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 22/28] oe-build-perf-test: enable logging into file Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 24/28] oeqa.buildperf: add git revision and branch to result data Markus Lehtonen
` (4 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
A new helper module for easier interaction with Git repositories.
Provides GitRepo class that represents one local Git repository clone.
The GitRepo class currently only has one method, run_cmd(), for running
arbitrary git commands in the repository. More specialized methods for
commonly used git operations can be added later.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/utils/git.py | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
create mode 100644 meta/lib/oeqa/utils/git.py
diff --git a/meta/lib/oeqa/utils/git.py b/meta/lib/oeqa/utils/git.py
new file mode 100644
index 0000000..a4c6741
--- /dev/null
+++ b/meta/lib/oeqa/utils/git.py
@@ -0,0 +1,38 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+"""Git repository interactions"""
+from oeqa.utils.commands import runCmd
+
+
+class GitError(Exception):
+ """Git error handling"""
+ pass
+
+class GitRepo(object):
+ """Class representing a Git repository clone"""
+ def __init__(self, cwd):
+ self.top_dir = self._run_git_cmd_at(['rev-parse', '--show-toplevel'],
+ cwd)
+
+ @staticmethod
+ def _run_git_cmd_at(git_args, cwd, **kwargs):
+ """Run git command at a specified directory"""
+ git_cmd = 'git ' if isinstance(git_args, str) else ['git']
+ git_cmd += git_args
+ ret = runCmd(git_cmd, ignore_status=True, cwd=cwd, **kwargs)
+ if ret.status:
+ cmd_str = git_cmd if isinstance(git_cmd, str) \
+ else ' '.join(git_cmd)
+ raise GitError("'{}' failed with exit code {}: {}".format(
+ cmd_str, ret.status, ret.output))
+ return ret.output.strip()
+
+ def run_cmd(self, git_args):
+ """Run Git command"""
+ return self._run_git_cmd_at(git_args, self.top_dir)
+
+
+
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 24/28] oeqa.buildperf: add git revision and branch to result data
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (22 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 23/28] oeqa.utils: add git module Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 25/28] oe-build-perf-test: implement --globalres-file option Markus Lehtonen
` (3 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
BuildPerfTestRunner determines these from the Git repository under which
it is being run (i.e. where the build directory exists). The branch and
revision may be defined/overridden with OE_BUILDPERFTEST_GIT_BRANCH
and OE_BUILDPERFTEST_GIT_BRANCH environment variables, if needed. This
makes it possible to run the build performance test script even if the
top directory is not a git repository clone, for example.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/base.py | 32 ++++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index c0318a1..e10cbf4 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -22,6 +22,7 @@ import traceback
from datetime import datetime, timedelta
from oeqa.utils.commands import runCmd, get_bb_vars
+from oeqa.utils.git import GitError, GitRepo
# Get logger for this module
log = logging.getLogger('build-perf')
@@ -85,10 +86,41 @@ class BuildPerfTestRunner(object):
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
+ # Get Git parameters
+ try:
+ self.repo = GitRepo('.')
+ except GitError:
+ self.repo = None
+ self.git_rev, self.git_branch = self.get_git_revision()
+ log.info("Using Git branch:revision %s:%s", self.git_branch,
+ self.git_rev)
+
+ def get_git_revision(self):
+ """Get git branch and revision under testing"""
+ rev = os.getenv('OE_BUILDPERFTEST_GIT_REVISION')
+ branch = os.getenv('OE_BUILDPERFTEST_GIT_BRANCH')
+ if not self.repo and (not rev or not branch):
+ log.info("The current working directory doesn't seem to be a Git "
+ "repository clone. You can specify branch and revision "
+ "used in test results with OE_BUILDPERFTEST_GIT_REVISION "
+ "and OE_BUILDPERFTEST_GIT_BRANCH environment variables")
+ else:
+ if not rev:
+ rev = self.repo.run_cmd(['rev-parse', 'HEAD'])
+ if not branch:
+ try:
+ # Strip 11 chars, i.e. 'refs/heads' from the beginning
+ branch = self.repo.run_cmd(['symbolic-ref', 'HEAD'])[11:]
+ except GitError:
+ log.debug('Currently on detached HEAD')
+ branch = None
+ return str(rev), str(branch)
def run_tests(self):
"""Method that actually runs the tests"""
self.results['schema_version'] = 1
+ self.results['git_revision'] = self.git_rev
+ self.results['git_branch'] = self.git_branch
self.results['tester_host'] = socket.gethostname()
start_time = datetime.utcnow()
self.results['start_time'] = start_time
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 25/28] oe-build-perf-test: implement --globalres-file option
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (23 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 24/28] oeqa.buildperf: add git revision and branch to result data Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 26/28] oe-build-perf-test: enable locking Markus Lehtonen
` (2 subsequent siblings)
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Using this option the script appends test results into a 'global results
file'. A CSV-formatted output of the results. This option is to provide
compatibility with the old build-perf-test.sh.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
meta/lib/oeqa/buildperf/base.py | 32 ++++++++++++++++++++++++++++++++
scripts/oe-build-perf-test | 5 +++++
2 files changed, 37 insertions(+)
diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py
index e10cbf4..527563b 100644
--- a/meta/lib/oeqa/buildperf/base.py
+++ b/meta/lib/oeqa/buildperf/base.py
@@ -152,6 +152,38 @@ class BuildPerfTestRunner(object):
os.makedirs(os.path.dirname(tgt_dir))
shutil.copytree(src_dir, tgt_dir)
+ def update_globalres_file(self, filename):
+ """Write results to globalres csv file"""
+ if self.repo:
+ git_tag_rev = self.repo.run_cmd(['describe', self.git_rev])
+ else:
+ git_tag_rev = self.git_rev
+ times = []
+ sizes = []
+ for test in self.results['tests'].values():
+ for measurement in test['measurements']:
+ res_type = measurement['type']
+ values = measurement['values']
+ if res_type == BuildPerfTest.SYSRES:
+ e_sec = values['elapsed_time'].total_seconds()
+ times.append('{:d}:{:02d}:{:.2f}'.format(
+ int(e_sec / 3600),
+ int((e_sec % 3600) / 60),
+ e_sec % 60))
+ elif res_type == BuildPerfTest.DISKUSAGE:
+ sizes.append(str(values['size']))
+ else:
+ log.warning("Unable to handle '%s' values in "
+ "globalres.log", res_type)
+
+ log.debug("Writing globalres log to %s", filename)
+ with open(filename, 'a') as fobj:
+ fobj.write('{},{}:{},{},'.format(self.results['tester_host'],
+ self.results['git_branch'],
+ self.results['git_revision'],
+ git_tag_rev))
+ fobj.write(','.join(times + sizes) + '\n')
+
def perf_test_case(obj):
"""Decorator for adding test classes"""
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index ca90f69..9dd073c 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -70,6 +70,8 @@ def parse_args(argv):
parser.add_argument('-D', '--debug', action='store_true',
help='Enable debug level logging')
+ parser.add_argument('--globalres-file',
+ help="Append results to 'globalres' csv file")
return parser.parse_args(argv)
@@ -94,6 +96,9 @@ def main(argv=None):
# Run actual tests
runner = BuildPerfTestRunner(out_dir)
ret = runner.run_tests()
+ if not ret:
+ if args.globalres_file:
+ runner.update_globalres_file(args.globalres_file)
return ret
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 26/28] oe-build-perf-test: enable locking
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (24 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 25/28] oe-build-perf-test: implement --globalres-file option Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 27/28] oe-build-perf-test: add --out-dir command line argument Markus Lehtonen
2016-06-24 10:37 ` [PATCH 28/28] scripts/contrib: introduce build-perf-test-wrapper.sh Markus Lehtonen
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
Makes it possible to guard that multiple tests are not run in parallel.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/oe-build-perf-test | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index 9dd073c..64873c9 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -15,6 +15,8 @@
#
"""Build performance test script"""
import argparse
+import errno
+import fcntl
import logging
import os
import sys
@@ -33,6 +35,19 @@ logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
log = logging.getLogger()
+def acquire_lock(lock_f):
+ """Acquire flock on file"""
+ log.debug("Acquiring lock %s", os.path.abspath(lock_f.name))
+ try:
+ fcntl.flock(lock_f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except IOError as err:
+ if err.errno == errno.EAGAIN:
+ return False
+ raise
+ log.debug("Lock acquired")
+ return True
+
+
def pre_run_sanity_check():
"""Sanity check of build environment"""
build_dir = os.environ.get("BUILDDIR")
@@ -72,6 +87,9 @@ def parse_args(argv):
help='Enable debug level logging')
parser.add_argument('--globalres-file',
help="Append results to 'globalres' csv file")
+ parser.add_argument('--lock-file', default='./oe-build-perf.lock',
+ metavar='FILENAME',
+ help="Lock file to use")
return parser.parse_args(argv)
@@ -83,6 +101,11 @@ def main(argv=None):
if args.debug:
log.setLevel(logging.DEBUG)
+ lock_f = open(args.lock_file, 'w')
+ if not acquire_lock(lock_f):
+ log.error("Another instance of this script is running, exiting...")
+ return 1
+
if not pre_run_sanity_check():
return 1
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 27/28] oe-build-perf-test: add --out-dir command line argument
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (25 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 26/28] oe-build-perf-test: enable locking Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
2016-06-24 10:37 ` [PATCH 28/28] scripts/contrib: introduce build-perf-test-wrapper.sh Markus Lehtonen
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
The new option defines the output directory for the test result data.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/oe-build-perf-test | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test
index 64873c9..996996b 100755
--- a/scripts/oe-build-perf-test
+++ b/scripts/oe-build-perf-test
@@ -90,6 +90,8 @@ def parse_args(argv):
parser.add_argument('--lock-file', default='./oe-build-perf.lock',
metavar='FILENAME',
help="Lock file to use")
+ parser.add_argument('-o', '--out-dir', default='results-{date}',
+ help="Output directory for test results")
return parser.parse_args(argv)
@@ -113,7 +115,7 @@ def main(argv=None):
KernelDropCaches.check()
# Set-up log file
- out_dir = 'results-{}'.format(datetime.now().strftime('%Y%m%d%H%M%S'))
+ out_dir = args.out_dir.format(date=datetime.now().strftime('%Y%m%d%H%M%S'))
setup_file_logging(os.path.join(out_dir, 'output.log'))
# Run actual tests
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread
* [PATCH 28/28] scripts/contrib: introduce build-perf-test-wrapper.sh
2016-06-24 10:37 [PATCH 00/28] Implement build performance test script in Python Markus Lehtonen
` (26 preceding siblings ...)
2016-06-24 10:37 ` [PATCH 27/28] oe-build-perf-test: add --out-dir command line argument Markus Lehtonen
@ 2016-06-24 10:37 ` Markus Lehtonen
27 siblings, 0 replies; 31+ messages in thread
From: Markus Lehtonen @ 2016-06-24 10:37 UTC (permalink / raw)
To: openembedded-core
A shell script wrapper around oe-build-perf-test script. The purpose of
this wrapper is to act as a executor script, making it possible to run
the tests with a single command. The wrapper script initializes the
build environment, runs oe-build-perf-test and archives the results.
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
---
scripts/contrib/build-perf-test-wrapper.sh | 102 +++++++++++++++++++++++++++++
1 file changed, 102 insertions(+)
create mode 100755 scripts/contrib/build-perf-test-wrapper.sh
diff --git a/scripts/contrib/build-perf-test-wrapper.sh b/scripts/contrib/build-perf-test-wrapper.sh
new file mode 100755
index 0000000..e8e8021
--- /dev/null
+++ b/scripts/contrib/build-perf-test-wrapper.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+#
+# Build performance test script wrapper
+#
+# Copyright (c) 2016, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+#
+# This script is a simple wrapper around the actual build performance tester
+# script. This script initializes the build environment, runs
+# oe-build-perf-test and archives the results.
+
+script=`basename $0`
+usage () {
+ echo "Usage: $script [COMMITISH]"
+}
+
+if [ $# -gt 1 ]; then
+ usage
+ exit 1
+fi
+commitish=$1
+
+echo "Running on `uname -n`"
+
+if ! git_topdir=$(git rev-parse --show-toplevel); then
+ echo "The current working dir doesn't seem to be a git clone. Please cd there before running `basename $0`"
+ exit 1
+fi
+
+cd "$git_topdir"
+
+if [ -n "$commitish" ]; then
+ # Checkout correct revision
+ echo "Checking out $commitish"
+ git fetch &> /dev/null
+ git checkout HEAD^0 &> /dev/null
+ git branch -D $commitish &> /dev/null
+ if ! git checkout -f $commitish &> /dev/null; then
+ echo "Git checkout failed"
+ exit 1
+ fi
+fi
+
+# Setup build environment
+timestamp=`date "+%Y%m%d%H%M%S"`
+git_rev=$(git rev-parse --short HEAD) || exit 1
+base_dir="$git_topdir/build-perf-test"
+build_dir="$base_dir/build-$git_rev-$timestamp"
+results_dir="$base_dir/results-$git_rev-$timestamp"
+globalres_log="$base_dir/globalres.log"
+
+mkdir -p "$base_dir"
+source ./oe-init-build-env $build_dir >/dev/null || exit 1
+
+# Additional config
+auto_conf="$build_dir/conf/auto.conf"
+echo 'MACHINE = "qemux86"' > "$auto_conf"
+echo 'BB_NUMBER_THREADS = "8"' >> "$auto_conf"
+echo 'PARALLEL_MAKE = "-j 8"' >> "$auto_conf"
+echo "DL_DIR = \"$base_dir/downloads\"" >> "$auto_conf"
+# Disabling network sanity check slightly reduces the variance of timing results
+echo 'CONNECTIVITY_CHECK_URIS = ""' >> "$auto_conf"
+# Possibility to define extra settings
+if [ -f "$base_dir/auto.conf.extra" ]; then
+ cat "$base_dir/auto.conf.extra" >> "$auto_conf"
+fi
+
+# Run actual test script
+if ! oe-build-perf-test --out-dir "$results_dir" \
+ --globalres-file "$globalres_log" \
+ --lock-file "$base_dir/oe-build-perf.lock"; then
+ echo "oe-build-perf-test script failed!"
+ exit 1
+fi
+
+echo -ne "\n\n-----------------\n"
+echo "Global results file:"
+echo -ne "\n"
+
+cat "$globalres_log"
+
+echo -ne "\n\n-----------------\n"
+echo "Archiving results dir..."
+archive_dir=~/perf-results/archives
+mkdir -p "$archive_dir"
+results_basename=`basename "$results_dir"`
+results_dirname=`dirname "$results_dir"`
+tar -czf "$archive_dir/`uname -n`-${results_basename}.tar.gz" -C "$results_dirname" "$results_basename"
+
+rm -rf "$build_dir"
+rm -rf "$results_dir"
+
+echo "DONE"
--
2.6.6
^ permalink raw reply related [flat|nested] 31+ messages in thread