* [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py
@ 2016-02-26 14:40 Daniel Istrate
2016-02-26 14:40 ` [PATCH 2/9] selftest: Moved coverage functionality " Daniel Istrate
` (7 more replies)
0 siblings, 8 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
Moved methods to runner and added parameters to them in
order for them to be used by other runners.
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/runner.py | 344 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/oe-selftest | 325 +++------------------------------------------
2 files changed, 359 insertions(+), 310 deletions(-)
create mode 100644 meta/lib/oeqa/runner.py
diff --git a/meta/lib/oeqa/runner.py b/meta/lib/oeqa/runner.py
new file mode 100644
index 0000000..ccb1e3b
--- /dev/null
+++ b/meta/lib/oeqa/runner.py
@@ -0,0 +1,344 @@
+import os
+import logging
+import sys
+import time
+import oeqa.utils.ftools as ftools
+from oeqa.utils.commands import bitbake, get_bb_var, get_test_layer
+
+
+class Tc:
+ """ Holds information about test cases """
+
+ def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
+ self.tcname = tcname
+ self.tcclass = tcclass
+ self.fullpath = '.'.join([tcmodule, tcclass, tcname])
+ # Trim prefix from tcmodule
+ self.tcmodule = tcmodule.split('.')[-1]
+ self.tcid = tcid
+ # A test case can have multiple tags (as tuples) otherwise str will suffice
+ self.tctag = tctag
+
+
+class Runner:
+
+ def __init__(self, caller, base_class, test_mod_dir):
+ """
+ :param caller: eg. selftest, test-recipe (log files will use this name)
+ :param base_class: eg. oeSelfTest, RecipeTests
+ :param test_mod_dir: eg oeqa.selftest, oeqa.recipetests
+ """
+
+ self.caller = caller
+ self.base_class = base_class
+ self.test_mod_dir = test_mod_dir
+ self.log = self.logger_create(self.caller)
+ self.builddir = os.environ.get("BUILDDIR")
+
+ def logger_create(self, log_name):
+ """ Create logger obj with logging file as <log_name-date.log> and symlink to it as <log_name.log> """
+
+ log_link = '%s.log' % log_name
+ log_file = '%s-%s.log' % (log_name, time.strftime("%Y-%m-%d_%H:%M:%S"))
+
+ if os.path.exists(log_link):
+ os.remove(log_link)
+ os.symlink(log_file, log_link)
+
+ log = logging.getLogger(log_name)
+ log.setLevel(logging.DEBUG)
+
+ fh = logging.FileHandler(filename=log_file, mode='w')
+ fh.setLevel(logging.DEBUG)
+
+ ch = logging.StreamHandler(sys.stdout)
+ ch.setLevel(logging.INFO)
+
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', "%Y-%m-%d %H:%M:%S")
+ fh.setFormatter(formatter)
+ ch.setFormatter(formatter)
+
+ log.addHandler(fh)
+ log.addHandler(ch)
+
+ return log
+
+ def preflight_check(self):
+ """ Check that the environment is sourced, meta-selftest included in bblayers and current dir is BUILDDIR """
+
+ self.log.info("Checking that everything is in order before running the tests")
+
+ if not self.builddir:
+ self.log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
+ return False
+
+ if os.getcwd() != self.builddir:
+ self.log.info("Changing cwd to %s" % self.builddir)
+ os.chdir(self.builddir)
+
+ if "meta-selftest" not in get_bb_var("BBLAYERS"):
+ self.log.error("You don't seem to have the meta-selftest layer in BBLAYERS")
+ return False
+
+ self.log.info("Running bitbake -p")
+ bitbake("-p")
+
+ return True
+
+ def add_include(self, include_files, include_to):
+ """ Include in include_to (local.conf, bblayers.conf) the specified files """
+
+ include_msg_header = "# include added by %s\n" % self.caller
+ include_msg = include_msg_header
+ if isinstance(include_files, (list, tuple)):
+ for f in include_files:
+ include_msg += 'include %s\n' % f
+ else:
+ include_msg += 'include %s\n' % include_files
+
+ if include_msg_header not in ftools.read_file(os.path.join(self.builddir, 'conf', include_to)):
+ self.log.info("Adding: %s in %s" % (include_files, include_to))
+ ftools.append_file(os.path.join(self.builddir, 'conf', include_to), include_msg)
+
+ def remove_include(self, remove_files, remove_from):
+ """ Remove from remove_from (local.conf, bblayers.conf) the specified files """
+
+ if self.builddir is None:
+ return
+
+ remove_msg_header = "# include added by %s\n" % self.caller
+ remove_msg = remove_msg_header
+ if isinstance(remove_files, (list, tuple)):
+ for f in remove_files:
+ remove_msg += 'include %s\n' % f
+ else:
+ remove_msg += 'include %s\n' % remove_files
+
+ if remove_msg_header in ftools.read_file(os.path.join(self.builddir, 'conf', remove_from)):
+ self.log.info("Removing the include from %s" % remove_from)
+ ftools.remove_from_file(os.path.join(self.builddir, 'conf', remove_from), remove_msg)
+
+ def remove_inc_files(self, remove_inc_list):
+ """ Remove remove_inc_list from BUILDDIR/conf/ """
+
+ # Also remove the test_recipe.inc if available
+ try:
+ for root, _, files in os.walk(get_test_layer()):
+ pass
+ for f in files:
+ if f == 'test_recipe.inc':
+ os.remove(os.path.join(root, f))
+ except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set
+ pass
+
+ # log.info('Removing: %s' % remove_inc_list)
+ for incl_file in remove_inc_list:
+ try:
+ os.remove(os.path.join(self.builddir, 'conf', incl_file))
+ except:
+ pass
+
+ # FIXME: this method duplicates some of the code. Keep it here for now
+ def get_tests(self, exclusive_modules=[], include_hidden=False):
+ testslist = []
+ prefix = self.test_mod_dir.__name__
+ for x in exclusive_modules:
+ testslist.append('.'.join([prefix, x]))
+ if not testslist:
+ for testpath in self.test_mod_dir.__path__:
+ files = sorted([f for f in os.listdir(testpath) if f.endswith('.py') and not (f.startswith('_') and not
+ include_hidden) and not f.startswith('__') and f != 'base.py'])
+ for f in files:
+ module = '.'.join([prefix, os.path.splitext(f)[0]])
+ if module not in testslist:
+ testslist.append(module)
+
+ return testslist
+
+ def get_tests_from_module(self, tmod):
+ """ Get a list of Tcs from module extending the base_class """
+
+ tlist = []
+
+ try:
+ import importlib
+ modlib = importlib.import_module(tmod)
+ for mod in vars(modlib).values():
+ if isinstance(mod, type(self.base_class)) and issubclass(mod, self.base_class) and mod is not self.base_class:
+ for test in dir(mod):
+ if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
+ # Get test case id and feature tag
+ # NOTE: if testcase decorator or feature tag are not set it will throw an error
+ try:
+ tid = vars(mod)[test].test_case
+ except:
+ print 'DEBUG: tc id missing for ' + str(test)
+ tid = None
+ try:
+ ttag = vars(mod)[test].tag__feature
+ except:
+ # print 'DEBUG: feature tag missing for ' + str(test)
+ ttag = None
+
+ tlist.append(Tc(test, mod.__name__, mod.__module__, tid, ttag))
+ except:
+ pass
+
+ return tlist
+
+ def get_all_tests(self):
+ """ Get all test from test_mod_dir (eg. oeqa.selftest) that extends base_class (eg. oeSelfTest)"""
+
+ tmodules = set()
+ testlist = []
+ prefix = self.test_mod_dir.__name__
+
+ # Get all the test modules (except the hidden ones)
+ for tpath in self.test_mod_dir.__path__:
+ files = sorted([f for f in os.listdir(tpath) if f.endswith('.py') and not
+ f.startswith(('_', '__')) and f != 'base.py'])
+ for f in files:
+ tmodules.add('.'.join([prefix, os.path.splitext(f)[0]]))
+
+ # Get all the tests from modules
+ tmodules = sorted(list(tmodules))
+
+ for tmod in tmodules:
+ testlist += self.get_tests_from_module(tmod)
+
+ return testlist
+
+ def list_tests(self):
+ """ List all available tests """
+
+ ts = self.get_all_tests()
+
+ print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module')
+ print '_' * 150
+ for t in ts:
+ if isinstance(t.tctag, (tuple, list)):
+ print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t.tcid, ', '.join(t.tctag), t.tcname, t.tcclass, t.tcmodule)
+ else:
+ print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t.tcid, t.tctag, t.tcname, t.tcclass, t.tcmodule)
+ print '_' * 150
+ print 'Total found:\t %s' % len(ts)
+
+ def list_tags(self):
+ """
+ Get all tags set to test cases
+ Note: This is useful when setting tags to test cases
+ Note: The list of tags should be kept as minimal as possible
+ """
+
+ tags = set()
+ all_tests = self.get_all_tests()
+
+ for tc in all_tests:
+ if isinstance(tc.tctag, (tuple, list)):
+ tags.update(set(tc.tctag))
+ else:
+ tags.add(tc.tctag)
+
+ print 'Tags:\t%s' % ', '.join(str(x) for x in tags)
+
+ def get_testsuite_by(self, criteria, keyword):
+ """
+ Get a testsuite based on 'keyword'
+ :param criteria: name, class, module, id, tag
+ :param keyword: a list of tests, classes, modules, ids, tags
+ """
+ ts = set()
+ all_tests = self.get_all_tests()
+
+ if criteria == 'name':
+ for tc in all_tests:
+ if tc.tcname in keyword:
+ ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+
+ elif criteria == 'class':
+ for tc in all_tests:
+ if tc.tcclass in keyword:
+ ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+
+ elif criteria == 'module':
+ for tc in all_tests:
+ if tc.tcmodule in keyword:
+ ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+ elif criteria == 'id':
+ for tc in all_tests:
+ if str(tc.tcid) in keyword:
+ ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+ elif criteria == 'tag':
+ for tc in all_tests:
+ # tc can have multiple tags (as list or tuple) otherwise as str
+ if isinstance(tc.tctag, (list, tuple)):
+ for tag in tc.tctag:
+ if str(tag) in keyword:
+ ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+ elif str(tc.tctag) in keyword:
+ ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
+
+ return sorted(list(ts))
+
+ # NOTE: I would probably merge create_testsuite_by and get_testsuite_by into one method
+ def create_testsuite_by(self, criteria, keyword):
+ """
+ Create a testsuite based on 'keyword'
+ :param criteria: name, class, module, id, tag
+ :param keyword: a list of tests, classes, modules, ids, tags
+ """
+
+ ts = set()
+ all_tests = self.get_all_tests()
+
+ if criteria == 'name':
+ for tc in all_tests:
+ if tc.tcname in keyword:
+ ts.add(tc.fullpath)
+
+ elif criteria == 'class':
+ for tc in all_tests:
+ if tc.tcclass in keyword:
+ ts.add(tc.fullpath)
+
+ elif criteria == 'module':
+ for tc in all_tests:
+ if tc.tcmodule in keyword:
+ ts.add(tc.fullpath)
+ elif criteria == 'id':
+ for tc in all_tests:
+ if str(tc.tcid) in keyword:
+ ts.add(tc.fullpath)
+ elif criteria == 'tag':
+ for tc in all_tests:
+ # tc can have multiple tags (as list or tuple) otherwise as str
+ if isinstance(tc.tctag, (list, tuple)):
+ for tag in tc.tctag:
+ if str(tag) in keyword:
+ ts.add(tc.fullpath)
+ elif tc.tctag in keyword:
+ ts.add(tc.fullpath)
+
+ return sorted(list(ts))
+
+ def list_testsuite_by(self, criteria, keyword):
+ """
+ List a testsuite based on 'keyword'
+ :param criteria: name, class, module, id, tag
+ :param keyword: a list of tests, classes, modules, ids, tags
+ """
+
+ ts = self.get_testsuite_by(criteria, keyword)
+
+ print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module')
+ print '_' * 150
+ for t in ts:
+ if isinstance(t[1], (tuple, list)):
+ print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4])
+ else:
+ print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t
+ print '_' * 150
+ print 'Filtering by:\t %s' % criteria
+ print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword)
+ print 'Total found:\t %s' % len(ts)
+
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index 4eb404b..b592511 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -44,30 +44,11 @@ import oeqa.utils.ftools as ftools
from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
from oeqa.selftest.base import oeSelfTest, get_available_machines
-def logger_create():
- log_file = "oe-selftest-" + t.strftime("%Y-%m-%d_%H:%M:%S") + ".log"
- if os.path.exists("oe-selftest.log"): os.remove("oe-selftest.log")
- os.symlink(log_file, "oe-selftest.log")
+from oeqa.runner import Runner
- log = logging.getLogger("selftest")
- log.setLevel(logging.DEBUG)
+test_runner = Runner('selftest', oeSelfTest, oeqa.selftest)
+log = test_runner.log
- fh = logging.FileHandler(filename=log_file, mode='w')
- fh.setLevel(logging.DEBUG)
-
- ch = logging.StreamHandler(sys.stdout)
- ch.setLevel(logging.INFO)
-
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
- fh.setFormatter(formatter)
- ch.setFormatter(formatter)
-
- log.addHandler(fh)
- log.addHandler(ch)
-
- return log
-
-log = logger_create()
def get_args_parser():
description = "Script that runs unit tests agains bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information."
@@ -91,284 +72,6 @@ def get_args_parser():
return parser
-def preflight_check():
-
- log.info("Checking that everything is in order before running the tests")
-
- if not os.environ.get("BUILDDIR"):
- log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
- return False
-
- builddir = os.environ.get("BUILDDIR")
- if os.getcwd() != builddir:
- log.info("Changing cwd to %s" % builddir)
- os.chdir(builddir)
-
- if not "meta-selftest" in get_bb_var("BBLAYERS"):
- log.error("You don't seem to have the meta-selftest layer in BBLAYERS")
- return False
-
- log.info("Running bitbake -p")
- runCmd("bitbake -p")
-
- return True
-
-def add_include():
- builddir = os.environ.get("BUILDDIR")
- if "#include added by oe-selftest.py" \
- not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
- log.info("Adding: \"include selftest.inc\" in local.conf")
- ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
- "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
-
- if "#include added by oe-selftest.py" \
- not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
- log.info("Adding: \"include bblayers.inc\" in bblayers.conf")
- ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \
- "\n#include added by oe-selftest.py\ninclude bblayers.inc")
-
-def remove_include():
- builddir = os.environ.get("BUILDDIR")
- if builddir is None:
- return
- if "#include added by oe-selftest.py" \
- in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
- log.info("Removing the include from local.conf")
- ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
- "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
-
- if "#include added by oe-selftest.py" \
- in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
- log.info("Removing the include from bblayers.conf")
- ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \
- "\n#include added by oe-selftest.py\ninclude bblayers.inc")
-
-def remove_inc_files():
- try:
- os.remove(os.path.join(os.environ.get("BUILDDIR"), "conf/selftest.inc"))
- for root, _, files in os.walk(get_test_layer()):
- for f in files:
- if f == 'test_recipe.inc':
- os.remove(os.path.join(root, f))
- except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set
- pass
-
- for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']:
- try:
- os.remove(os.path.join(os.environ.get("BUILDDIR"), incl_file))
- except:
- pass
-
-def get_tests(exclusive_modules=[], include_hidden=False):
- testslist = []
- for x in exclusive_modules:
- testslist.append('oeqa.selftest.' + x)
- if not testslist:
- for testpath in oeqa.selftest.__path__:
- files = sorted([f for f in os.listdir(testpath) if f.endswith('.py') and not (f.startswith('_') and not include_hidden) and not f.startswith('__') and f != 'base.py'])
- for f in files:
- module = 'oeqa.selftest.' + f[:-3]
- if module not in testslist:
- testslist.append(module)
-
- return testslist
-
-
-class Tc:
- def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
- self.tcname = tcname
- self.tcclass = tcclass
- self.tcmodule = tcmodule
- self.tcid = tcid
- # A test case can have multiple tags (as list or as tuples) otherwise str suffice
- self.tctag = tctag
- self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
-
-
-def get_tests_from_module(tmod):
- tlist = []
- prefix = 'oeqa.selftest.'
-
- try:
- import importlib
- modlib = importlib.import_module(tmod)
- for mod in vars(modlib).values():
- if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
- for test in dir(mod):
- if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
- # Get test case id and feature tag
- # NOTE: if testcase decorator or feature tag not set will throw error
- try:
- tid = vars(mod)[test].test_case
- except:
- print 'DEBUG: tc id missing for ' + str(test)
- tid = None
- try:
- ttag = vars(mod)[test].tag__feature
- except:
- # print 'DEBUG: feature tag missing for ' + str(test)
- ttag = None
-
- # NOTE: for some reason lstrip() doesn't work for mod.__module__
- tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
- except:
- pass
-
- return tlist
-
-
-def get_all_tests():
- tmodules = set()
- testlist = []
- prefix = 'oeqa.selftest.'
-
- # Get all the test modules (except the hidden ones)
- for tpath in oeqa.selftest.__path__:
- files = sorted([f for f in os.listdir(tpath) if f.endswith('.py') and not
- f.startswith(('_', '__')) and f != 'base.py'])
- for f in files:
- tmodules.add(prefix + f.rstrip('.py'))
-
- # Get all the tests from modules
- tmodules = sorted(list(tmodules))
-
- for tmod in tmodules:
- testlist += get_tests_from_module(tmod)
-
- return testlist
-
-
-def create_testsuite_by(criteria, keyword):
- # Create a testsuite based on 'keyword'
- # criteria: name, class, module, id, tag
- # keyword: a list of tests, classes, modules, ids, tags
- # NOTE: globing would be nice?
-
- ts = set()
- all_tests = get_all_tests()
-
- if criteria == 'name':
- for tc in all_tests:
- if tc.tcname in keyword:
- ts.add(tc.fullpath)
-
- elif criteria == 'class':
- for tc in all_tests:
- if tc.tcclass in keyword:
- ts.add(tc.fullpath)
-
- elif criteria == 'module':
- for tc in all_tests:
- if tc.tcmodule in keyword:
- ts.add(tc.fullpath)
- elif criteria == 'id':
- for tc in all_tests:
- if str(tc.tcid) in keyword:
- ts.add(tc.fullpath)
- elif criteria == 'tag':
- for tc in all_tests:
- # tc can have multiple tags (as list or tuple) otherwise as str
- if isinstance(tc.tctag, (list, tuple)):
- for tag in tc.tctag:
- if str(tag) in keyword:
- ts.add(tc.fullpath)
- elif tc.tctag in keyword:
- ts.add(tc.fullpath)
-
- return sorted(list(ts))
-
-
-def get_testsuite_by(criteria, keyword):
- # Get a testsuite based on 'keyword'
- # criteria: name, class, module, id, tag
- # keyword: a list of tests, classes, modules, ids, tags
- # NOTE: globing would be nice?
- ts = set()
- all_tests = get_all_tests()
-
- if criteria == 'name':
- for tc in all_tests:
- if tc.tcname in keyword:
- ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
-
- elif criteria == 'class':
- for tc in all_tests:
- if tc.tcclass in keyword:
- ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
-
- elif criteria == 'module':
- for tc in all_tests:
- if tc.tcmodule in keyword:
- ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
- elif criteria == 'id':
- for tc in all_tests:
- if str(tc.tcid) in keyword:
- ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
- elif criteria == 'tag':
- for tc in all_tests:
- # tc can have multiple tags (as list or tuple) otherwise as str
- if isinstance(tc.tctag, (list, tuple)):
- for tag in tc.tctag:
- if str(tag) in keyword:
- ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
- elif str(tc.tctag) in keyword:
- ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
-
- return sorted(list(ts))
-
-
-def list_testsuite_by(criteria, keyword):
- # Get a testsuite based on 'keyword'
- # criteria: name, class, module, id, tag
- # keyword: a list of tests, classes, modules, ids, tags
- # NOTE: globing would be nice?
-
- ts = get_testsuite_by(criteria, keyword)
-
- print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module')
- print '_' * 150
- for t in ts:
- if isinstance(t[1], (tuple, list)):
- print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4])
- else:
- print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t
- print '_' * 150
- print 'Filtering by:\t %s' % criteria
- print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword)
- print 'Total found:\t %s' % len(ts)
-
-
-def list_tests():
- # List all available oe-selftest tests
-
- ts = get_all_tests()
-
- print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module')
- print '_' * 150
- for t in ts:
- if isinstance(t.tctag, (tuple, list)):
- print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t.tcid, ', '.join(t.tctag), t.tcname, t.tcclass, t.tcmodule)
- else:
- print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t.tcid, t.tctag, t.tcname, t.tcclass, t.tcmodule)
- print '_' * 150
- print 'Total found:\t %s' % len(ts)
-
-
-def list_tags():
- # Get all tags set to test cases
- # This is useful when setting tags to test cases
- # The list of tags should be kept as minimal as possible
- tags = set()
- all_tests = get_all_tests()
-
- for tc in all_tests:
- if isinstance(tc.tctag, (tuple, list)):
- tags.update(set(tc.tctag))
- else:
- tags.add(tc.tctag)
-
- print 'Tags:\t%s' % ', '.join(str(x) for x in tags)
-
def coverage_setup(run_tests, run_all_tests):
""" Set up the coverage measurement for the testcases to be run """
builddir = os.environ.get("BUILDDIR")
@@ -438,7 +141,7 @@ def main():
else:
criteria = args.run_tests_by[0]
keyword = args.run_tests_by[1:]
- ts = create_testsuite_by(criteria, keyword)
+ ts = test_runner.create_testsuite_by(criteria, keyword)
if args.list_tests_by and len(args.list_tests_by) >= 2:
valid_options = ['name', 'class', 'module', 'id', 'tag']
@@ -448,20 +151,20 @@ def main():
else:
criteria = args.list_tests_by[0]
keyword = args.list_tests_by[1:]
- list_testsuite_by(criteria, keyword)
+ test_runner.list_testsuite_by(criteria, keyword)
if args.list_tests:
- list_tests()
+ test_runner.list_tests()
if args.list_tags:
- list_tags()
+ test_runner.list_tags()
if args.list_allclasses:
args.list_modules = True
if args.list_modules:
log.info('Listing all available test modules:')
- testslist = get_tests(include_hidden=True)
+ testslist = test_runner.get_tests(include_hidden=True)
for test in testslist:
module = test.split('.')[-1]
info = ''
@@ -485,13 +188,13 @@ def main():
pass
if args.run_tests or args.run_all_tests or args.run_tests_by:
- if not preflight_check():
+ if not test_runner.preflight_check():
return 1
if args.run_tests_by:
testslist = ts
else:
- testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
+ testslist = test_runner.get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
suite = unittest.TestSuite()
loader = unittest.TestLoader()
@@ -508,7 +211,8 @@ def main():
log.error("Failed to import %s" % test)
log.error(e)
return 1
- add_include()
+ test_runner.add_include(['machine.inc', 'selftest.inc'], 'local.conf')
+ test_runner.add_include('bblayers.inc', 'bblayers.conf')
if args.machine:
# Custom machine sets only weak default values (??=) for MACHINE in machine.inc
@@ -595,6 +299,7 @@ if __name__ == "__main__":
import traceback
traceback.print_exc(5)
finally:
- remove_include()
- remove_inc_files()
+ test_runner.remove_include(['machine.inc', 'selftest.inc'], 'local.conf')
+ test_runner.remove_include('bblayers.inc', 'bblayers.conf')
+ test_runner.remove_inc_files(['selftest.inc', 'machine.inc', 'bblayers.inc'])
sys.exit(ret)
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 2/9] selftest: Moved coverage functionality from oe-selftest to oeqa/runner.py
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
2016-02-26 14:40 ` [PATCH 3/9] selftest: Moved list_classes, list_modules, run methods to runner.py Daniel Istrate
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/runner.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++-
scripts/oe-selftest | 106 +----------------------------------------------
2 files changed, 108 insertions(+), 106 deletions(-)
diff --git a/meta/lib/oeqa/runner.py b/meta/lib/oeqa/runner.py
index ccb1e3b..5bfd4d1 100644
--- a/meta/lib/oeqa/runner.py
+++ b/meta/lib/oeqa/runner.py
@@ -2,6 +2,7 @@ import os
import logging
import sys
import time
+import unittest
import oeqa.utils.ftools as ftools
from oeqa.utils.commands import bitbake, get_bb_var, get_test_layer
@@ -35,7 +36,8 @@ class Runner:
self.log = self.logger_create(self.caller)
self.builddir = os.environ.get("BUILDDIR")
- def logger_create(self, log_name):
+ @staticmethod
+ def logger_create(log_name):
""" Create logger obj with logging file as <log_name-date.log> and symlink to it as <log_name.log> """
log_link = '%s.log' % log_name
@@ -342,3 +344,107 @@ class Runner:
print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword)
print 'Total found:\t %s' % len(ts)
+ @staticmethod
+ def coverage_setup(run_tests, run_all_tests):
+ """ Set up the coverage measurement for the testcases to be run """
+ builddir = os.environ.get("BUILDDIR")
+ coveragerc = "%s/.coveragerc" % builddir
+ data_file = "%s/.coverage." % builddir
+ data_file += ((run_tests and '.'.join(run_tests)) or (run_all_tests and "all_tests") or "")
+ if os.path.isfile(data_file):
+ os.remove(data_file)
+ with open(coveragerc, 'w') as cps:
+ cps.write("[run]\n")
+ cps.write("data_file = %s\n" % data_file)
+ cps.write("branch = True\n")
+ # Measure just BBLAYERS, scripts and bitbake folders
+ cps.write("source = \n")
+ for layer in get_bb_var('BBLAYERS').split():
+ cps.write(" %s\n" % layer)
+ corebase = get_bb_var('COREBASE')
+ cps.write(" %s\n" % os.path.join(corebase, 'scripts'))
+ cps.write(" %s\n" % os.path.join(corebase, 'bitbake'))
+
+ return coveragerc
+
+ @staticmethod
+ def coverage_report():
+ """ Loads the coverage data gathered and reports it back """
+ try:
+ # Coverage4 uses coverage.Coverage
+ from coverage import Coverage
+ except:
+ # Coverage under version 4 uses coverage.coverage
+ from coverage import coverage as Coverage
+
+ import cStringIO as StringIO
+ from coverage.misc import CoverageException
+
+ cov_output = StringIO.StringIO()
+ # Creating the coverage data with the setting from the configuration file
+ cov = Coverage(config_file=os.environ.get('COVERAGE_PROCESS_START'))
+ try:
+ # Load data from the data file specified in the configuration
+ cov.load()
+ # Store report data in a StringIO variable
+ cov.report(file = cov_output, show_missing=False)
+ print "\n%s" % cov_output.getvalue()
+ except CoverageException as e:
+ # Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception
+ print "%s" % str(e)
+ finally:
+ cov_output.close()
+
+ @classmethod
+ def buildResultClass(cls, args):
+ """Build a Result Class to use in the testcase execution"""
+
+ class StampedResult(unittest.TextTestResult):
+ """
+ Custom TestResult that prints the time when a test starts. As oe-selftest
+ can take a long time (ie a few hours) to run, timestamps help us understand
+ what tests are taking a long time to execute.
+ If coverage is required, this class executes the coverage setup and reporting.
+ """
+ def startTest(self, test):
+ import time
+ self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
+ super(StampedResult, self).startTest(test)
+
+ def startTestRun(self):
+ """ Setup coverage before running any testcase """
+ if args.coverage:
+ try:
+ # check if user can do coverage
+ import coverage
+ print "Coverage is enabled"
+
+ # In case the user has not set the variable COVERAGE_PROCESS_START,
+ # create a default one and export it. The COVERAGE_PROCESS_START
+ # value indicates where the coverage configuration file resides
+ # More info on https://pypi.python.org/pypi/coverage
+ if not os.environ.get('COVERAGE_PROCESS_START'):
+ os.environ['COVERAGE_PROCESS_START'] = cls.coverage_setup(args.run_tests, args.run_all_tests)
+
+ self.coverage_installed = True
+ except:
+ print '\n'.join(["python coverage is not installed",
+ "Make sure your coverage takes into account sub-process",
+ "More info on https://pypi.python.org/pypi/coverage"])
+ self.coverage_installed = False
+
+ def stopTestRun(self):
+ """ Report coverage data after the testcases are run """
+
+ if args.coverage and self.coverage_installed:
+ with open(os.environ['COVERAGE_PROCESS_START']) as ccf:
+ print "Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START')
+ print "==========================="
+ print "\n%s" % "".join(ccf.readlines())
+
+ print "Coverage Report"
+ print "==============="
+
+ cls.coverage_report()
+
+ return StampedResult
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index b592511..9f71154 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -72,56 +72,6 @@ def get_args_parser():
return parser
-def coverage_setup(run_tests, run_all_tests):
- """ Set up the coverage measurement for the testcases to be run """
- builddir = os.environ.get("BUILDDIR")
- coveragerc = "%s/.coveragerc" % builddir
- data_file = "%s/.coverage." % builddir
- data_file += ((run_tests and '.'.join(run_tests)) or
- (run_all_tests and "all_tests") or "")
- if os.path.isfile(data_file):
- os.remove(data_file)
- with open(coveragerc, 'w') as cps:
- cps.write("[run]\n")
- cps.write("data_file = %s\n" % data_file)
- cps.write("branch = True\n")
- # Measure just BBLAYERS, scripts and bitbake folders
- cps.write("source = \n")
- for layer in get_bb_var('BBLAYERS').split():
- cps.write(" %s\n" % layer)
- cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__)))
- cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
-
- return coveragerc
-
-def coverage_report():
- """ Loads the coverage data gathered and reports it back """
- try:
- # Coverage4 uses coverage.Coverage
- from coverage import Coverage
- except:
- # Coverage under version 4 uses coverage.coverage
- from coverage import coverage as Coverage
-
- import cStringIO as StringIO
- from coverage.misc import CoverageException
-
- cov_output = StringIO.StringIO()
- # Creating the coverage data with the setting from the configuration file
- cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START'))
- try:
- # Load data from the data file specified in the configuration
- cov.load()
- # Store report data in a StringIO variable
- cov.report(file = cov_output, show_missing=False)
- log.info("\n%s" % cov_output.getvalue())
- except CoverageException as e:
- # Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception
- log.warn("%s" % str(e))
- finally:
- cov_output.close()
-
-
def main():
parser = get_args_parser()
args = parser.parse_args()
@@ -199,7 +149,7 @@ def main():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
loader.sortTestMethodsUsing = None
- runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args))
+ runner = unittest.TextTestRunner(verbosity=2, resultclass=test_runner.buildResultClass(args))
# we need to do this here, otherwise just loading the tests
# will take 2 minutes (bitbake -e calls)
oeSelfTest.testlayer_path = get_test_layer()
@@ -237,60 +187,6 @@ def main():
else:
return 1
-def buildResultClass(args):
- """Build a Result Class to use in the testcase execution"""
-
- class StampedResult(unittest.TextTestResult):
- """
- Custom TestResult that prints the time when a test starts. As oe-selftest
- can take a long time (ie a few hours) to run, timestamps help us understand
- what tests are taking a long time to execute.
- If coverage is required, this class executes the coverage setup and reporting.
- """
- def startTest(self, test):
- import time
- self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
- super(StampedResult, self).startTest(test)
-
- def startTestRun(self):
- """ Setup coverage before running any testcase """
- if args.coverage:
- try:
- # check if user can do coverage
- import coverage
- log.info("Coverage is enabled")
-
- # In case the user has not set the variable COVERAGE_PROCESS_START,
- # create a default one and export it. The COVERAGE_PROCESS_START
- # value indicates where the coverage configuration file resides
- # More info on https://pypi.python.org/pypi/coverage
- if not os.environ.get('COVERAGE_PROCESS_START'):
- os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.run_tests, args.run_all_tests)
-
- self.coverage_installed = True
- except:
- log.warn('\n'.join(["python coverage is not installed",
- "Make sure your coverage takes into account sub-process",
- "More info on https://pypi.python.org/pypi/coverage"]))
- self.coverage_installed = False
-
- def stopTestRun(self):
- """ Report coverage data after the testcases are run """
-
- if args.coverage and self.coverage_installed:
- with open(os.environ['COVERAGE_PROCESS_START']) as ccf:
- log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START'))
- log.info("===========================")
- log.info("\n%s" % "".join(ccf.readlines()))
-
- log.info("Coverage Report")
- log.info("===============")
-
- coverage_report()
-
- return StampedResult
-
-
if __name__ == "__main__":
try:
ret = main()
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 3/9] selftest: Moved list_classes, list_modules, run methods to runner.py
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
2016-02-26 14:40 ` [PATCH 2/9] selftest: Moved coverage functionality " Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
2016-02-26 14:40 ` [PATCH 4/9] oeqa/utils/commands: Added 3 new methods for recipes Daniel Istrate
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/runner.py | 59 ++++++++++++++++++++++++++++++++++++++++++++
scripts/oe-selftest | 65 ++++++++++---------------------------------------
2 files changed, 72 insertions(+), 52 deletions(-)
diff --git a/meta/lib/oeqa/runner.py b/meta/lib/oeqa/runner.py
index 5bfd4d1..e729478 100644
--- a/meta/lib/oeqa/runner.py
+++ b/meta/lib/oeqa/runner.py
@@ -344,6 +344,65 @@ class Runner:
print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword)
print 'Total found:\t %s' % len(ts)
+ def list_modules(self):
+ """ List all available modules """
+
+ self.log.info('Listing all available test modules:')
+ testslist = self.get_tests(include_hidden=True)
+ for test in testslist:
+ module = test.split('.')[-1]
+ print module + ' (hidden)' if module.startswith('_') else module
+
+ def list_all_classes(self):
+ """ List all tests with their corresponding class and module (Hierarchy format) """
+
+ testslist = self.get_tests(include_hidden=True)
+ for test in testslist:
+ module = test.split('.')[-1]
+ print module + ' (hidden)' if module.startswith('_') else module
+ try:
+ import importlib
+ modlib = importlib.import_module(test)
+ for v in vars(modlib):
+ t = vars(modlib)[v]
+ if isinstance(t, type(self.base_class)) and issubclass(t, self.base_class) and t != self.base_class:
+ print " --", v
+ for method in dir(t):
+ if method.startswith("test_") and callable(vars(t)[method]):
+ print " -- --", method
+
+ except (AttributeError, ImportError) as e:
+ print e
+ pass
+
+ def run(self, testlist, args):
+ """
+ :param testlist: ['oeqa.selftest.archiver', 'oeqa.selftest.bblayers', ..]
+ :param args: the args the calling script was invoked with (used by coverage)
+ """
+
+ suite = unittest.TestSuite()
+ loader = unittest.TestLoader()
+ loader.sortTestMethodsUsing = None
+ runner = unittest.TextTestRunner(verbosity=2, resultclass=self.buildResultClass(args))
+
+ for test in testlist:
+ self.log.info("Loading tests from: %s" % test)
+ try:
+ suite.addTests(loader.loadTestsFromName(test))
+ except AttributeError as e:
+ self.log.error("Failed to import %s" % test)
+ self.log.error(e)
+ return 1
+
+ result = runner.run(suite)
+ self.log.info("Finished")
+
+ if result.wasSuccessful():
+ return 0
+ else:
+ return 1
+
@staticmethod
def coverage_setup(run_tests, run_all_tests):
""" Set up the coverage measurement for the testcases to be run """
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index 9f71154..ba73502 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -27,11 +27,6 @@
import os
import sys
-import unittest
-import logging
-import argparse
-import subprocess
-import time as t
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
import scriptpath
@@ -40,8 +35,7 @@ scriptpath.add_oe_lib_path()
import argparse_oe
import oeqa.selftest
-import oeqa.utils.ftools as ftools
-from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
+from oeqa.utils.commands import get_bb_var, get_test_layer
from oeqa.selftest.base import oeSelfTest, get_available_machines
from oeqa.runner import Runner
@@ -102,40 +96,23 @@ def main():
criteria = args.list_tests_by[0]
keyword = args.list_tests_by[1:]
test_runner.list_testsuite_by(criteria, keyword)
+ return 0
if args.list_tests:
test_runner.list_tests()
+ return 0
if args.list_tags:
test_runner.list_tags()
+ return 0
if args.list_allclasses:
- args.list_modules = True
+ test_runner.list_all_classes()
+ return 0
if args.list_modules:
- log.info('Listing all available test modules:')
- testslist = test_runner.get_tests(include_hidden=True)
- for test in testslist:
- module = test.split('.')[-1]
- info = ''
- if module.startswith('_'):
- info = ' (hidden)'
- print module + info
- if args.list_allclasses:
- try:
- import importlib
- modlib = importlib.import_module(test)
- for v in vars(modlib):
- t = vars(modlib)[v]
- if isinstance(t, type(oeSelfTest)) and issubclass(t, oeSelfTest) and t!=oeSelfTest:
- print " --", v
- for method in dir(t):
- if method.startswith("test_") and callable(vars(t)[method]):
- print " -- --", method
-
- except (AttributeError, ImportError) as e:
- print e
- pass
+ test_runner.list_modules()
+ return 0
if args.run_tests or args.run_all_tests or args.run_tests_by:
if not test_runner.preflight_check():
@@ -146,21 +123,10 @@ def main():
else:
testslist = test_runner.get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
- suite = unittest.TestSuite()
- loader = unittest.TestLoader()
- loader.sortTestMethodsUsing = None
- runner = unittest.TextTestRunner(verbosity=2, resultclass=test_runner.buildResultClass(args))
# we need to do this here, otherwise just loading the tests
# will take 2 minutes (bitbake -e calls)
oeSelfTest.testlayer_path = get_test_layer()
- for test in testslist:
- log.info("Loading tests from: %s" % test)
- try:
- suite.addTests(loader.loadTestsFromName(test))
- except AttributeError as e:
- log.error("Failed to import %s" % test)
- log.error(e)
- return 1
+
test_runner.add_include(['machine.inc', 'selftest.inc'], 'local.conf')
test_runner.add_include('bblayers.inc', 'bblayers.conf')
@@ -170,22 +136,17 @@ def main():
log.info('Custom machine mode enabled. MACHINE set to %s' % args.machine)
if args.machine == 'random':
os.environ['CUSTOMMACHINE'] = 'random'
- result = runner.run(suite)
+ result = test_runner.run(testslist, args)
else: # all
machines = get_available_machines()
for m in machines:
log.info('Run tests with custom MACHINE set to: %s' % m)
os.environ['CUSTOMMACHINE'] = m
- result = runner.run(suite)
+ result = test_runner.run(testslist, args)
else:
- result = runner.run(suite)
-
- log.info("Finished")
+ result = test_runner.run(testslist, args)
- if result.wasSuccessful():
- return 0
- else:
- return 1
+ return result
if __name__ == "__main__":
try:
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 4/9] oeqa/utils/commands: Added 3 new methods for recipes
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
2016-02-26 14:40 ` [PATCH 2/9] selftest: Moved coverage functionality " Daniel Istrate
2016-02-26 14:40 ` [PATCH 3/9] selftest: Moved list_classes, list_modules, run methods to runner.py Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
2016-02-26 14:40 ` [PATCH 5/9] scripts: test-recipe Tool for running tests on recipes Daniel Istrate
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
get_all_available_recipes, is_recipe_valid, get_tasks_for_recipe
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/utils/commands.py | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 32e001c..6ae09d2 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -165,6 +165,26 @@ def get_test_layer():
break
return testlayer
+
+def get_all_available_recipes():
+ ret = bitbake('-s')
+ available_recipes = re.findall(r'\n(\S+)\s+:', ret.output)
+
+ return available_recipes
+
+
+def is_recipe_valid(recipe):
+ return recipe in get_all_available_recipes()
+
+
+def get_tasks_for_recipe(recipe):
+ """ Get available tasks for recipe """
+ ret = bitbake('-c listtasks %s' % recipe)
+ tasks = re.findall(':\s+do_(\S+)\s+', ret.output)
+
+ return tasks
+
+
def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'):
os.makedirs(os.path.join(templayerdir, 'conf'))
with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 5/9] scripts: test-recipe Tool for running tests on recipes
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
` (2 preceding siblings ...)
2016-02-26 14:40 ` [PATCH 4/9] oeqa/utils/commands: Added 3 new methods for recipes Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
2016-02-26 14:40 ` [PATCH 6/9] recipetests: base - Base Class for Test Recipes Daniel Istrate
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
It shares many functionality with oe-selftest.
It requires an aditional argument --recipe <recipe>
in order to know on which recipe to test against.
Test recipes should be located at:
meta/lib/oeqa/recipetests/
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
scripts/test-recipe | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 168 insertions(+)
create mode 100755 scripts/test-recipe
diff --git a/scripts/test-recipe b/scripts/test-recipe
new file mode 100755
index 0000000..3fe80a5
--- /dev/null
+++ b/scripts/test-recipe
@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2016, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION: Will run recipe tests on each provided recipe.
+#
+# USAGE: recipe-test --recipes <recipe> --run-all-tests
+#
+# OPTIONS: --recipes <recipe1>
+# --list-tests
+# --list-tests-by <criteria> <keyword>
+# --run-all-tests
+# --run-tests-by <criteria> <keyword>
+#
+# NOTE: tests are located in lib/oeqa/recipetests
+#
+# AUTHOR: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
+#
+
+
+import sys
+import os
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
+import scriptpath
+scriptpath.add_bitbake_lib_path()
+scriptpath.add_oe_lib_path()
+import argparse_oe
+
+import oeqa.recipetests
+from oeqa.utils.commands import get_test_layer, get_bb_var, bitbake, is_recipe_valid
+from oeqa.selftest.base import oeSelfTest
+from oeqa.recipetests.base import RecipeTests
+
+from oeqa.runner import Runner
+
+test_runner = Runner('test-recipe', RecipeTests, oeqa.recipetests)
+log = test_runner.log
+
+
+def get_args_parser():
+ description = 'Will run recipe tests on provided recipe.'
+ parser = argparse_oe.ArgumentParser(description=description)
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument('--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None,
+ help='Select what tests to run (modules, classes or test methods). '
+ 'Format should be: <module>.<class>.<test_method>')
+ group.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False,
+ help='Run all (unhidden) tests')
+ group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False,
+ help='List all available test modules.')
+ group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False,
+ help='List all available test classes.')
+ parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
+ group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
+ help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
+ group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
+ help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
+ group.add_argument('--list-tests', required=False, action="store_true", dest="list_tests", default=False,
+ help='List all available tests.')
+ group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
+ help='List all tags that have been set to test cases.')
+ parser.add_argument('-r', '--recipes', nargs='+', default=False, dest='recipes',
+ help='recipe(s) to run tests against.')
+ return parser
+
+
+def main():
+ parser = get_args_parser()
+ args = parser.parse_args()
+
+ if args.list_tests:
+ test_runner.list_tests()
+ return 0
+
+ if args.list_tags:
+ test_runner.list_tags()
+ return 0
+
+ if args.list_allclasses:
+ test_runner.list_all_classes()
+ return 0
+
+ if args.list_modules:
+ test_runner.list_modules()
+ return 0
+
+ if args.list_tests_by and len(args.list_tests_by) >= 2:
+ valid_options = ['name', 'class', 'module', 'id', 'tag']
+ if args.list_tests_by[0] not in valid_options:
+ print '--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0]
+ return 1
+ else:
+ criteria = args.list_tests_by[0]
+ keyword = args.list_tests_by[1:]
+ test_runner.list_testsuite_by(criteria, keyword)
+ return 0
+
+ if args.run_tests_by and len(args.run_tests_by) >= 2:
+ valid_options = ['name', 'class', 'module', 'id', 'tag']
+ if args.run_tests_by[0] not in valid_options:
+ print '--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0]
+ return 1
+ else:
+ criteria = args.run_tests_by[0]
+ keyword = args.run_tests_by[1:]
+ ts = test_runner.create_testsuite_by(criteria, keyword)
+
+ if not ((args.run_tests or args.run_all_tests or args.run_tests_by) and args.recipes):
+ print 'Please specify the recipe(s) to tests against ( -r or --recipes).'
+ return 1
+
+ # Do we want to be able to test multiple recipes?
+ if not is_recipe_valid(args.recipes[0]):
+ print '"%s" is not a valid recipe. Make sure it shows up in "bitbake -s". Check your spelling.' % args.recipes[0]
+ return 1
+
+ if args.run_tests or args.run_all_tests or args.run_tests_by:
+ if not test_runner.preflight_check():
+ return 1
+
+ os.environ['TESTRECIPE'] = args.recipes[0]
+ log.info('Running tests for recipe "%s" ...' % args.recipes[0])
+
+ if args.run_tests_by:
+ testslist = ts
+ else:
+ testslist = test_runner.get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
+
+ # we need to do this here, otherwise just loading the tests
+ # will take 2 minutes (bitbake -e calls)
+ oeSelfTest.testlayer_path = get_test_layer()
+
+ test_runner.add_include(['required.inc', 'testrecipe.inc'], 'local.conf')
+ test_runner.add_include('bblayers.inc', 'bblayers.conf')
+
+ result = test_runner.run(testslist, args)
+
+ return result
+
+if __name__ == '__main__':
+
+ try:
+ ret = main()
+ except Exception:
+ ret = 1
+ import traceback
+ traceback.print_exc(5)
+ finally:
+ test_runner.remove_include(['required.inc', 'testrecipe.inc'], 'local.conf')
+ test_runner.remove_include('bblayers.inc', 'bblayers.conf')
+ test_runner.remove_inc_files(['testrecipe.inc', 'required.inc', 'bblayers.inc'])
+ sys.exit(ret)
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 6/9] recipetests: base - Base Class for Test Recipes
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
` (3 preceding siblings ...)
2016-02-26 14:40 ` [PATCH 5/9] scripts: test-recipe Tool for running tests on recipes Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
2016-02-26 14:40 ` [PATCH 7/9] recipetests: buildrecipe: Test suite for recipe tests Daniel Istrate
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
Test Recipes should inherit this class.
It sets up a custom tmp dir for the recipe under test.
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/recipetests/base.py | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
create mode 100644 meta/lib/oeqa/recipetests/base.py
diff --git a/meta/lib/oeqa/recipetests/base.py b/meta/lib/oeqa/recipetests/base.py
new file mode 100644
index 0000000..148c123
--- /dev/null
+++ b/meta/lib/oeqa/recipetests/base.py
@@ -0,0 +1,25 @@
+import os
+import shutil
+from oeqa.selftest.base import oeSelfTest
+import oeqa.utils.ftools as ftools
+
+
+class RecipeTests(oeSelfTest):
+
+ def __init__(self, methodName="runTest"):
+ super(RecipeTests, self).__init__(methodName)
+ self.testinc_path = os.path.join(self.builddir, 'conf/testrecipe.inc')
+ self.required_path = os.path.join(self.builddir, 'conf/required.inc')
+ self.testrecipe = os.getenv("TESTRECIPE")
+
+ # Use a clean TMPDIR for each run
+ tmpdir_name = self.builddir + '/tmp_' + self.testrecipe
+ shutil.rmtree(tmpdir_name, ignore_errors=True)
+
+ feature = 'TMPDIR = "%s"\n' % tmpdir_name
+ self.set_required_config(feature)
+
+ # write to <builddir>/conf/requred.inc
+ def set_required_config(self, data):
+ self.log.debug("Writing to: %s\n%s\n" % (self.required_path, data))
+ ftools.write_file(self.required_path, data)
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 7/9] recipetests: buildrecipe: Test suite for recipe tests
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
` (4 preceding siblings ...)
2016-02-26 14:40 ` [PATCH 6/9] recipetests: base - Base Class for Test Recipes Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
2016-02-26 14:40 ` [PATCH 8/9] oeqa/utils/commands: Added method - get_all_bbappends Daniel Istrate
2016-02-26 14:40 ` [PATCH 9/9] recipetests: buildrecipe: Test combinations of bbappend Daniel Istrate
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
Includes tests for:
1. test build recipe for all major architectures
2. test rebuild recipe from sstate 1 (with sstate)
3. test_rebuild_recipe_from_sstate_2 (without sstate)
4. test cleaning operations on recipe
5. test force major tasks on recipe
fix for [YOCTO #6370]
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/recipetests/buildrecipe.py | 62 ++++++++++++++++++++++++++++++++
1 file changed, 62 insertions(+)
create mode 100644 meta/lib/oeqa/recipetests/buildrecipe.py
diff --git a/meta/lib/oeqa/recipetests/buildrecipe.py b/meta/lib/oeqa/recipetests/buildrecipe.py
new file mode 100644
index 0000000..c303b29
--- /dev/null
+++ b/meta/lib/oeqa/recipetests/buildrecipe.py
@@ -0,0 +1,62 @@
+from oeqa.recipetests.base import RecipeTests
+from oeqa.selftest.base import get_available_machines
+from oeqa.utils.commands import bitbake, get_tasks_for_recipe
+import logging
+
+
+class BuildRecipeTests(RecipeTests):
+
+ log = logging.getLogger('test-recipe.build-recipe')
+
+ def test_build_recipe_for_all_major_architectures(self):
+ """ Build the recipe with all major architectures(qemux86, qemux86-64, qemuarm, qemuppc, qemumips) """
+
+ machines = get_available_machines()
+ qemu_machines = [m for m in machines if 'qemu' in m]
+
+ # Build the recipe for all major architectures
+ for m in qemu_machines:
+ self.log.info('Building recipe "%s" for architecture "%s"' % (self.testrecipe, m))
+ self.write_config('MACHINE = "%s"' % m)
+ bitbake(self.testrecipe)
+
+ def test_rebuild_recipe_from_sstate_1(self):
+ """ Rebuild the recipe from sstate with sstate file for the recipe """
+ bitbake(self.testrecipe)
+ bitbake('-c clean %s' % self.testrecipe)
+ bitbake(self.testrecipe)
+
+ def test_rebuild_recipe_from_sstate_2(self):
+ """ Rebuild the recipe from sstate without sstate file for the recipe """
+ bitbake(self.testrecipe)
+ bitbake('-c cleansstate %s' % self.testrecipe)
+ bitbake(self.testrecipe)
+
+ def test_cleaning_operations_on_recipe(self):
+ """ Perform cleaning operations on the recipe(cleansstate, clean, cleanall) """
+
+ clean_tasks = ['cleansstate', 'clean', 'cleanall']
+
+ for task in clean_tasks:
+ bitbake(self.testrecipe)
+ self.log.info('Performing %s for recipe %s' % (task, self.testrecipe))
+ bitbake('-c %s %s' % (task, self.testrecipe))
+
+ def test_force_major_tasks_on_recipe(self):
+ """ Force all major tasks on the recipe (bitbake -C <task> <recipe>) """
+ major_tasks = ['unpack', 'patch', 'configure', 'compile', 'install', 'populate_sysroot', 'package',
+ 'package_write_rpm', 'package_write_deb', 'package_write_ipk']
+
+ feature = 'PACKAGE_CLASSES = "package_rpm package_deb package_ipk"\n'
+ self.write_config(feature)
+
+ available_tasks = get_tasks_for_recipe(self.testrecipe)
+
+ for task in major_tasks:
+ # Check if task is available for recipe
+ if task not in available_tasks:
+ self.log.warning('Task %s not available for recipe %s' % (task, self.testrecipe))
+ continue
+ # Force task on recipe
+ self.log.info('Forcing task %s' % task)
+ bitbake('-C %s %s' % (task, self.testrecipe))
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 8/9] oeqa/utils/commands: Added method - get_all_bbappends
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
` (5 preceding siblings ...)
2016-02-26 14:40 ` [PATCH 7/9] recipetests: buildrecipe: Test suite for recipe tests Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
2016-02-26 14:40 ` [PATCH 9/9] recipetests: buildrecipe: Test combinations of bbappend Daniel Istrate
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
This method runs 'bitbake-layers show-appends' and
returns a dictionary {recipe: [bbappends], ..}
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/utils/commands.py | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 6ae09d2..8fdb80d 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -184,6 +184,28 @@ def get_tasks_for_recipe(recipe):
return tasks
+def get_all_bbappends():
+ """ Get all bbappends (bitbake-layers show-appends)
+ :return:a dict with {recipe: [bbappends]}
+ """
+ ret_dict = {}
+ ret = runCmd('bitbake-layers show-appends')
+
+ for line in ret.output.splitlines():
+ bb = re.findall('^(\S+\.bb.*):', line)
+ bbappend = re.findall('^\s+(\S+\.bbappend)', line)
+ if bb:
+ key = bb[0]
+ value_list = []
+ continue
+ if bbappend:
+ value = bbappend[0]
+ value_list.append(value)
+ ret_dict[key] = value_list
+ continue
+
+ return ret_dict
+
def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'):
os.makedirs(os.path.join(templayerdir, 'conf'))
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 9/9] recipetests: buildrecipe: Test combinations of bbappend
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
` (6 preceding siblings ...)
2016-02-26 14:40 ` [PATCH 8/9] oeqa/utils/commands: Added method - get_all_bbappends Daniel Istrate
@ 2016-02-26 14:40 ` Daniel Istrate
7 siblings, 0 replies; 9+ messages in thread
From: Daniel Istrate @ 2016-02-26 14:40 UTC (permalink / raw)
To: openembedded-core
Selectively use each combination of .bbappend files with the recipe;
All the combinations should not break the recipe build
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
---
meta/lib/oeqa/recipetests/buildrecipe.py | 59 +++++++++++++++++++++++++++++++-
1 file changed, 58 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oeqa/recipetests/buildrecipe.py b/meta/lib/oeqa/recipetests/buildrecipe.py
index c303b29..900d042 100644
--- a/meta/lib/oeqa/recipetests/buildrecipe.py
+++ b/meta/lib/oeqa/recipetests/buildrecipe.py
@@ -1,7 +1,9 @@
from oeqa.recipetests.base import RecipeTests
from oeqa.selftest.base import get_available_machines
-from oeqa.utils.commands import bitbake, get_tasks_for_recipe
+from oeqa.utils.commands import bitbake, get_tasks_for_recipe, get_bb_var, ftools, get_all_bbappends
import logging
+import os
+import random
class BuildRecipeTests(RecipeTests):
@@ -60,3 +62,58 @@ class BuildRecipeTests(RecipeTests):
# Force task on recipe
self.log.info('Forcing task %s' % task)
bitbake('-C %s %s' % (task, self.testrecipe))
+
+ def test_combinations_of_bbappend(self):
+ """ Selectively use each combination of .bbappend files with the recipe """
+
+ test_recipe_pv = get_bb_var('PV', self.testrecipe)
+ recipe_append_file = self.testrecipe + '_' + test_recipe_pv + '.bbappend'
+
+ bbappend_msgs = {1: 'msg 1', 2: 'msg 2', 3: 'msg 3', 4: 'msg 4'}
+ bbappend_files = {}
+
+ # Install all bbappends for recipe
+ for i in bbappend_msgs:
+ recipe_append_dir = self.testrecipe + '_test_' + str(i)
+ recipe_append_path = os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir, recipe_append_file)
+ os.mkdir(os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir))
+ feature = 'SUMMARY += "%s"\n' % bbappend_msgs[i]
+ ftools.write_file(recipe_append_path, feature)
+ bbappend_files[i] = recipe_append_path
+
+ self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, 'recipes-test',
+ self.testrecipe + '_test_*'))
+
+ test_recipe_bb = '%s_%s.bb' % (self.testrecipe, test_recipe_pv)
+ all_bbappends = get_all_bbappends()
+ self.log.info('All bbappends for recipe %s: %s' % (self.testrecipe, all_bbappends.get(test_recipe_bb)))
+
+ # Build recipe with all bbappends
+ bitbake(self.testrecipe)
+
+ # Mask two random bbappends (some times it might be the same one, which is ok)
+ for i in range(len(bbappend_files)):
+ choice1 = random.choice(bbappend_msgs.keys())
+ choice2 = random.choice(bbappend_msgs.keys())
+ mask1 = bbappend_files.get(choice1)
+ mask2 = bbappend_files.get(choice2)
+
+ feature = 'BBMASK = "%s"\n' % mask1
+ feature += 'BBMASK += "%s"\n' % mask2
+ self.write_config(feature)
+ self.log.info('Applied MASKs [%s, %s]' % (mask1, mask2))
+
+ # Make sure the masked bbappends don't show up
+ current_bbappends = get_all_bbappends()
+ self.log.info('Current bbappends for recipe %s: %s' % (self.testrecipe, current_bbappends.get(test_recipe_bb)))
+ self.assertNotIn(mask1, current_bbappends.get(test_recipe_bb))
+ self.assertNotIn(mask2, current_bbappends.get(test_recipe_bb))
+
+ # Make sure the summary was updated
+ current_summary = get_bb_var('SUMMARY', self.testrecipe)
+ self.log.info('Current summary: "%s"' % current_summary)
+ self.assertNotIn(bbappend_msgs.get(choice1), current_summary)
+ self.assertNotIn(bbappend_msgs.get(choice2), current_summary)
+
+ # Build recipe with custom bbappends
+ bitbake(self.testrecipe)
--
2.1.0
^ permalink raw reply related [flat|nested] 9+ messages in thread
end of thread, other threads:[~2016-02-26 14:37 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-02-26 14:40 [PATCH 1/9] selftest: Moved method from oe-selftest to oeqa/runner.py Daniel Istrate
2016-02-26 14:40 ` [PATCH 2/9] selftest: Moved coverage functionality " Daniel Istrate
2016-02-26 14:40 ` [PATCH 3/9] selftest: Moved list_classes, list_modules, run methods to runner.py Daniel Istrate
2016-02-26 14:40 ` [PATCH 4/9] oeqa/utils/commands: Added 3 new methods for recipes Daniel Istrate
2016-02-26 14:40 ` [PATCH 5/9] scripts: test-recipe Tool for running tests on recipes Daniel Istrate
2016-02-26 14:40 ` [PATCH 6/9] recipetests: base - Base Class for Test Recipes Daniel Istrate
2016-02-26 14:40 ` [PATCH 7/9] recipetests: buildrecipe: Test suite for recipe tests Daniel Istrate
2016-02-26 14:40 ` [PATCH 8/9] oeqa/utils/commands: Added method - get_all_bbappends Daniel Istrate
2016-02-26 14:40 ` [PATCH 9/9] recipetests: buildrecipe: Test combinations of bbappend Daniel Istrate
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.