* [PATCH v2 2/3] OETestContext: order test cases by prio & depends
[not found] <20210126180143.605303-1-kweihmann@outlook.com>
@ 2021-01-26 18:01 ` Konrad Weihmann
2021-01-26 18:01 ` [PATCH v2 3/3] oeqa: add tests for OETestPriority Konrad Weihmann
1 sibling, 0 replies; 2+ messages in thread
From: Konrad Weihmann @ 2021-01-26 18:01 UTC (permalink / raw)
To: openembedded-core; +Cc: Konrad Weihmann
Order the overall test case list in the resulting test suite
by priority (by default all test cases do have a priority of 50).
In a second round move test cases with test dependencies accordingly.
New method 'orderSuites' can be explictly overridden to disable
usage of the new 'priority' decorator.
Signed-off-by: Konrad Weihmann <kweihmann@outlook.com>
---
v2:
- mind test cases without any decorator
- rename new method to orderSuites
meta/lib/oeqa/core/context.py | 30 +++++++++++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 2abe353d27..2ca9ec0c56 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -66,6 +66,34 @@ class OETestContext(object):
if (cid + '.').startswith(skip + '.'):
setattr(tclass, 'setUpHooker', skipfuncgen('Skip by the command line argument "%s"' % skip))
+ def orderSuites(self, suites):
+ _tests = {}
+ for _test in suites._tests:
+ _prio = 50
+ _depends = []
+ for x in getattr(_test, "decorators", []):
+ if hasattr(x, "depends"):
+ _depends += getattr(x, "depends")
+ if hasattr(x, "priority"):
+ _prio = getattr(x, "priority")
+ _tests[_test.id()] = {"prio": _prio, "deps": _depends, "ref": _test}
+
+ def set_suite_prio(suites, suite):
+ for dep in suite["deps"]:
+ if dep not in suites:
+ try:
+ dep = [x for x in suites.keys() if x.endswith(dep)][0]
+ except:
+ continue
+ suite["prio"] = max(suite["prio"], set_suite_prio(suites, suites[dep]) + 1)
+ return suite["prio"]
+
+ for k, v in _tests.items():
+ _tests[k]["prio"] = set_suite_prio(_tests, v)
+
+ suites._tests = [x[1]["ref"] for x in sorted(_tests.items(), key=lambda tup: tup[1]["prio"])]
+ return suites
+
def loadTests(self, module_paths, modules=[], tests=[],
modules_manifest="", modules_required=[], **kwargs):
if modules_manifest:
@@ -73,7 +101,7 @@ class OETestContext(object):
self.loader = self.loaderClass(self, module_paths, modules, tests,
modules_required, **kwargs)
- self.suites = self.loader.discover()
+ self.suites = self.orderSuites(self.loader.discover())
def prepareSuite(self, suites, processes):
return suites
--
2.25.1
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [PATCH v2 3/3] oeqa: add tests for OETestPriority
[not found] <20210126180143.605303-1-kweihmann@outlook.com>
2021-01-26 18:01 ` [PATCH v2 2/3] OETestContext: order test cases by prio & depends Konrad Weihmann
@ 2021-01-26 18:01 ` Konrad Weihmann
1 sibling, 0 replies; 2+ messages in thread
From: Konrad Weihmann @ 2021-01-26 18:01 UTC (permalink / raw)
To: openembedded-core; +Cc: Konrad Weihmann
Add test cases for the newly introduced test decorator OETestPriority
Signed-off-by: Konrad Weihmann <kweihmann@outlook.com>
---
meta/lib/oeqa/core/tests/cases/priority.py | 61 +++++++++++++++
meta/lib/oeqa/core/tests/test_decorators.py | 82 ++++++++++++++++++++-
2 files changed, 142 insertions(+), 1 deletion(-)
create mode 100644 meta/lib/oeqa/core/tests/cases/priority.py
diff --git a/meta/lib/oeqa/core/tests/cases/priority.py b/meta/lib/oeqa/core/tests/cases/priority.py
new file mode 100644
index 0000000000..cb48eb0424
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/priority.py
@@ -0,0 +1,61 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.priority import OETestPriority
+from oeqa.core.decorator.depends import OETestDepends
+
+class PriorityTest(OETestCase):
+
+ @OETestPriority(0)
+ def testDependsBoundsLower(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestPriority(00)
+ def testDependsBoundsUpper(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestPriority(-1)
+ def testDependsBoundsLowerOff(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestPriority(100)
+ def testDependsBoundsUpperOff(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestPriority("abc")
+ def testDependsBoundsType(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestPriority(80)
+ def testPriority80(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testPriority80'])
+ def testPriority80plus(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestPriority(30)
+ def testPriority30(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testPriority30'])
+ @OETestPriority(25)
+ def testPriority25(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testPriority25'])
+ @OETestPriority(15)
+ def testPriority15(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ def testPriorityNoDepends(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ def testPriorityNoDepends2(self):
+ self.assertTrue(True, msg='How is this possible?')
+
+ @OETestDepends(['testPriorityNoDepends'])
+ def testPriorityDependsNoPrio(self):
+ self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/test_decorators.py b/meta/lib/oeqa/core/tests/test_decorators.py
index b798bf7d33..5a81be1c68 100755
--- a/meta/lib/oeqa/core/tests/test_decorators.py
+++ b/meta/lib/oeqa/core/tests/test_decorators.py
@@ -11,7 +11,7 @@ import unittest
from common import setup_sys_path, TestBase
setup_sys_path()
-from oeqa.core.exception import OEQADependency
+from oeqa.core.exception import OEQADependency, OEQAException
from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames, getSuiteCasesIDs
class TestTagDecorator(TestBase):
@@ -133,5 +133,85 @@ class TestTimeoutDecorator(TestBase):
msg = "OETestTimeout didn't restore SIGALRM"
self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
+class TestPriorityDecorator(TestBase):
+ modules = ['priority']
+
+ def test_depends_bounds_lower(self):
+ tests = ['priority.PriorityTest.testDependsBoundsLowerOff']
+ self.assertRaises(OEQAException, self._testLoader, modules=self.modules, tests=tests)
+
+ def test_depends_bounds_upper(self):
+ tests = ['priority.PriorityTest.testDependsBoundsUpperOff']
+ self.assertRaises(OEQAException, self._testLoader, modules=self.modules, tests=tests)
+
+ def test_depends_bounds_type(self):
+ tests = ['priority.PriorityTest.testDependsBoundsType']
+ self.assertRaises(OEQAException, self._testLoader, modules=self.modules, tests=tests)
+
+ def test_depends_bounds(self):
+ tests = ['priority.PriorityTest.testDependsBoundsLower',
+ 'priority.PriorityTest.testDependsBoundsUpper',
+ ]
+ try:
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ test_loaded = getSuiteCasesIDs(tc.suites)
+
+ self.assertEquals(test_loaded, tests, msg="Tests are loaded in the correct order")
+ except OEQAException as e:
+ self.fail(msg="Exception while loaded test suite: %s" % e)
+
+ def test_depends_nodecorator(self):
+ tests = [
+ 'priority.PriorityTest.testPriorityNoDepends',
+ 'priority.PriorityTest.testPriorityNoDepends2',
+ ]
+ try:
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ test_loaded = getSuiteCasesIDs(tc.suites)
+
+ self.assertEquals(test_loaded, tests, msg="Tests are loaded in the correct order")
+ except OEQAException as e:
+ self.fail(msg="Exception while loaded test suite: %s" % e)
+
+ def test_depends_nopriority(self):
+ tests = [
+ 'priority.PriorityTest.testPriorityDependsNoPrio',
+ 'priority.PriorityTest.testPriorityNoDepends',
+ ]
+ expected_order = [
+ 'priority.PriorityTest.testPriorityNoDepends',
+ 'priority.PriorityTest.testPriorityDependsNoPrio',
+ ]
+ try:
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ test_loaded = getSuiteCasesIDs(tc.suites)
+
+ self.assertEquals(test_loaded, expected_order, msg="Tests are loaded in the correct order")
+ except OEQAException as e:
+ self.fail(msg="Exception while loaded test suite: %s" % e)
+
+ def test_depends_order(self):
+ tests = [
+ 'priority.PriorityTest.testPriority15',
+ 'priority.PriorityTest.testPriority25',
+ 'priority.PriorityTest.testPriority30',
+ 'priority.PriorityTest.testPriority80',
+ 'priority.PriorityTest.testPriority80plus',
+ ]
+ expected_order = [
+ 'priority.PriorityTest.testPriority30',
+ 'priority.PriorityTest.testPriority25',
+ 'priority.PriorityTest.testPriority15',
+ 'priority.PriorityTest.testPriority80',
+ 'priority.PriorityTest.testPriority80plus',
+ ]
+ try:
+ tc = self._testLoader(modules=self.modules, tests=tests)
+ test_loaded = getSuiteCasesIDs(tc.suites)
+
+ self.assertEquals(test_loaded, expected_order, msg="Tests are loaded in the correct order")
+ except OEQAException as e:
+ self.fail(msg="Exception while loaded test suite: %s" % e)
+
if __name__ == '__main__':
unittest.main()
--
2.25.1
^ permalink raw reply related [flat|nested] 2+ messages in thread