[PATCH v2 2/3] OETestContext: order test cases by prio & depends

[PATCH v2 2/3] OETestContext: order test cases by prio & depends

[Konrad Weihmann]

Order the overall test case list in the resulting test suite
by priority (by default all test cases do have a priority of 50).
In a second round move test cases with test dependencies accordingly.
New method 'orderSuites' can be explictly overridden to disable
usage of the new 'priority' decorator.

Signed-off-by: Konrad Weihmann <kweihmann@outlook.com>
---
v2:
  - mind test cases without any decorator
  - rename new method to orderSuites

 meta/lib/oeqa/core/context.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 2abe353d27..2ca9ec0c56 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -66,6 +66,34 @@ class OETestContext(object):
                 if (cid + '.').startswith(skip + '.'):
                     setattr(tclass, 'setUpHooker', skipfuncgen('Skip by the command line argument "%s"' % skip))
 
+    def orderSuites(self, suites):
+        _tests = {}
+        for _test in suites._tests:
+            _prio = 50
+            _depends = []
+            for x in getattr(_test, "decorators", []):
+                if hasattr(x, "depends"):
+                    _depends += getattr(x, "depends")
+                if hasattr(x, "priority"):
+                    _prio = getattr(x, "priority")
+            _tests[_test.id()] = {"prio": _prio, "deps": _depends, "ref": _test}
+
+        def set_suite_prio(suites, suite):
+            for dep in suite["deps"]:
+                if dep not in suites:
+                    try:
+                        dep = [x for x in suites.keys() if x.endswith(dep)][0]
+                    except:
+                        continue
+                suite["prio"] = max(suite["prio"], set_suite_prio(suites, suites[dep]) + 1)
+            return suite["prio"]
+
+        for k, v in _tests.items():
+            _tests[k]["prio"] = set_suite_prio(_tests, v)
+
+        suites._tests = [x[1]["ref"] for x in sorted(_tests.items(), key=lambda tup: tup[1]["prio"])]
+        return suites
+
     def loadTests(self, module_paths, modules=[], tests=[],
             modules_manifest="", modules_required=[], **kwargs):
         if modules_manifest:
@@ -73,7 +101,7 @@ class OETestContext(object):
 
         self.loader = self.loaderClass(self, module_paths, modules, tests,
                 modules_required, **kwargs)
-        self.suites = self.loader.discover()
+        self.suites = self.orderSuites(self.loader.discover())
 
     def prepareSuite(self, suites, processes):
         return suites
-- 
2.25.1

[PATCH v2 3/3] oeqa: add tests for OETestPriority

[Konrad Weihmann]

Add test cases for the newly introduced test decorator OETestPriority

Signed-off-by: Konrad Weihmann <kweihmann@outlook.com>
---
 meta/lib/oeqa/core/tests/cases/priority.py  | 61 +++++++++++++++
 meta/lib/oeqa/core/tests/test_decorators.py | 82 ++++++++++++++++++++-
 2 files changed, 142 insertions(+), 1 deletion(-)
 create mode 100644 meta/lib/oeqa/core/tests/cases/priority.py

diff --git a/meta/lib/oeqa/core/tests/cases/priority.py b/meta/lib/oeqa/core/tests/cases/priority.py
new file mode 100644
index 0000000000..cb48eb0424
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/priority.py
@@ -0,0 +1,61 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.priority import OETestPriority
+from oeqa.core.decorator.depends import OETestDepends
+
+class PriorityTest(OETestCase):
+
+    @OETestPriority(0)
+    def testDependsBoundsLower(self):
+        self.assertTrue(True, msg='How is this possible?')
+    
+    @OETestPriority(00)
+    def testDependsBoundsUpper(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestPriority(-1)
+    def testDependsBoundsLowerOff(self):
+        self.assertTrue(True, msg='How is this possible?')
+    
+    @OETestPriority(100)
+    def testDependsBoundsUpperOff(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestPriority("abc")
+    def testDependsBoundsType(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestPriority(80)
+    def testPriority80(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testPriority80'])
+    def testPriority80plus(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestPriority(30)
+    def testPriority30(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testPriority30'])
+    @OETestPriority(25)
+    def testPriority25(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testPriority25'])
+    @OETestPriority(15)
+    def testPriority15(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testPriorityNoDepends(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testPriorityNoDepends2(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testPriorityNoDepends'])
+    def testPriorityDependsNoPrio(self):
+        self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/test_decorators.py b/meta/lib/oeqa/core/tests/test_decorators.py
index b798bf7d33..5a81be1c68 100755
--- a/meta/lib/oeqa/core/tests/test_decorators.py
+++ b/meta/lib/oeqa/core/tests/test_decorators.py
@@ -11,7 +11,7 @@ import unittest
 from common import setup_sys_path, TestBase
 setup_sys_path()
 
-from oeqa.core.exception import OEQADependency
+from oeqa.core.exception import OEQADependency, OEQAException
 from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames, getSuiteCasesIDs
 
 class TestTagDecorator(TestBase):
@@ -133,5 +133,85 @@ class TestTimeoutDecorator(TestBase):
         msg = "OETestTimeout didn't restore SIGALRM"
         self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
 
+class TestPriorityDecorator(TestBase):
+    modules = ['priority']
+
+    def test_depends_bounds_lower(self):
+        tests =  ['priority.PriorityTest.testDependsBoundsLowerOff']
+        self.assertRaises(OEQAException, self._testLoader, modules=self.modules, tests=tests)
+
+    def test_depends_bounds_upper(self):
+        tests =  ['priority.PriorityTest.testDependsBoundsUpperOff']
+        self.assertRaises(OEQAException, self._testLoader, modules=self.modules, tests=tests)
+
+    def test_depends_bounds_type(self):
+        tests =  ['priority.PriorityTest.testDependsBoundsType']
+        self.assertRaises(OEQAException, self._testLoader, modules=self.modules, tests=tests)
+
+    def test_depends_bounds(self):
+        tests =  ['priority.PriorityTest.testDependsBoundsLower',
+                  'priority.PriorityTest.testDependsBoundsUpper',
+                 ]
+        try:
+            tc = self._testLoader(modules=self.modules, tests=tests)
+            test_loaded = getSuiteCasesIDs(tc.suites)
+
+            self.assertEquals(test_loaded, tests, msg="Tests are loaded in the correct order")
+        except OEQAException as e:
+            self.fail(msg="Exception while loaded test suite: %s" % e)
+
+    def test_depends_nodecorator(self):
+        tests =  [
+                  'priority.PriorityTest.testPriorityNoDepends',
+                  'priority.PriorityTest.testPriorityNoDepends2',
+                 ]
+        try:
+            tc = self._testLoader(modules=self.modules, tests=tests)
+            test_loaded = getSuiteCasesIDs(tc.suites)
+
+            self.assertEquals(test_loaded, tests, msg="Tests are loaded in the correct order")
+        except OEQAException as e:
+            self.fail(msg="Exception while loaded test suite: %s" % e)
+
+    def test_depends_nopriority(self):
+        tests =  [
+                  'priority.PriorityTest.testPriorityDependsNoPrio',
+                  'priority.PriorityTest.testPriorityNoDepends',
+                 ]
+        expected_order =  [
+                  'priority.PriorityTest.testPriorityNoDepends',
+                  'priority.PriorityTest.testPriorityDependsNoPrio',
+                 ]
+        try:
+            tc = self._testLoader(modules=self.modules, tests=tests)
+            test_loaded = getSuiteCasesIDs(tc.suites)
+
+            self.assertEquals(test_loaded, expected_order, msg="Tests are loaded in the correct order")
+        except OEQAException as e:
+            self.fail(msg="Exception while loaded test suite: %s" % e)
+
+    def test_depends_order(self):
+        tests =  [
+                  'priority.PriorityTest.testPriority15',
+                  'priority.PriorityTest.testPriority25',
+                  'priority.PriorityTest.testPriority30',
+                  'priority.PriorityTest.testPriority80',
+                  'priority.PriorityTest.testPriority80plus',
+                 ]
+        expected_order = [
+                  'priority.PriorityTest.testPriority30',
+                  'priority.PriorityTest.testPriority25',
+                  'priority.PriorityTest.testPriority15',
+                  'priority.PriorityTest.testPriority80',
+                  'priority.PriorityTest.testPriority80plus',
+                 ]
+        try:
+            tc = self._testLoader(modules=self.modules, tests=tests)
+            test_loaded = getSuiteCasesIDs(tc.suites)
+
+            self.assertEquals(test_loaded, expected_order, msg="Tests are loaded in the correct order")
+        except OEQAException as e:
+            self.fail(msg="Exception while loaded test suite: %s" % e)
+
 if __name__ == '__main__':
     unittest.main()
-- 
2.25.1