[PATCH 0/4] Enable automated sdk tests

[PATCH 0/4] Enable automated sdk tests

[Corneliu Stoicescu]

This series of patches contain the changes and additions needed to enable automated sdk tests. They are handeled similar to the automated runtime tests, sharing both code and test files.

The original version of these patches was created by Richard Purdie <richard.purdie@linuxfoundation.org>.

However, I did not manage to sort out all the isses, and here they are:
- no setUpModule is used in the sdk test modules. This mean these tests will run for every sdk test run(not really an issue, as long as the tests are configured well)
- I left out the testsdk_auto task (should wotk similar to testimage_auto) because I did not get it to work properly. Please note that, for some reason, testimage_auto is also faulty. I will open a bug for this.

Here is the TODO list for this:
- add a cache system for package downloading (using /tmp to download packages for testing means we have to re-download them every time)
- investigate: unify the files/ directory for all tests (now we have a specific runtime/files/ directory for example)


Corneliu Stoicescu (4):
  meta/classes/testimage.bbclass: add testsdk task and enable
    functionality for it.
  oeqa/oetest.py: enable sdk testing
  oeqa/utils/targetbuild.py: add support for sdk tests
  oeqa/sdk/: add sdk tests for sudoku, iptables and cvs

 meta/classes/testimage.bbclass     | 97 +++++++++++++++++++++++++++++++++++---
 meta/lib/oeqa/oetest.py            | 29 +++++++-----
 meta/lib/oeqa/sdk/__init__.py      |  3 ++
 meta/lib/oeqa/sdk/buildcvs.py      | 25 ++++++++++
 meta/lib/oeqa/sdk/buildiptables.py | 26 ++++++++++
 meta/lib/oeqa/sdk/buildsudoku.py   | 22 +++++++++
 meta/lib/oeqa/utils/targetbuild.py | 94 ++++++++++++++++++++++++++++++------
 7 files changed, 263 insertions(+), 33 deletions(-)
 create mode 100644 meta/lib/oeqa/sdk/__init__.py
 create mode 100644 meta/lib/oeqa/sdk/buildcvs.py
 create mode 100644 meta/lib/oeqa/sdk/buildiptables.py
 create mode 100644 meta/lib/oeqa/sdk/buildsudoku.py

-- 
1.8.3.2

[PATCH 1/4] meta/classes/testimage.bbclass: add testsdk task and enable functionality for it.

[Corneliu Stoicescu]

- add new testsdk task for meta-toolchain testing.
- enable the get_tests_list method to work with sdk tests.
- add default TEST_SUITES value for meta-toolchain package

NOTE: Original patch made by: Richard Purdie <richard.purdie@linuxfoundation.org>

Signed-off-by: Corneliu Stoicescu <corneliux.stoicescu@intel.com>
---
 meta/classes/testimage.bbclass | 97 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 90 insertions(+), 7 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 285c6a9..97d0380 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -19,7 +19,7 @@
 # Note that order in TEST_SUITES is important (it's the order tests run) and it influences tests dependencies.
 # A layer can add its own tests in lib/oeqa/runtime, provided it extends BBPATH as normal in its layer.conf.
 
-# TEST_LOG_DIR contains a ssh log (what command is running, output and return codes) and a qemu boot log till login
+# TEST_LOG_DIR contains a command ssh log and may contain infromation about what command is running, output and return codes and for qemu a boot log till login.
 # Booting is handled by this class, and it's not a test in itself.
 # TEST_QEMUBOOT_TIMEOUT can be used to set the maximum time in seconds the launch code will wait for the login prompt.
 
@@ -32,6 +32,7 @@ DEFAULT_TEST_SUITES = "ping auto"
 DEFAULT_TEST_SUITES_pn-core-image-minimal = "ping"
 DEFAULT_TEST_SUITES_pn-core-image-sato = "ping ssh df connman syslog xorg scp vnc date rpm smart dmesg python"
 DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "ping ssh df connman syslog xorg scp vnc date perl ldd gcc rpm smart kernelmodule dmesg python"
+DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto"
 TEST_SUITES ?= "${DEFAULT_TEST_SUITES}"
 
 TEST_QEMUBOOT_TIMEOUT ?= "1000"
@@ -53,8 +54,15 @@ do_testimage[nostamp] = "1"
 do_testimage[depends] += "${TESTIMAGEDEPENDS}"
 do_testimage[lockfiles] += "${TESTIMAGELOCK}"
 
+python do_testsdk() {
+    testsdk_main(d)
+}
+addtask testsdk
+do_testsdk[nostamp] = "1"
+do_testsdk[depends] += "${TESTIMAGEDEPENDS}"
+do_testsdk[lockfiles] += "${TESTIMAGELOCK}"
 
-def get_tests_list(d):
+def get_tests_list(d, type="runtime"):
     testsuites = d.getVar("TEST_SUITES", True).split()
     bbpath = d.getVar("BBPATH", True).split(':')
 
@@ -65,12 +73,12 @@ def get_tests_list(d):
         if testname != "auto":
             found = False
             for p in bbpath:
-                if os.path.exists(os.path.join(p, 'lib', 'oeqa', 'runtime', testname + '.py')):
-                    testslist.append("oeqa.runtime." + testname)
+                if os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname + '.py')):
+                    testslist.append("oeqa." + type + "." + testname)
                     found = True
                     break
             if not found:
-                bb.error('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname)
+                bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname)
 
     if "auto" in testsuites:
         def add_auto_list(path):
@@ -78,12 +86,12 @@ def get_tests_list(d):
                 bb.fatal('Tests directory %s exists but is missing __init__.py' % path)
             files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')])
             for f in files:
-                module = 'oeqa.runtime.' + f[:-3]
+                module = 'oeqa.' + type + '.' + f[:-3]
                 if module not in testslist:
                     testslist.append(module)
 
         for p in bbpath:
-            testpath = os.path.join(p, 'lib', 'oeqa', 'runtime')
+            testpath = os.path.join(p, 'lib', 'oeqa', type)
             bb.debug(2, 'Searching for tests in %s' % testpath)
             if os.path.exists(testpath):
                 add_auto_list(testpath)
@@ -230,3 +238,78 @@ def testimage_main(d):
         target.stop()
 
 testimage_main[vardepsexclude] =+ "BB_ORIGENV"
+
+
+def testsdk_main(d):
+    import unittest
+    import os
+    import glob
+    import oeqa.runtime
+    import oeqa.sdk
+    import time
+    import subprocess
+    from oeqa.oetest import loadTests, runTests
+
+    pn = d.getVar("PN", True)
+    bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
+
+    # tests in TEST_SUITES become required tests
+    # they won't be skipped even if they aren't suitable.
+    # testslist is what we'll actually pass to the unittest loader
+    testslist = get_tests_list(d, "sdk")
+    testsrequired = [t for t in d.getVar("TEST_SUITES", True).split() if t != "auto"]
+
+    sdktestdir = d.expand("${WORKDIR}/testimage-sdk/")
+    bb.utils.remove(sdktestdir, True)
+    bb.utils.mkdirhier(sdktestdir)
+
+    tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh")
+    if not os.path.exists(tcname):
+        bb.fatal("The toolchain is not built. Build it before running the tests: 'bitbake meta-toolchain' .")
+    subprocess.call("cd %s; %s <<EOF\n./tc\nY\nEOF" % (sdktestdir, tcname), shell=True)
+
+    targets = glob.glob(d.expand(sdktestdir + "/tc/sysroots/*${TARGET_VENDOR}-linux*"))
+    if len(targets) > 1:
+        bb.fatal("Error, multiple targets within the SDK found and we don't know which to test? %s" % str(targets))
+    sdkenv = sdktestdir + "/tc/environment-setup-" + os.path.basename(targets[0])
+
+    class TestContext(object):
+        def __init__(self):
+            self.d = d
+            self.testslist = testslist
+            self.testsrequired = testsrequired
+            self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files")
+            self.sdktestdir = sdktestdir
+            self.sdkenv = sdkenv
+
+    # test context
+    tc = TestContext()
+
+    # this is a dummy load of tests
+    # we are doing that to find compile errors in the tests themselves
+    # before booting the image
+    try:
+        loadTests(tc, "sdk")
+    except Exception as e:
+        import traceback
+        bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
+
+    try:
+        starttime = time.time()
+        result = runTests(tc, "sdk")
+        stoptime = time.time()
+        if result.wasSuccessful():
+            bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime))
+            msg = "%s - OK - All required tests passed" % pn
+            skipped = len(result.skipped)
+            if skipped:
+                msg += " (skipped=%d)" % skipped
+            bb.plain(msg)
+        else:
+            raise bb.build.FuncFailed("%s - FAILED - check the task log and the commands log" % pn )
+    finally:
+        pass
+        bb.utils.remove(sdktestdir, True)
+
+testsdk_main[vardepsexclude] =+ "BB_ORIGENV"
+
-- 
1.8.3.2

[PATCH 2/4] oeqa/oetest.py: enable sdk testing

[Corneliu Stoicescu]

- add support for sdk tests in the loadTests and runTests methods
- add new oeSDKTest test object

NOTE: Original patch made by: Richard Purdie <richard.purdie@linuxfoundation.org>

Signed-off-by: Corneliu Stoicescu <corneliux.stoicescu@intel.com>
---
 meta/lib/oeqa/oetest.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index dcbd498..5552c43 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -10,25 +10,29 @@
 import os, re, mmap
 import unittest
 import inspect
+import subprocess
 from oeqa.utils.decorators import LogResults
 
-def loadTests(tc):
-
-    # set the context object passed from the test class
-    setattr(oeTest, "tc", tc)
-    # set ps command to use
-    setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps")
-    # prepare test suite, loader and runner
-    suite = unittest.TestSuite()
+def loadTests(tc, type="runtime"):
+    if type == "runtime":
+        # set the context object passed from the test class
+        setattr(oeTest, "tc", tc)
+        # set ps command to use
+        setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps")
+        # prepare test suite, loader and runner
+        suite = unittest.TestSuite()
+    elif type == "sdk":
+        # set the context object passed from the test class
+        setattr(oeSDKTest, "tc", tc)
     testloader = unittest.TestLoader()
     testloader.sortTestMethodsUsing = None
     suite = testloader.loadTestsFromNames(tc.testslist)
 
     return suite
 
-def runTests(tc):
+def runTests(tc, type="runtime"):
 
-    suite = loadTests(tc)
+    suite = loadTests(tc, type)
     print("Test modules  %s" % tc.testslist)
     print("Found %s tests" % suite.countTestCases())
     runner = unittest.TextTestRunner(verbosity=2)
@@ -58,11 +62,14 @@ class oeTest(unittest.TestCase):
             return False
 
 class oeRuntimeTest(oeTest):
-
     def __init__(self, methodName='runTest'):
         self.target = oeRuntimeTest.tc.target
         super(oeRuntimeTest, self).__init__(methodName)
 
+class oeSDKTest(unittest.TestCase):
+    def __init__(self, methodName='runTest'):
+        self.sdktestdir = oeSDKTest.tc.sdktestdir
+        super(oeSDKTest, self).__init__(methodName)
 
 def getmodule(pos=2):
     # stack returns a list of tuples containg frame information
-- 
1.8.3.2

[PATCH 3/4] oeqa/utils/targetbuild.py: add support for sdk tests

[Corneliu Stoicescu]

- Create new abstract class BuildProject that provides basic functionality for a project/package building class
        * contains abstract method _run() that needs to be implemented by all extending classes.
- The old TargetBuildProject class now extends the abstract BuildProjct class
- Introducing new SDKBuildProject that extends the abstract BuildProjct class

NOTE: Original patch made by: Richard Purdie <richard.purdie@linuxfoundation.org>

Signed-off-by: Corneliu Stoicescu <corneliux.stoicescu@intel.com>
---
 meta/lib/oeqa/utils/targetbuild.py | 94 ++++++++++++++++++++++++++++++++------
 1 file changed, 79 insertions(+), 15 deletions(-)

diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py
index 3229676..eeb08ba 100644
--- a/meta/lib/oeqa/utils/targetbuild.py
+++ b/meta/lib/oeqa/utils/targetbuild.py
@@ -6,23 +6,25 @@
 
 import os
 import re
+import bb.utils
 import subprocess
+from abc import ABCMeta, abstractmethod
 
+class BuildProject():
 
-class TargetBuildProject():
+    __metaclass__ = ABCMeta
 
-    def __init__(self, target, d, uri, foldername=None):
-        self.target = target
+    def __init__(self, d, uri, foldername=None, tmpdir="/tmp/"):
         self.d = d
         self.uri = uri
-        self.targetdir = "~/"
         self.archive = os.path.basename(uri)
-        self.localarchive = "/tmp/" + self.archive
+        self.localarchive = os.path.join(tmpdir,self.archive)
         self.fname = re.sub(r'.tar.bz2|tar.gz$', '', self.archive)
         if foldername:
             self.fname = foldername
 
-    def download_archive(self):
+    # Download self.archive to self.localarchive
+    def _download_archive(self):
 
         exportvars = ['HTTP_PROXY', 'http_proxy',
                       'HTTPS_PROXY', 'https_proxy',
@@ -41,6 +43,38 @@ class TargetBuildProject():
         cmd = cmd + "wget -O %s %s" % (self.localarchive, self.uri)
         subprocess.check_call(cmd, shell=True)
 
+    # This method should provide a way to run a command in the desired environment.
+    @abstractmethod
+    def _run(self, cmd):
+        pass
+
+    # The timeout parameter of target.run is set to 0 to make the ssh command
+    # run with no timeout.
+    def run_configure(self, configure_args=''):
+        return self._run('cd %s; ./configure %s' % (self.targetdir, configure_args))
+
+    def run_make(self, make_args=''):
+        return self._run('cd %s; make %s' % (self.targetdir, make_args))
+
+    def run_install(self, install_args=''):
+        return self._run('cd %s; make install %s' % (self.targetdir, install_args))
+
+    def clean(self):
+        self._run('rm -rf %s' % self.targetdir)
+        subprocess.call('rm -f %s' % self.localarchive, shell=True)
+        pass
+
+class TargetBuildProject(BuildProject):
+
+    def __init__(self, target, d, uri, foldername=None):
+        self.target = target
+        self.targetdir = "~/"
+        BuildProject.__init__(self, d, uri, foldername, tmpdir="/tmp")
+
+    def download_archive(self):
+
+        self._download_archive()
+
         (status, output) = self.target.copy_to(self.localarchive, self.targetdir)
         if status != 0:
             raise Exception("Failed to copy archive to target, output: %s" % output)
@@ -54,15 +88,45 @@ class TargetBuildProject():
 
     # The timeout parameter of target.run is set to 0 to make the ssh command
     # run with no timeout.
-    def run_configure(self):
-        return self.target.run('cd %s; ./configure' % self.targetdir, 0)[0]
+    def _run(self, cmd):
+        return self.target.run(cmd, 0)[0]
 
-    def run_make(self):
-        return self.target.run('cd %s; make' % self.targetdir, 0)[0]
 
-    def run_install(self):
-        return self.target.run('cd %s; make install' % self.targetdir, 0)[0]
+class SDKBuildProject(BuildProject):
+
+    def __init__(self, testpath, sdkenv, d, uri, foldername=None):
+        self.sdkenv = sdkenv
+        self.testdir = testpath
+        self.targetdir = testpath
+        bb.utils.mkdirhier(testpath)
+        self.datetime = d.getVar('DATETIME', True)
+        self.testlogdir = d.getVar("TEST_LOG_DIR", True)
+        bb.utils.mkdirhier(self.testlogdir)
+        self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime)
+        BuildProject.__init__(self, d, uri, foldername, tmpdir=testpath)
+
+    def download_archive(self):
+
+        self._download_archive()
+
+        cmd = 'tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir)
+        subprocess.check_call(cmd, shell=True)
+
+        #Change targetdir to project folder
+        self.targetdir = self.targetdir + self.fname
+
+    def run_configure(self, configure_args=''):
+        return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'))
+
+    def run_install(self, install_args=''):
+        return super(SDKBuildProject, self).run_install(install_args=(install_args or "DESTDIR=%s/../install" % self.targetdir))
+
+    def log(self, msg):
+        if self.logfile:
+            with open(self.logfile, "a") as f:
+               f.write("%s\n" % msg)
+
+    def _run(self, cmd):
+        self.log("Running source %s; " % self.sdkenv + cmd)
+        return subprocess.call("source %s; " % self.sdkenv + cmd, shell=True)
 
-    def clean(self):
-        self.target.run('rm -rf %s' % self.targetdir)
-        subprocess.call('rm -f %s' % self.localarchive, shell=True)
-- 
1.8.3.2

[PATCH 4/4] oeqa/sdk/: add sdk tests for sudoku, iptables and cvs

[Corneliu Stoicescu]

Add test modules for sdk tests.

NOTE: Original patch made by: Richard Purdie <richard.purdie@linuxfoundation.org>

Signed-off-by: Corneliu Stoicescu <corneliux.stoicescu@intel.com>
---
 meta/lib/oeqa/sdk/__init__.py      |  3 +++
 meta/lib/oeqa/sdk/buildcvs.py      | 25 +++++++++++++++++++++++++
 meta/lib/oeqa/sdk/buildiptables.py | 26 ++++++++++++++++++++++++++
 meta/lib/oeqa/sdk/buildsudoku.py   | 22 ++++++++++++++++++++++
 4 files changed, 76 insertions(+)
 create mode 100644 meta/lib/oeqa/sdk/__init__.py
 create mode 100644 meta/lib/oeqa/sdk/buildcvs.py
 create mode 100644 meta/lib/oeqa/sdk/buildiptables.py
 create mode 100644 meta/lib/oeqa/sdk/buildsudoku.py

diff --git a/meta/lib/oeqa/sdk/__init__.py b/meta/lib/oeqa/sdk/__init__.py
new file mode 100644
index 0000000..4cf3fa7
--- /dev/null
+++ b/meta/lib/oeqa/sdk/__init__.py
@@ -0,0 +1,3 @@
+# Enable other layers to have tests in the same named directory
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
diff --git a/meta/lib/oeqa/sdk/buildcvs.py b/meta/lib/oeqa/sdk/buildcvs.py
new file mode 100644
index 0000000..c7146fa
--- /dev/null
+++ b/meta/lib/oeqa/sdk/buildcvs.py
@@ -0,0 +1,25 @@
+from oeqa.oetest import oeSDKTest, skipModule
+from oeqa.utils.decorators import *
+from oeqa.utils.targetbuild import SDKBuildProject
+
+class BuildCvsTest(oeSDKTest):
+
+    @classmethod
+    def setUpClass(self):
+        self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/cvs/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d,
+                        "http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2")
+        self.project.download_archive()
+
+    def test_cvs(self):
+        self.assertEqual(self.project.run_configure(), 0,
+                        msg="Running configure failed")
+
+        self.assertEqual(self.project.run_make(), 0,
+                        msg="Running make failed")
+
+        self.assertEqual(self.project.run_install(), 0,
+                        msg="Running make install failed")
+
+    @classmethod
+    def tearDownClass(self):
+        self.project.clean()
diff --git a/meta/lib/oeqa/sdk/buildiptables.py b/meta/lib/oeqa/sdk/buildiptables.py
new file mode 100644
index 0000000..062e531
--- /dev/null
+++ b/meta/lib/oeqa/sdk/buildiptables.py
@@ -0,0 +1,26 @@
+from oeqa.oetest import oeSDKTest
+from oeqa.utils.decorators import *
+from oeqa.utils.targetbuild import SDKBuildProject
+
+
+class BuildIptablesTest(oeSDKTest):
+
+    @classmethod
+    def setUpClass(self):
+        self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/iptables/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d,
+                        "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2")
+        self.project.download_archive()
+
+    def test_iptables(self):
+        self.assertEqual(self.project.run_configure(), 0,
+                        msg="Running configure failed")
+
+        self.assertEqual(self.project.run_make(), 0,
+                        msg="Running make failed")
+
+        self.assertEqual(self.project.run_install(), 0,
+                        msg="Running make install failed")
+
+    @classmethod
+    def tearDownClass(self):
+        self.project.clean()
diff --git a/meta/lib/oeqa/sdk/buildsudoku.py b/meta/lib/oeqa/sdk/buildsudoku.py
new file mode 100644
index 0000000..6a60c76
--- /dev/null
+++ b/meta/lib/oeqa/sdk/buildsudoku.py
@@ -0,0 +1,22 @@
+from oeqa.oetest import oeSDKTest, skipModule
+from oeqa.utils.decorators import *
+from oeqa.utils.targetbuild import SDKBuildProject
+
+class SudokuTest(oeSDKTest):
+
+    @classmethod
+    def setUpClass(self):
+        self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/sudoku/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d,
+                        "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2")
+        self.project.download_archive()
+
+    def test_sudoku(self):
+        self.assertEqual(self.project.run_configure(), 0,
+                        msg="Running configure failed")
+
+        self.assertEqual(self.project.run_make(), 0,
+                        msg="Running make failed")
+
+    @classmethod
+    def tearDownClass(self):
+        self.project.clean()
-- 
1.8.3.2