[PATCH 01/12] oeqa/sdkext/devtool: rename recipe name to match upstream

[PATCH 01/12] oeqa/sdkext/devtool: rename recipe name to match upstream

[Richard Purdie]

From: Ross Burton <ross.burton@intel.com>

Call the generated recipe librdfa instead of bb-example to make it clearer what
is happening.

Signed-off-by: Ross Burton <ross.burton@intel.com>
---
 meta/lib/oeqa/sdkext/cases/devtool.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/sdkext/cases/devtool.py b/meta/lib/oeqa/sdkext/cases/devtool.py
index f0abc95..baa528f 100644
--- a/meta/lib/oeqa/sdkext/cases/devtool.py
+++ b/meta/lib/oeqa/sdkext/cases/devtool.py
@@ -73,7 +73,7 @@ class DevtoolTest(OESDKExtTestCase):
     @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_extend_autotools_recipe_creation(self):
         req = 'https://github.com/rdfa/librdfa'
-        recipe = "bbexample"
+        recipe = "librdfa"
         self._run('devtool add %s %s' % (recipe, req) )
         try:
             self._run('devtool build %s' % recipe)
-- 
2.7.4

[PATCH 02/12] testsdk: Handle minimal eSDK and avoid download costs

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/testsdk.bbclass | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass
index 75b4027..802e57f 100644
--- a/meta/classes/testsdk.bbclass
+++ b/meta/classes/testsdk.bbclass
@@ -149,6 +149,17 @@ def testsdkext_main(d):
         bb.plain("Extensible SDK testing environment: %s" % s)
 
         sdk_env = sdk_envs[s]
+
+        # Use our own SSTATE_DIR and DL_DIR so that updates to the eSDK come from our sstate cache
+        # and we don't spend hours downloading kernels for the kernel module test
+        with open(os.path.join(sdk_dir, 'conf', 'local.conf'), 'a+') as f:
+            f.write('SSTATE_MIRRORS = "file://.* file://%s/PATH"\n' % test_data.get('SSTATE_DIR'))
+            f.write('SOURCE_MIRROR_URL = "file://%s"\n' % test_data.get('DL_DIR'))
+            f.write('INHERIT += "own-mirrors"')
+
+        # We need to do this in case we have a minimal SDK
+        subprocess.check_output(". %s > /dev/null; devtool sdk-install meta-extsdk-toolchain" % sdk_env, cwd=sdk_dir, shell=True)
+
         tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir,
             sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
             host_pkg_manifest=host_pkg_manifest)
-- 
2.7.4

[PATCH 03/12] devtool/sdk: Run build-sysroots after installing new things

[Richard Purdie]

After running sdk-install we need to ensure that the standalone sysroots are
updated as done when the eSDK is originally built. Add such a call so this
happens automatically and the envrionment scripts in the SDK work correctly
after updates.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/lib/devtool/sdk.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index f629db1..e8bf0ad 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -306,6 +306,12 @@ def sdk_install(args, config, basepath, workspace):
         if failed:
             return 2
 
+        try:
+            exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots', watch=True)
+        except bb.process.ExecutionError as e:
+            raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e)))
+
+
 def register_commands(subparsers, context):
     """Register devtool subcommands from the sdk plugin"""
     if context.fixed_setup:
-- 
2.7.4

[PATCH 04/12] oeqa/sdkext: Ensure we run a deterministic set of tests

[Richard Purdie]

The directory list of sdk tests to run can vary so this code effectively selects
a random set of SDK tests to run in the eSDK. We want to attemp all the SDK tests
so remove the element selection.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/sdkext/context.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/sdkext/context.py b/meta/lib/oeqa/sdkext/context.py
index 8dbcd80..bee8c39 100644
--- a/meta/lib/oeqa/sdkext/context.py
+++ b/meta/lib/oeqa/sdkext/context.py
@@ -14,8 +14,8 @@ class OESDKExtTestContextExecutor(OESDKTestContextExecutor):
     help = 'esdk test component'
     description = 'executes esdk tests'
 
-    default_cases = [OESDKTestContextExecutor.default_cases[0],
-            os.path.join(os.path.abspath(os.path.dirname(__file__)), 'cases')]
+    default_cases = OESDKTestContextExecutor.default_cases + \
+            [os.path.join(os.path.abspath(os.path.dirname(__file__)), 'cases')]
     default_test_data = None
 
 _executor_class = OESDKExtTestContextExecutor
-- 
2.7.4

[PATCH 05/12] oeqs/sdk*/case: Use universal_newlines for subprocess calls

[Richard Purdie]

This removes the need for some of the ugly decode calls with hardcoded
locales.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/sdk/case.py    | 2 +-
 meta/lib/oeqa/sdkext/case.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/sdk/case.py b/meta/lib/oeqa/sdk/case.py
index 782db8b..963aa8d 100644
--- a/meta/lib/oeqa/sdk/case.py
+++ b/meta/lib/oeqa/sdk/case.py
@@ -9,4 +9,4 @@ class OESDKTestCase(OETestCase):
     def _run(self, cmd):
         return subprocess.check_output(". %s > /dev/null; %s;" % \
                 (self.tc.sdk_env, cmd), shell=True,
-                stderr=subprocess.STDOUT).decode("utf-8")
+                stderr=subprocess.STDOUT, universal_newlines=True)
diff --git a/meta/lib/oeqa/sdkext/case.py b/meta/lib/oeqa/sdkext/case.py
index 6f708aa..21b7188 100644
--- a/meta/lib/oeqa/sdkext/case.py
+++ b/meta/lib/oeqa/sdkext/case.py
@@ -18,4 +18,4 @@ class OESDKExtTestCase(OESDKTestCase):
 
         return subprocess.check_output(". %s > /dev/null;"\
             " %s;" % (self.tc.sdk_env, cmd), stderr=subprocess.STDOUT,
-            shell=True, env=env).decode("utf-8")
+            shell=True, env=env, universal_newlines=True)
-- 
2.7.4

[PATCH 06/12] oeqa/sdkext/context: Work around broken dependency checks to get sdk tests running

[Richard Purdie]

This is admitted a bit of a hack but it does allow a number of significant sdk
tests to run successfully and hence improves testing of eSDK which is good.

I'm therefore proposing we do this until we come up with a better solution
since the current lack of testing is worrying and would have caught other issues
had it been present.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/sdkext/context.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/meta/lib/oeqa/sdkext/context.py b/meta/lib/oeqa/sdkext/context.py
index bee8c39..65da4c6 100644
--- a/meta/lib/oeqa/sdkext/context.py
+++ b/meta/lib/oeqa/sdkext/context.py
@@ -7,6 +7,14 @@ from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor
 class OESDKExtTestContext(OESDKTestContext):
     esdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files")
 
+    # FIXME - We really need to do better mapping of names here, this at
+    # least allows some tests to run
+    def hasHostPackage(self, pkg):
+        # We force a toolchain to be installed into the eSDK even if its minimal
+        if pkg.startswith("packagegroup-cross-canadian-"):
+            return True
+        return self._hasPackage(self.host_pkg_manifest, pkg)
+
 class OESDKExtTestContextExecutor(OESDKTestContextExecutor):
     _context_class = OESDKExtTestContext
 
-- 
2.7.4

[PATCH 07/12] populate_sdk_ext: Allow generation of meta-extsdk-toolchain even for minimal SDKs

[Richard Purdie]

If you build a minimal eSDK currently, you don't build meta-extesdk-toolchain
even if you will have built most of its dependencies. This means when you try
and install a toolchain into the eSDK, it fails, breaking our automated testing
of the eSDK.

Therefore add the dependency unconditionally even when a minimal eSDK is being
built and allow the automated testing to work.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/populate_sdk_ext.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index fff0f12..51b98a8 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -649,7 +649,7 @@ python do_sdk_depends() {
 addtask sdk_depends
 
 do_sdk_depends[dirs] = "${WORKDIR}"
-do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)}"
+do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot"
 do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}"
 do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}"
 do_sdk_depends[rdepends] = "${@get_sdk_ext_rdepends(d)}"
-- 
2.7.4

[PATCH 08/12] oeqa/selftest: Drop http sstate sharing

[Richard Purdie]

Using httpServer from python for sharing sstate is known to be buggy, it can't
cope with the number/type of requests coming from bitbake and quietly fails
to share files.

This causes intermittent build failures which are hard to debug. We can
use a file:// url for the sstate mirror instead, removing the need for
the http server.

The sdk-update test is simply dropped since the SDK is never published
to this location and hence it would never have any update. Its equiavalent
to pointing at an empty web server. There is a better eSDK update test in
testsdk so rather than improve this one, lets drop it and concentrate on
the one there.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/selftest/eSDK.py | 55 ++++++++++++------------------------------
 1 file changed, 16 insertions(+), 39 deletions(-)

diff --git a/meta/lib/oeqa/selftest/eSDK.py b/meta/lib/oeqa/selftest/eSDK.py
index a66ff92..1596c6e 100644
--- a/meta/lib/oeqa/selftest/eSDK.py
+++ b/meta/lib/oeqa/selftest/eSDK.py
@@ -9,7 +9,6 @@ import oeqa.utils.ftools as ftools
 from oeqa.utils.decorators import testcase
 from oeqa.selftest.base import oeSelfTest
 from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
-from oeqa.utils.httpserver import HTTPService
 
 class oeSDKExtSelfTest(oeSelfTest):
     """
@@ -55,9 +54,6 @@ class oeSDKExtSelfTest(oeSelfTest):
     @staticmethod
     def update_configuration(cls, image, tmpdir_eSDKQA, env_eSDK, ext_sdk_path):
         sstate_dir = os.path.join(os.environ['BUILDDIR'], 'sstate-cache')
-        cls.http_service = HTTPService(sstate_dir)
-        cls.http_service.start()
-        cls.http_url = "http://127.0.0.1:%d" % cls.http_service.port
 
         oeSDKExtSelfTest.generate_eSDK(cls.image)
 
@@ -68,51 +64,39 @@ class oeSDKExtSelfTest(oeSelfTest):
 
         sstate_config="""
 SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS"
-SSTATE_MIRRORS =  "file://.* http://%s/PATH"
+SSTATE_MIRRORS =  "file://.* file://%s/PATH"
 CORE_IMAGE_EXTRA_INSTALL = "perl"
-        """ % cls.http_url
+        """ % sstate_dir
 
         with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f:
             f.write(sstate_config)
 
     @classmethod
     def setUpClass(cls):
-        # If there is an exception in setUpClass it will not run tearDownClass
-        # method and it leaves HTTP server running forever, so we need to be
-        # sure tearDownClass is run.
-        try:
-            cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA')
+        cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA')
 
-            # Start to serve sstate dir
-            sstate_dir = get_bb_var('SSTATE_DIR')
-            cls.http_service = HTTPService(sstate_dir)
-            cls.http_url = "http://127.0.0.1:%d" % cls.http_service.port
-            cls.http_service.start()
+        sstate_dir = get_bb_var('SSTATE_DIR')
 
-            cls.image = 'core-image-minimal'
-            oeSDKExtSelfTest.generate_eSDK(cls.image)
+        cls.image = 'core-image-minimal'
+        oeSDKExtSelfTest.generate_eSDK(cls.image)
 
-            # Install eSDK
-            cls.ext_sdk_path = oeSDKExtSelfTest.get_eSDK_toolchain(cls.image)
-            runCmd("%s -y -d \"%s\"" % (cls.ext_sdk_path, cls.tmpdir_eSDKQA))
+        # Install eSDK
+        cls.ext_sdk_path = oeSDKExtSelfTest.get_eSDK_toolchain(cls.image)
+        runCmd("%s -y -d \"%s\"" % (cls.ext_sdk_path, cls.tmpdir_eSDKQA))
 
-            cls.env_eSDK = oeSDKExtSelfTest.get_esdk_environment('', cls.tmpdir_eSDKQA)
+        cls.env_eSDK = oeSDKExtSelfTest.get_esdk_environment('', cls.tmpdir_eSDKQA)
 
-            # Configure eSDK to use sstate mirror from poky
-            sstate_config="""
+        # Configure eSDK to use sstate mirror from poky
+        sstate_config="""
 SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS"
-SSTATE_MIRRORS =  "file://.* http://%s/PATH"
-            """ % cls.http_url
-            with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f:
-                f.write(sstate_config)
-        except:
-            cls.tearDownClass()
-            raise
+SSTATE_MIRRORS =  "file://.* file://%s/PATH"
+            """ % sstate_dir
+        with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f:
+            f.write(sstate_config)
 
     @classmethod
     def tearDownClass(cls):
         shutil.rmtree(cls.tmpdir_eSDKQA)
-        cls.http_service.stop()
 
     @testcase (1602)
     def test_install_libraries_headers(self):
@@ -127,12 +111,5 @@ SSTATE_MIRRORS =  "file://.* http://%s/PATH"
         cmd = "devtool build-image %s" % image
         oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd)
 
-    @testcase(1567)
-    def test_sdk_update_http(self):
-        cmd = "devtool sdk-update %s" % self.http_url
-        oeSDKExtSelfTest.update_configuration(self, self.image, self.tmpdir_eSDKQA, self.env_eSDK, self.ext_sdk_path)
-        oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd)
-        self.http_service.stop()
-
 if __name__ == '__main__':
     unittest.main()
-- 
2.7.4

[PATCH 09/12] oeqa/esdk/devtool: clean setUpClass/tearDownClass

[Richard Purdie]

From: Ross Burton <ross.burton@intel.com>

These methods are class not instance methods, so the argument should be cls not
self.

Also don't put variables into cls that we don't need there.

Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/sdkext/cases/devtool.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/meta/lib/oeqa/sdkext/cases/devtool.py b/meta/lib/oeqa/sdkext/cases/devtool.py
index baa528f..ffaa2f0 100644
--- a/meta/lib/oeqa/sdkext/cases/devtool.py
+++ b/meta/lib/oeqa/sdkext/cases/devtool.py
@@ -11,14 +11,19 @@ from oeqa.core.decorator.data import skipIfNotDataVar
 
 class DevtoolTest(OESDKExtTestCase):
     @classmethod
-    def setUpClass(self):
-        self.myapp_src = os.path.join(self.tc.esdk_files_dir, "myapp")
-        self.myapp_dst = os.path.join(self.tc.sdk_dir, "myapp")
-        shutil.copytree(self.myapp_src, self.myapp_dst)
+    def setUpClass(cls):
+        myapp_src = os.path.join(cls.tc.esdk_files_dir, "myapp")
+        cls.myapp_dst = os.path.join(cls.tc.sdk_dir, "myapp")
+        shutil.copytree(myapp_src, cls.myapp_dst)
 
-        self.myapp_cmake_src = os.path.join(self.tc.esdk_files_dir, "myapp_cmake")
-        self.myapp_cmake_dst = os.path.join(self.tc.sdk_dir, "myapp_cmake")
-        shutil.copytree(self.myapp_cmake_src, self.myapp_cmake_dst)
+        myapp_cmake_src = os.path.join(cls.tc.esdk_files_dir, "myapp_cmake")
+        cls.myapp_cmake_dst = os.path.join(cls.tc.sdk_dir, "myapp_cmake")
+        shutil.copytree(myapp_cmake_src, cls.myapp_cmake_dst)
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.myapp_dst)
+        shutil.rmtree(cls.myapp_cmake_dst)
 
     def _test_devtool_build(self, directory):
         self._run('devtool add myapp %s' % directory)
@@ -111,8 +116,3 @@ class DevtoolTest(OESDKExtTestCase):
             self._run('devtool reset %s' % package_nodejs)
             raise e
         self._run('devtool reset %s '% package_nodejs)
-
-    @classmethod
-    def tearDownClass(self):
-        shutil.rmtree(self.myapp_dst)
-        shutil.rmtree(self.myapp_cmake_dst)
-- 
2.7.4

[PATCH 10/12] oeqa: add output to subprocess exceptions

[Richard Purdie]

From: Ross Burton <ross.burton@intel.com>

Out of the box subprocess.CalledProcessError.__str__() just displays the command
and exit code, which isn't very useful for debugging.

Add a function to oeqa.utils.subprocesstweak to monkey-patch __str__() so that
it can also display the value of stdout and stderr.

Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/oetest.py                |  9 ---------
 meta/lib/oeqa/utils/subprocesstweak.py | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 9 deletions(-)
 create mode 100644 meta/lib/oeqa/utils/subprocesstweak.py

diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index a89bd11..1dad763 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -135,15 +135,6 @@ class oeRuntimeTest(oeTest):
             if status != 0:
                 return status
 
-class OETestCalledProcessError(subprocess.CalledProcessError):
-    def __str__(self):
-        if hasattr(self, "stderr"):
-            return "Command '%s' returned non-zero exit status %d with output %s and stderr %s" % (self.cmd, self.returncode, self.output, self.stderr)
-        else:
-            return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.returncode, self.output)
-
-subprocess.CalledProcessError = OETestCalledProcessError
-
 def getmodule(pos=2):
     # stack returns a list of tuples containg frame information
     # First element of the list the is current frame, caller is 1
diff --git a/meta/lib/oeqa/utils/subprocesstweak.py b/meta/lib/oeqa/utils/subprocesstweak.py
new file mode 100644
index 0000000..1f7d11b
--- /dev/null
+++ b/meta/lib/oeqa/utils/subprocesstweak.py
@@ -0,0 +1,19 @@
+import subprocess
+
+class OETestCalledProcessError(subprocess.CalledProcessError):
+    def __str__(self):
+        def strify(o):
+            if isinstance(o, bytes):
+                return o.decode("utf-8", errors="replace")
+            else:
+                return o
+
+        s = "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
+        if hasattr(self, "output") and self.output:
+            s = s + "\nStandard Output: " + strify(self.output)
+        if hasattr(self, "stderr") and self.stderr:
+            s = s + "\nStandard Error: " + strify(self.stderr)
+        return s
+
+def errors_have_output():
+    subprocess.CalledProcessError = OETestCalledProcessError
-- 
2.7.4

[PATCH 11/12] oeqa/sdkext/devtool: use finally instead of repeating cleanup

[Richard Purdie]

From: Ross Burton <ross.burton@intel.com>

Use the finally: block to always to cleanup.

Now that the test harness in testsdk.bbclass has monkey-patched
CalledProcessException to display the output we don't need to do that in the
test case.

Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/testsdk.bbclass          |  4 +++-
 meta/lib/oeqa/sdkext/cases/devtool.py | 35 ++++++++++-------------------------
 2 files changed, 13 insertions(+), 26 deletions(-)

diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass
index 802e57f..0e6949e 100644
--- a/meta/classes/testsdk.bbclass
+++ b/meta/classes/testsdk.bbclass
@@ -100,7 +100,7 @@ def testsdkext_main(d):
     import logging
 
     from bb.utils import export_proxies
-    from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible
+    from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak
     from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor
 
     pn = d.getVar("PN")
@@ -109,6 +109,8 @@ def testsdkext_main(d):
     # extensible sdk use network
     export_proxies(d)
 
+    subprocesstweak.errors_have_output()
+
     # extensible sdk can be contaminated if native programs are
     # in PATH, i.e. use perl-native instead of eSDK one.
     paths_to_avoid = [d.getVar('STAGING_DIR'),
diff --git a/meta/lib/oeqa/sdkext/cases/devtool.py b/meta/lib/oeqa/sdkext/cases/devtool.py
index ffaa2f0..63431aa 100644
--- a/meta/lib/oeqa/sdkext/cases/devtool.py
+++ b/meta/lib/oeqa/sdkext/cases/devtool.py
@@ -29,38 +29,32 @@ class DevtoolTest(OESDKExtTestCase):
         self._run('devtool add myapp %s' % directory)
         try:
             self._run('devtool build myapp')
-        except Exception as e:
-            print(e.output)
+        finally:
             self._run('devtool reset myapp')
-            raise e
-        self._run('devtool reset myapp')
 
     def _test_devtool_build_package(self, directory):
         self._run('devtool add myapp %s' % directory)
         try:
             self._run('devtool package myapp')
-        except Exception as e:
-            print(e.output)
+        finally:
             self._run('devtool reset myapp')
-            raise e
-        self._run('devtool reset myapp')
 
     def test_devtool_location(self):
         output = self._run('which devtool')
         self.assertEqual(output.startswith(self.tc.sdk_dir), True, \
             msg="Seems that devtool isn't the eSDK one: %s" % output)
-    
+
     @OETestDepends(['test_devtool_location'])
     def test_devtool_add_reset(self):
         self._run('devtool add myapp %s' % self.myapp_dst)
         self._run('devtool reset myapp')
-    
+
     @OETestID(1605)
     @OETestDepends(['test_devtool_location'])
     @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_devtool_build_make(self):
         self._test_devtool_build(self.myapp_dst)
-    
+
     @OETestID(1606)
     @OETestDepends(['test_devtool_location'])
     @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
@@ -72,7 +66,7 @@ class DevtoolTest(OESDKExtTestCase):
     @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_devtool_build_cmake(self):
         self._test_devtool_build(self.myapp_cmake_dst)
-    
+
     @OETestID(1608)
     @OETestDepends(['test_devtool_location'])
     @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
@@ -82,11 +76,8 @@ class DevtoolTest(OESDKExtTestCase):
         self._run('devtool add %s %s' % (recipe, req) )
         try:
             self._run('devtool build %s' % recipe)
-        except Exception as e:
-            print(e.output)
+        finally:
             self._run('devtool reset %s' % recipe)
-            raise e
-        self._run('devtool reset %s' % recipe)
 
     @OETestID(1609)
     @OETestDepends(['test_devtool_location'])
@@ -97,11 +88,8 @@ class DevtoolTest(OESDKExtTestCase):
         self._run('devtool add %s %s' % (recipe, docfile) )
         try:
             self._run('devtool build %s' % recipe)
-        except Exception as e:
-            print(e.output)
+        finally:
             self._run('devtool reset %s' % recipe)
-            raise e
-        self._run('devtool reset %s' % recipe)
 
     @OETestID(1610)
     @OETestDepends(['test_devtool_location'])
@@ -111,8 +99,5 @@ class DevtoolTest(OESDKExtTestCase):
         self._run('devtool add %s ' % package_nodejs)
         try:
             self._run('devtool build %s ' % package_nodejs)
-        except Exception as e:
-            print(e.output)
-            self._run('devtool reset %s' % package_nodejs)
-            raise e
-        self._run('devtool reset %s '% package_nodejs)
+        finally:
+            self._run('devtool reset %s '% package_nodejs)
-- 
2.7.4

[PATCH 12/12] oeqa/sdkext: don't skip tests if there isn't a toolchain

[Richard Purdie]

From: Ross Burton <ross.burton@intel.com>

Skipping the tests if a toolchain wasn't installed out of the box (for example,
a minimal eSDK) doesn't make sense as the first thing the tests should do is
install a toolchain.

Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/sdkext/cases/devtool.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/meta/lib/oeqa/sdkext/cases/devtool.py b/meta/lib/oeqa/sdkext/cases/devtool.py
index 63431aa..1072fb6 100644
--- a/meta/lib/oeqa/sdkext/cases/devtool.py
+++ b/meta/lib/oeqa/sdkext/cases/devtool.py
@@ -7,7 +7,6 @@ import subprocess
 from oeqa.sdkext.case import OESDKExtTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotDataVar
 
 class DevtoolTest(OESDKExtTestCase):
     @classmethod
@@ -51,25 +50,21 @@ class DevtoolTest(OESDKExtTestCase):
 
     @OETestID(1605)
     @OETestDepends(['test_devtool_location'])
-    @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_devtool_build_make(self):
         self._test_devtool_build(self.myapp_dst)
 
     @OETestID(1606)
     @OETestDepends(['test_devtool_location'])
-    @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_devtool_build_esdk_package(self):
         self._test_devtool_build_package(self.myapp_dst)
 
     @OETestID(1607)
     @OETestDepends(['test_devtool_location'])
-    @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_devtool_build_cmake(self):
         self._test_devtool_build(self.myapp_cmake_dst)
 
     @OETestID(1608)
     @OETestDepends(['test_devtool_location'])
-    @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_extend_autotools_recipe_creation(self):
         req = 'https://github.com/rdfa/librdfa'
         recipe = "librdfa"
@@ -81,7 +76,6 @@ class DevtoolTest(OESDKExtTestCase):
 
     @OETestID(1609)
     @OETestDepends(['test_devtool_location'])
-    @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_devtool_kernelmodule(self):
         docfile = 'https://github.com/umlaeute/v4l2loopback.git'
         recipe = 'v4l2loopback-driver'
@@ -93,7 +87,6 @@ class DevtoolTest(OESDKExtTestCase):
 
     @OETestID(1610)
     @OETestDepends(['test_devtool_location'])
-    @skipIfNotDataVar('SDK_INCLUDE_TOOLCHAIN', '1', 'SDK does not include toolchain')
     def test_recipes_for_nodejs(self):
         package_nodejs = "npm://registry.npmjs.org;name=winston;version=2.2.0"
         self._run('devtool add %s ' % package_nodejs)
-- 
2.7.4

Re: [PATCH 02/12] testsdk: Handle minimal eSDK and avoid download costs

[Richard Purdie]

Sorry I missed this one when fixing the patches, I'll add a commit
message which says:

"""
When using a minimal eSDK, testing currently fails as the sdk isn't
populated. We therefore setup the eSDK under test to point at local
sstate and execute a command to ensure the toolchain is populated
since most of the tests depend on this being present.

At the same time, add in a link to DL_DIR through own-mirrors so
that tests which fetch source (e.g. the kernel module one) can use the
local stash. This cuts test execution of the kernel module test from
2000s to 120s.

We did try using DL_DIR directly but that causes uninative issues
requiring other workarounds so own-mirrors is neater.

Together these fixes unbreak eSDK testing on the autobuilder.
"""

On Wed, 2017-03-01 at 16:02 +0000, Richard Purdie wrote:
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/classes/testsdk.bbclass | 11 +++++++++++
>  1 file changed, 11 insertions(+)
> 
> diff --git a/meta/classes/testsdk.bbclass
> b/meta/classes/testsdk.bbclass
> index 75b4027..802e57f 100644
> --- a/meta/classes/testsdk.bbclass
> +++ b/meta/classes/testsdk.bbclass
> @@ -149,6 +149,17 @@ def testsdkext_main(d):
>          bb.plain("Extensible SDK testing environment: %s" % s)
>  
>          sdk_env = sdk_envs[s]
> +
> +        # Use our own SSTATE_DIR and DL_DIR so that updates to the
> eSDK come from our sstate cache
> +        # and we don't spend hours downloading kernels for the
> kernel module test
> +        with open(os.path.join(sdk_dir, 'conf', 'local.conf'), 'a+')
> as f:
> +            f.write('SSTATE_MIRRORS = "file://.* file://%s/PATH"\n'
> % test_data.get('SSTATE_DIR'))
> +            f.write('SOURCE_MIRROR_URL = "file://%s"\n' %
> test_data.get('DL_DIR'))
> +            f.write('INHERIT += "own-mirrors"')
> +
> +        # We need to do this in case we have a minimal SDK
> +        subprocess.check_output(". %s > /dev/null; devtool sdk-
> install meta-extsdk-toolchain" % sdk_env, cwd=sdk_dir, shell=True)
> +
>          tc = OESDKExtTestContext(td=test_data, logger=logger,
> sdk_dir=sdk_dir,
>              sdk_env=sdk_env,
> target_pkg_manifest=target_pkg_manifest,
>              host_pkg_manifest=host_pkg_manifest)