[PATCH 0/4] eSDK support runs of compatibility SDK tests

[PATCH 0/4] eSDK support runs of compatibility SDK tests

[Aníbal Limón]

The following changes since commit b1d5267f0e31605b346af40778da0ac1ff298b78:

  README.hardware: update flashing instructions for MPC8315 (2017-06-09 17:13:45 +0100)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib fjpedraz/emptyManifests
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=fjpedraz/emptyManifests

Aníbal Limón (1):
  oeqa/sdk/cases: Fix skip of buildgcalculator test

Francisco Pedraza (3):
  lib/oe/sdk: Adds get_extra_sdk_info to reuse code in buildhistory
  meta/classes/populate_sdk: Adds support for generating eSDK manifest
    files
  oeqa/sdk/cases: Added validation for SDK compatibility tests with eSDK

 meta/classes/buildhistory.bbclass          | 19 ++++---------
 meta/classes/populate_sdk_base.bbclass     | 44 +++++++++++++++++-------------
 meta/classes/populate_sdk_ext.bbclass      | 33 ++++++++++++++++++++++
 meta/lib/oe/sdk.py                         | 19 +++++++++++++
 meta/lib/oeqa/sdk/cases/buildgalculator.py |  2 +-
 meta/lib/oeqa/sdk/cases/buildiptables.py   |  3 +-
 meta/lib/oeqa/sdk/cases/gcc.py             |  3 +-
 meta/lib/oeqa/sdk/cases/perl.py            |  3 +-
 meta/lib/oeqa/sdk/cases/python.py          |  3 +-
 9 files changed, 92 insertions(+), 37 deletions(-)

-- 
2.1.4

[PATCH 1/4] lib/oe/sdk: Adds get_extra_sdk_info to reuse code in buildhistory

[Aníbal Limón]

From: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>

This function is going to be used for generating the target and host
manifest files packages for eSDK. Added some fixes for buildhistory.bblclass,
and docstring for get_extra_sdkinfo at oe.sdk

[YOCTO #9038]

Signed-off-by: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/classes/buildhistory.bbclass | 18 ++++++------------
 meta/lib/oe/sdk.py                | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 3e907fc..bfcc88a 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -582,25 +582,19 @@ END
 python buildhistory_get_extra_sdkinfo() {
     import operator
     import math
+    from oe.sdk import get_extra_sdkinfo
+
+    sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
+    extra_info = get_extra_sdkinfo(sstate_dir)
 
     if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
             "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
-        tasksizes = {}
-        filesizes = {}
-        for root, _, files in os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')):
-            for fn in files:
-                if fn.endswith('.tgz'):
-                    fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
-                    task = fn.rsplit(':', 1)[1].split('_', 1)[1].split('.')[0]
-                    origtotal = tasksizes.get(task, 0)
-                    tasksizes[task] = origtotal + fsize
-                    filesizes[fn] = fsize
         with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f:
-            filesizes_sorted = sorted(filesizes.items(), key=operator.itemgetter(1, 0), reverse=True)
+            filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
             for fn, size in filesizes_sorted:
                 f.write('%10d KiB %s\n' % (size, fn))
         with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f:
-            tasksizes_sorted = sorted(tasksizes.items(), key=operator.itemgetter(1, 0), reverse=True)
+            tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True)
             for task, size in tasksizes_sorted:
                 f.write('%10d KiB %s\n' % (size, task))
 }
diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
index deb823b..1c5409e 100644
--- a/meta/lib/oe/sdk.py
+++ b/meta/lib/oe/sdk.py
@@ -372,5 +372,24 @@ def populate_sdk(d, manifest_dir=None):
     os.environ.clear()
     os.environ.update(env_bkp)
 
+def get_extra_sdkinfo(sstate_dir):
+    """
+    This function is going to be used for generating the target and host manifest files packages of eSDK.
+    """
+    import math
+    
+    extra_info = {}
+    extra_info['tasksizes'] = {}
+    extra_info['filesizes'] = {}
+    for root, _, files in os.walk(sstate_dir):
+        for fn in files:
+            if fn.endswith('.tgz'):
+                fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
+                task = fn.rsplit(':',1)[1].split('_',1)[1].split(',')[0]
+                origtotal = extra_info['tasksizes'].get(task, 0)
+                extra_info['tasksizes'][task] = origtotal + fsize
+                extra_info['filesizes'][fn] = fsize
+    return extra_info
+
 if __name__ == "__main__":
     pass
-- 
2.1.4

[PATCH 2/4] meta/classes/populate_sdk: Adds support for generating eSDK manifest files

[Aníbal Limón]

From: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>

Add get_extra_sdk_info to reuse code in buildhistory

The functionalities to generate SDK and eSDK manifest files are different,
the SDK comes from package information and the eSDK comes from sstate artifacts.
Only execute write_sdk_{host, target}_manifest when is on populate_sdk class.

Adds new functions write_sdk{host, target}_ext_manifest to execute on postprocess
in populate_sdk_ext because at the end we have all the sstate artifacts to
generate the manifest.

[YOCTO #9038]

Signed-off-by: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/classes/buildhistory.bbclass      |  1 -
 meta/classes/populate_sdk_base.bbclass | 44 +++++++++++++++++++---------------
 meta/classes/populate_sdk_ext.bbclass  | 33 +++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 20 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index bfcc88a..def7a64 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -581,7 +581,6 @@ END
 
 python buildhistory_get_extra_sdkinfo() {
     import operator
-    import math
     from oe.sdk import get_extra_sdkinfo
 
     sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
index 563582e..01960ee 100644
--- a/meta/classes/populate_sdk_base.bbclass
+++ b/meta/classes/populate_sdk_base.bbclass
@@ -59,15 +59,19 @@ SDK_TITLE ?= "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} SDK"
 
 SDK_TARGET_MANIFEST = "${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"
 SDK_HOST_MANIFEST = "${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"
+SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
+SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
+
 python write_target_sdk_manifest () {
-    from oe.sdk import sdk_list_installed_packages
-    from oe.utils import format_pkg_list
-    sdkmanifestdir = os.path.dirname(d.getVar("SDK_TARGET_MANIFEST"))
-    pkgs = sdk_list_installed_packages(d, True)
-    if not os.path.exists(sdkmanifestdir):
-        bb.utils.mkdirhier(sdkmanifestdir)
-    with open(d.getVar('SDK_TARGET_MANIFEST'), 'w') as output:
-        output.write(format_pkg_list(pkgs, 'ver'))
+    if d.getVar('BB_CURRENTTASK') == 'populate_sdk':
+        from oe.sdk import sdk_list_installed_packages
+        from oe.utils import format_pkg_list
+        sdkmanifestdir = os.path.dirname(d.getVar("SDK_TARGET_MANIFEST"))
+        pkgs = sdk_list_installed_packages(d, True)
+        if not os.path.exists(sdkmanifestdir):
+            bb.utils.mkdirhier(sdkmanifestdir)
+        with open(d.getVar('SDK_TARGET_MANIFEST'), 'w') as output:
+            output.write(format_pkg_list(pkgs, 'ver'))
 }
 
 python write_sdk_test_data() {
@@ -78,20 +82,22 @@ python write_sdk_test_data() {
 }
 
 python write_host_sdk_manifest () {
-    from oe.sdk import sdk_list_installed_packages
-    from oe.utils import format_pkg_list
-    sdkmanifestdir = os.path.dirname(d.getVar("SDK_HOST_MANIFEST"))
-    pkgs = sdk_list_installed_packages(d, False)
-    if not os.path.exists(sdkmanifestdir):
-        bb.utils.mkdirhier(sdkmanifestdir)
-    with open(d.getVar('SDK_HOST_MANIFEST'), 'w') as output:
-        output.write(format_pkg_list(pkgs, 'ver'))
+    if d.getVar('BB_CURRENTTASK') == 'populate_sdk':
+        from oe.sdk import sdk_list_installed_packages
+        from oe.utils import format_pkg_list
+        sdkmanifestdir = os.path.dirname(d.getVar("SDK_HOST_MANIFEST"))
+        pkgs = sdk_list_installed_packages(d, False)
+        if not os.path.exists(sdkmanifestdir):
+            bb.utils.mkdirhier(sdkmanifestdir)
+        with open(d.getVar('SDK_HOST_MANIFEST'), 'w') as output:
+            output.write(format_pkg_list(pkgs, 'ver'))
 }
 
-POPULATE_SDK_POST_TARGET_COMMAND_append = " write_target_sdk_manifest ; write_sdk_test_data ; "
-POPULATE_SDK_POST_HOST_COMMAND_append = " write_host_sdk_manifest; "
+POPULATE_SDK_POST_TARGET_COMMAND_append = " write_sdk_test_data ; "
+POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk  = " write_target_sdk_manifest ; "
+POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " write_host_sdk_manifest; "
 SDK_PACKAGING_COMMAND = "${@'${SDK_PACKAGING_FUNC};' if '${SDK_PACKAGING_FUNC}' else ''}"
-SDK_POSTPROCESS_COMMAND = " create_sdk_files; check_sdk_sysroots; tar_sdk; ${SDK_PACKAGING_COMMAND} "
+SDK_POSTPROCESS_COMMAND = " create_sdk_files; check_sdk_sysroots; tar_sdk; write_target_sdk_manifest; write_host_sdk_manifest; ${SDK_PACKAGING_COMMAND} "
 
 def populate_sdk_common(d):
     from oe.sdk import populate_sdk
diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 69bc1d9..4791d74 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -83,6 +83,39 @@ TOOLCHAIN_OUTPUTNAME_task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}"
 SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
 SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
 
+python write_target_sdk_ext_manifest () {
+    from oe.sdk import get_extra_sdkinfo
+    sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
+    extra_info = get_extra_sdkinfo(sstate_dir)
+
+    target = d.getVar('TARGET_SYS')
+    target_multimach = d.getVar('MULTIMACH_TARGET_SYS')
+    real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS')
+
+    pkgs = {}
+    with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f:
+        for fn in extra_info['filesizes']:
+            info = fn.split(':')
+            if info[2] in (target, target_multimach, real_target_multimach) \
+                    or info[5] == 'allarch':
+                if not info[1] in pkgs:
+                    f.write("%s %s %s\n" % (info[1], info[2], info[3]))
+                    pkgs[info[1]] = {}
+}
+python write_host_sdk_ext_manifest () {
+    from oe.sdk import get_extra_sdkinfo
+    sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
+    extra_info = get_extra_sdkinfo(sstate_dir)
+    host = d.getVar('BUILD_SYS')
+    with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f:
+        for fn in extra_info['filesizes']:
+            info = fn.split(':')
+            if info[2] == host:
+                f.write("%s %s %s\n" % (info[1], info[2], info[3]))
+}
+
+SDK_POSTPROCESS_COMMAND_append_task-populate-sdk-ext = "write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; "    
+
 SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK"
 
 def clean_esdk_builddir(d, sdkbasepath):
-- 
2.1.4

[PATCH 3/4] oeqa/sdk/cases: Added validation for SDK compatibility tests with eSDK

[Aníbal Limón]

From: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>

The manifests for eSDK are generated using shared states so there is a
need to validate to different "packages names" into the test cases.

For example for perl:

SDK provides nativesdk-perl
eSDK provides perl-native

[YOCTO #9090]

Signed-off-by: Francisco Pedraza <francisco.j.pedraza.gonzalez@intel.com>
Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/lib/oeqa/sdk/cases/buildiptables.py | 3 ++-
 meta/lib/oeqa/sdk/cases/gcc.py           | 3 ++-
 meta/lib/oeqa/sdk/cases/perl.py          | 3 ++-
 meta/lib/oeqa/sdk/cases/python.py        | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oeqa/sdk/cases/buildiptables.py b/meta/lib/oeqa/sdk/cases/buildiptables.py
index 0bd00d1..419c7eb 100644
--- a/meta/lib/oeqa/sdk/cases/buildiptables.py
+++ b/meta/lib/oeqa/sdk/cases/buildiptables.py
@@ -17,7 +17,8 @@ class BuildIptablesTest(OESDKTestCase):
 
         machine = self.td.get("MACHINE")
 
-        if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine):
+        if not (self.tc.hasTargetPackage("packagegroup-cross-canadian-%s" % machine) or
+                self.tc.hasTargetPackage("gcc")):
             raise unittest.SkipTest("SDK doesn't contain a cross-canadian toolchain")
 
     def test_iptables(self):
diff --git a/meta/lib/oeqa/sdk/cases/gcc.py b/meta/lib/oeqa/sdk/cases/gcc.py
index 74ad2a2..d11f4b6 100644
--- a/meta/lib/oeqa/sdk/cases/gcc.py
+++ b/meta/lib/oeqa/sdk/cases/gcc.py
@@ -18,7 +18,8 @@ class GccCompileTest(OESDKTestCase):
 
     def setUp(self):
         machine = self.td.get("MACHINE")
-        if not self.tc.hasHostPackage("packagegroup-cross-canadian-%s" % machine):
+        if not (self.tc.hasTargetPackage("packagegroup-cross-canadian-%s" % machine) or
+                self.tc.hasTargetPackage("gcc")):
             raise unittest.SkipTest("GccCompileTest class: SDK doesn't contain a cross-canadian toolchain")
 
     def test_gcc_compile(self):
diff --git a/meta/lib/oeqa/sdk/cases/perl.py b/meta/lib/oeqa/sdk/cases/perl.py
index e1bded2..8085678 100644
--- a/meta/lib/oeqa/sdk/cases/perl.py
+++ b/meta/lib/oeqa/sdk/cases/perl.py
@@ -8,7 +8,8 @@ from oeqa.sdk.case import OESDKTestCase
 class PerlTest(OESDKTestCase):
     @classmethod
     def setUpClass(self):
-        if not self.tc.hasHostPackage("nativesdk-perl"):
+        if not (self.tc.hasHostPackage("nativesdk-perl") or
+                self.tc.hasHostPackage("perl-native")):
             raise unittest.SkipTest("No perl package in the SDK")
 
         for f in ['test.pl']:
diff --git a/meta/lib/oeqa/sdk/cases/python.py b/meta/lib/oeqa/sdk/cases/python.py
index 94a296f..72dfcc7 100644
--- a/meta/lib/oeqa/sdk/cases/python.py
+++ b/meta/lib/oeqa/sdk/cases/python.py
@@ -8,7 +8,8 @@ from oeqa.sdk.case import OESDKTestCase
 class PythonTest(OESDKTestCase):
     @classmethod
     def setUpClass(self):
-        if not self.tc.hasHostPackage("nativesdk-python"):
+        if not (self.tc.hasHostPackage("nativesdk-python") or
+                self.tc.hasHostPackage("python-native")):
             raise unittest.SkipTest("No python package in the SDK")
 
         for f in ['test.py']:
-- 
2.1.4

[PATCH 4/4] oeqa/sdk/cases: Fix skip of buildgcalculator test

[Aníbal Limón]

The tc.hasTargetPackage uses a re.search to see if gtk+3 is on
the manifest but + in regex means 1 or more causing the test
to be skipped.

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/lib/oeqa/sdk/cases/buildgalculator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/sdk/cases/buildgalculator.py b/meta/lib/oeqa/sdk/cases/buildgalculator.py
index 42e8ddb..780afcc 100644
--- a/meta/lib/oeqa/sdk/cases/buildgalculator.py
+++ b/meta/lib/oeqa/sdk/cases/buildgalculator.py
@@ -8,7 +8,7 @@ class GalculatorTest(OESDKTestCase):
 
     @classmethod
     def setUpClass(self):
-        if not (self.tc.hasTargetPackage("gtk+3") or\
+        if not (self.tc.hasTargetPackage("gtk\+3") or\
                 self.tc.hasTargetPackage("libgtk-3.0")):
             raise unittest.SkipTest("GalculatorTest class: SDK don't support gtk+3")
 
-- 
2.1.4