[PATCHv2 0/5] Fix bugs found within runtime testing

[PATCHv2 0/5] Fix bugs found within runtime testing

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This will fix some bugs found in testimage when executing tests
from images downloaded from the autobuilders.

Changes in v2:

- Fix some test cases that were changed during the migration.
- Add createrepo-native as dependency of test image.
- Fix TEST_SUITES in core-image-sato-sdk and core-image-lsb-sdk.
- Renamed syslog test to oe_syslog to make it run on debian/ubuntu.

The following changes since commit 0150dddeaf901c91d32be093efd0348a344fb545:

  sanity.conf: Update minimum bitbake version requirements to 1.33.0 (2017-01-27 10:42:39 +0000)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib mariano/bug10964
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug10964

Aníbal Limón (2):
  oeqa/core/context.py: Add validation for run-tests option
  oeqa/runtime/cases: Rename syslog module to oe_syslog

Mariano Lopez (3):
  runtime/cases: Fix case numbers, missing cases and unused classes
  runtime/cases/smart.py: Check for IMAGE_PKGTYPE instead of
    PACKAGE_CLASSES
  testimage.bbclass: Allow to run tests on autobuilder's images

 meta/classes/testimage.bbclass                       | 20 ++++++++++++--------
 meta/lib/oeqa/core/context.py                        |  6 +++++-
 meta/lib/oeqa/core/utils/misc.py                     |  7 +++++++
 meta/lib/oeqa/runtime/cases/connman.py               |  2 +-
 meta/lib/oeqa/runtime/cases/gcc.py                   |  3 ---
 meta/lib/oeqa/runtime/cases/multilib.py              |  2 +-
 .../oeqa/runtime/cases/{syslog.py => oe_syslog.py}   | 13 +++++++++++--
 meta/lib/oeqa/runtime/cases/smart.py                 |  2 +-
 8 files changed, 38 insertions(+), 17 deletions(-)
 rename meta/lib/oeqa/runtime/cases/{syslog.py => oe_syslog.py} (82%)

-- 
2.6.6

[PATCHv2 1/5] runtime/cases: Fix case numbers, missing cases and unused classes

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

connman: oeRuntimeTest class is not used anymore as part of runtime
migration, this particular case was missed, so fix it.

gcc: Removed unneded lines.

multilib: fixed case number.

syslog: added a missing test.

[YOCTO #10964]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/lib/oeqa/runtime/cases/connman.py  | 2 +-
 meta/lib/oeqa/runtime/cases/gcc.py      | 3 ---
 meta/lib/oeqa/runtime/cases/multilib.py | 2 +-
 meta/lib/oeqa/runtime/cases/syslog.py   | 9 +++++++++
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/meta/lib/oeqa/runtime/cases/connman.py b/meta/lib/oeqa/runtime/cases/connman.py
index 8b47108..12456b4 100644
--- a/meta/lib/oeqa/runtime/cases/connman.py
+++ b/meta/lib/oeqa/runtime/cases/connman.py
@@ -6,7 +6,7 @@ from oeqa.runtime.decorator.package import OEHasPackage
 class ConnmanTest(OERuntimeTestCase):
 
     def service_status(self, service):
-        if oeRuntimeTest.hasFeature("systemd"):
+        if 'systemd' in self.tc.td['DISTRO_FEATURES']:
             (_, output) = self.target.run('systemctl status -l %s' % service)
             return output
         else:
diff --git a/meta/lib/oeqa/runtime/cases/gcc.py b/meta/lib/oeqa/runtime/cases/gcc.py
index 064fa49..9110831 100644
--- a/meta/lib/oeqa/runtime/cases/gcc.py
+++ b/meta/lib/oeqa/runtime/cases/gcc.py
@@ -11,15 +11,12 @@ class GccCompileTest(OERuntimeTestCase):
     def setUpClass(cls):
         dst = '/tmp/'
         src = os.path.join(cls.tc.files_dir, 'test.c')
-        #dst = '/tmp/test.c'
         cls.tc.target.copyTo(src, dst)
 
         src = os.path.join(cls.tc.runtime_files_dir, 'testmakefile')
-        #dst = '/tmp/testmakefile'
         cls.tc.target.copyTo(src, dst)
 
         src = os.path.join(cls.tc.files_dir, 'test.cpp')
-        #dst = '/tmp/test.cpp'
         cls.tc.target.copyTo(src, dst)
 
     @classmethod
diff --git a/meta/lib/oeqa/runtime/cases/multilib.py b/meta/lib/oeqa/runtime/cases/multilib.py
index 8f6d2b2..8c167f1 100644
--- a/meta/lib/oeqa/runtime/cases/multilib.py
+++ b/meta/lib/oeqa/runtime/cases/multilib.py
@@ -23,9 +23,9 @@ class MultilibTest(OERuntimeTestCase):
         msg = "%s isn't %s (is %s)" % (binary, arch, theclass)
         self.assertEqual(theclass, arch, msg=msg)
 
+    @OETestID(1593)
     @skipIfNotInDataVar('MULTILIBS', 'multilib:lib32',
                         "This isn't a multilib:lib32 image")
-    @OETestID(201)
     @OETestDepends(['ssh.SSHTest.test_ssh'])
     def test_check_multilib_libc(self):
         """
diff --git a/meta/lib/oeqa/runtime/cases/syslog.py b/meta/lib/oeqa/runtime/cases/syslog.py
index 1016e67..537c519 100644
--- a/meta/lib/oeqa/runtime/cases/syslog.py
+++ b/meta/lib/oeqa/runtime/cases/syslog.py
@@ -34,6 +34,15 @@ class SyslogTestConfig(OERuntimeTestCase):
                ' Output: %s ' % output)
         self.assertEqual(status, 0, msg=msg)
 
+    @OETestID(1150)
+    @OETestDepends(['syslog.SyslogTest.test_syslog_running'])
+    def test_syslog_restart(self):
+        if "systemd" != self.tc.td.get("VIRTUAL-RUNTIME_init_manager", ""):
+            (_, _) = self.target.run('/etc/init.d/syslog restart')
+        else:
+            (_, _) = self.target.run('systemctl restart syslog.service')
+
+
     @OETestID(202)
     @OETestDepends(['syslog.SyslogTestConfig.test_syslog_logger'])
     @OEHasPackage(["!sysklogd", "busybox"])
-- 
2.6.6

[PATCHv2 2/5] runtime/cases/smart.py: Check for IMAGE_PKGTYPE instead of PACKAGE_CLASSES

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

smart test requires to build the image using rpm packages, this check was
included, but it checked for PACKAGE_CLASSES=='package_rpm', and this is
not true when building packages for rpm and deb/ipk. So this would check
IMAGE_PKGTYPE instead.

[YOCTO #10964]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/lib/oeqa/runtime/cases/smart.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/runtime/cases/smart.py b/meta/lib/oeqa/runtime/cases/smart.py
index 9b4d0d2..79bd9c8 100644
--- a/meta/lib/oeqa/runtime/cases/smart.py
+++ b/meta/lib/oeqa/runtime/cases/smart.py
@@ -23,7 +23,7 @@ class SmartBasicTest(SmartTest):
 
     @skipIfNotFeature('package-management',
                       'Test requires package-management to be in IMAGE_FEATURES')
-    @skipIfNotDataVar('PACKAGE_CLASSES', 'package_rpm',
+    @skipIfNotDataVar('IMAGE_PKGTYPE', 'rpm',
                       'RPM is not the primary package manager')
     @OEHasPackage(['smartpm'])
     @OETestID(716)
-- 
2.6.6

[PATCHv2 3/5] testimage.bbclass: Allow to run tests on autobuilder's images

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

With the change to the new framework data store dependecy was
removed, instead a new file is generated and used in testimage.
When testing builds from the autobuilders the test data values
are from the autobuilder, including the paths.

Some tests require paths to current environment in order to run,
this commit will update such paths and fix the error of running
images donwloaded from autobuilders.

[YOCTO #10964]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/testimage.bbclass   | 18 +++++++++++-------
 meta/lib/oeqa/core/utils/misc.py |  7 +++++++
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 73a5c1a..7495fe1 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -49,11 +49,11 @@ DEFAULT_TEST_SUITES_pn-core-image-x11 = "${MINTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-lsb = "${NETTESTSUITE} pam parselogs ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs ${RPMTESTSUITE} \
     ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python', '', d)}"
-DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} connman xorg perl python \
-    ${DEVTESTSUITE} parselogs ${RPMTESTSUITE}"
+DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} buildcvs buildiptables buildgalculator \
+    connman ${DEVTESTSUITE} logrotate perl parselogs python ${RPMTESTSUITE} xorg"
 DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcvs buildiptables buildgalculator \
-    connman ${DEVTESTSUITE} pam perl python parselogs ${RPMTESTSUITE}"
+    connman ${DEVTESTSUITE} logrotate pam parselogs perl python ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto"
 
 # aarch64 has no graphics
@@ -79,13 +79,15 @@ TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python-smartp
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'ipk', 'opkg-utils-native:do_populate_sysroot', '', d)}"
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'deb', 'apt-native:do_populate_sysroot', '', d)}"
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python-smartpm-native:do_populate_sysroot', '', d)}"
-
+TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-native:do_populate_sysroot', '', d)}"
 
 TESTIMAGELOCK = "${TMPDIR}/testimage.lock"
 TESTIMAGELOCK_qemuall = ""
 
 TESTIMAGE_DUMP_DIR ?= "/tmp/oe-saved-tests/"
 
+TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR"
+
 testimage_dump_target () {
     top -bn1
     ps
@@ -138,13 +140,12 @@ def testimage_sanity(d):
 
 def testimage_main(d):
     import os
-    import signal
     import json
-    import sys
+    import signal
     import logging
-    import time
 
     from bb.utils import export_proxies
+    from oeqa.core.utils.misc import updateTestData
     from oeqa.runtime.context import OERuntimeTestContext
     from oeqa.runtime.context import OERuntimeTestContextExecutor
     from oeqa.core.target.qemu import supported_fstypes
@@ -166,6 +167,9 @@ def testimage_main(d):
 
     tdname = "%s.testdata.json" % image_name
     td = json.load(open(tdname, "r"))
+    # Some variables need to be updates (mostly paths) with the
+    # ones of the current environment because some tests require them.
+    updateTestData(d, td, d.getVar('TESTIMAGE_UPDATE_VARS').split())
 
     image_manifest = "%s.manifest" % image_name
     image_packages = OERuntimeTestContextExecutor.readPackagesManifest(image_manifest)
diff --git a/meta/lib/oeqa/core/utils/misc.py b/meta/lib/oeqa/core/utils/misc.py
index 6ae58ad..d1eec13 100644
--- a/meta/lib/oeqa/core/utils/misc.py
+++ b/meta/lib/oeqa/core/utils/misc.py
@@ -35,3 +35,10 @@ def dataStoteToDict(d, variables):
         data[v] = d.getVar(v, True)
 
     return data
+
+def updateTestData(d, td, variables):
+    """
+    Updates variables with values of data store to test data.
+    """
+    for var in variables:
+        td[var] = d.getVar(var)
-- 
2.6.6

[PATCHv2 4/5] oeqa/core/context.py: Add validation for run-tests option

[mariano.lopez]

From: Aníbal Limón <anibal.limon@linux.intel.com>

The run-tests option is optional so if isn't specified set
to None instead of crash on split().

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/lib/oeqa/core/context.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index efed4e6..4476750 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -215,7 +215,11 @@ class OETestContextExecutor(object):
         else:
             self.tc_kwargs['init']['td'] = {}
 
-        self.tc_kwargs['load']['modules'] = args.run_tests.split()
+
+        if args.run_tests:
+            self.tc_kwargs['load']['modules'] = args.run_tests.split()
+        else:
+            self.tc_kwargs['load']['modules'] = None
 
         self.module_paths = args.CASES_PATHS
 
-- 
2.6.6

[PATCHv2 5/5] oeqa/runtime/cases: Rename syslog module to oe_syslog

[mariano.lopez]

From: Aníbal Limón <anibal.limon@linux.intel.com>

Debian based distros has a builtin syslog module so when
try to load tests using unittest it references the builtin
module instead of runtime/cases.

[YOCTO ##10964]

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/testimage.bbclass                          | 2 +-
 meta/lib/oeqa/runtime/cases/{syslog.py => oe_syslog.py} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename meta/lib/oeqa/runtime/cases/{syslog.py => oe_syslog.py} (92%)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 7495fe1..c6e4cec 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -38,7 +38,7 @@ TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged"
 RPMTESTSUITE = "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'smart rpm', '', d)}"
 SYSTEMDSUITE = "${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'systemd', '', d)}"
 MINTESTSUITE = "ping"
-NETTESTSUITE = "${MINTESTSUITE} ssh df date scp syslog ${SYSTEMDSUITE}"
+NETTESTSUITE = "${MINTESTSUITE} ssh df date scp oe_syslog ${SYSTEMDSUITE}"
 DEVTESTSUITE = "gcc kernelmodule ldd"
 
 DEFAULT_TEST_SUITES = "${MINTESTSUITE} auto"
diff --git a/meta/lib/oeqa/runtime/cases/syslog.py b/meta/lib/oeqa/runtime/cases/oe_syslog.py
similarity index 92%
rename from meta/lib/oeqa/runtime/cases/syslog.py
rename to meta/lib/oeqa/runtime/cases/oe_syslog.py
index 537c519..005b697 100644
--- a/meta/lib/oeqa/runtime/cases/syslog.py
+++ b/meta/lib/oeqa/runtime/cases/oe_syslog.py
@@ -18,7 +18,7 @@ class SyslogTest(OERuntimeTestCase):
 class SyslogTestConfig(OERuntimeTestCase):
 
     @OETestID(1149)
-    @OETestDepends(['syslog.SyslogTest.test_syslog_running'])
+    @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running'])
     def test_syslog_logger(self):
         status, output = self.target.run('logger foobar')
         msg = "Can't log into syslog. Output: %s " % output
@@ -35,7 +35,7 @@ class SyslogTestConfig(OERuntimeTestCase):
         self.assertEqual(status, 0, msg=msg)
 
     @OETestID(1150)
-    @OETestDepends(['syslog.SyslogTest.test_syslog_running'])
+    @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running'])
     def test_syslog_restart(self):
         if "systemd" != self.tc.td.get("VIRTUAL-RUNTIME_init_manager", ""):
             (_, _) = self.target.run('/etc/init.d/syslog restart')
@@ -44,7 +44,7 @@ class SyslogTestConfig(OERuntimeTestCase):
 
 
     @OETestID(202)
-    @OETestDepends(['syslog.SyslogTestConfig.test_syslog_logger'])
+    @OETestDepends(['oe_syslog.SyslogTestConfig.test_syslog_logger'])
     @OEHasPackage(["!sysklogd", "busybox"])
     @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd',
                    'Not appropiate for systemd image')
-- 
2.6.6

✗ patchtest: failure for Fix bugs found within runtime testing (rev2)

[Patchwork]

== Series Details ==

Series: Fix bugs found within runtime testing (rev2)
Revision: 2
URL   : https://patchwork.openembedded.org/series/4986/
State : failure

== Summary ==


Thank you for submitting this patch series to OpenEmbedded Core. This is
an automated response. Several tests have been executed on the proposed
series by patchtest resulting in the following failures:



* Patch            [PATCHv2,5/5] oeqa/runtime/cases: Rename syslog module to oe_syslog
 Issue             Yocto Project bugzilla tag is not correctly formatted [test_bugzilla_entry_format] 
  Suggested fix    Specify bugzilla ID in commit description with format: "[YOCTO #<bugzilla ID>]"



If you believe any of these test results are incorrect, please reply to the
mailing list (openembedded-core@lists.openembedded.org) raising your concerns.
Otherwise we would appreciate you correcting the issues and submitting a new
version of the patchset if applicable. Please ensure you add/increment the
version number when sending the new version (i.e. [PATCH] -> [PATCH v2] ->
[PATCH v3] -> ...).

---
Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest
Test suite:     http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe

[PATCHv3 1/5] runtime/cases: Fix case numbers, missing cases and unused classes

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

connman: oeRuntimeTest class is not used anymore as part of runtime
migration, this particular case was missed, so fix it.

gcc: Removed unneded lines.

multilib: fixed case number.

syslog: added a missing test.

[YOCTO #10964]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/lib/oeqa/runtime/cases/connman.py  | 2 +-
 meta/lib/oeqa/runtime/cases/gcc.py      | 3 ---
 meta/lib/oeqa/runtime/cases/multilib.py | 2 +-
 meta/lib/oeqa/runtime/cases/syslog.py   | 9 +++++++++
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/meta/lib/oeqa/runtime/cases/connman.py b/meta/lib/oeqa/runtime/cases/connman.py
index 8b47108..12456b4 100644
--- a/meta/lib/oeqa/runtime/cases/connman.py
+++ b/meta/lib/oeqa/runtime/cases/connman.py
@@ -6,7 +6,7 @@ from oeqa.runtime.decorator.package import OEHasPackage
 class ConnmanTest(OERuntimeTestCase):
 
     def service_status(self, service):
-        if oeRuntimeTest.hasFeature("systemd"):
+        if 'systemd' in self.tc.td['DISTRO_FEATURES']:
             (_, output) = self.target.run('systemctl status -l %s' % service)
             return output
         else:
diff --git a/meta/lib/oeqa/runtime/cases/gcc.py b/meta/lib/oeqa/runtime/cases/gcc.py
index 064fa49..9110831 100644
--- a/meta/lib/oeqa/runtime/cases/gcc.py
+++ b/meta/lib/oeqa/runtime/cases/gcc.py
@@ -11,15 +11,12 @@ class GccCompileTest(OERuntimeTestCase):
     def setUpClass(cls):
         dst = '/tmp/'
         src = os.path.join(cls.tc.files_dir, 'test.c')
-        #dst = '/tmp/test.c'
         cls.tc.target.copyTo(src, dst)
 
         src = os.path.join(cls.tc.runtime_files_dir, 'testmakefile')
-        #dst = '/tmp/testmakefile'
         cls.tc.target.copyTo(src, dst)
 
         src = os.path.join(cls.tc.files_dir, 'test.cpp')
-        #dst = '/tmp/test.cpp'
         cls.tc.target.copyTo(src, dst)
 
     @classmethod
diff --git a/meta/lib/oeqa/runtime/cases/multilib.py b/meta/lib/oeqa/runtime/cases/multilib.py
index 8f6d2b2..8c167f1 100644
--- a/meta/lib/oeqa/runtime/cases/multilib.py
+++ b/meta/lib/oeqa/runtime/cases/multilib.py
@@ -23,9 +23,9 @@ class MultilibTest(OERuntimeTestCase):
         msg = "%s isn't %s (is %s)" % (binary, arch, theclass)
         self.assertEqual(theclass, arch, msg=msg)
 
+    @OETestID(1593)
     @skipIfNotInDataVar('MULTILIBS', 'multilib:lib32',
                         "This isn't a multilib:lib32 image")
-    @OETestID(201)
     @OETestDepends(['ssh.SSHTest.test_ssh'])
     def test_check_multilib_libc(self):
         """
diff --git a/meta/lib/oeqa/runtime/cases/syslog.py b/meta/lib/oeqa/runtime/cases/syslog.py
index 1016e67..537c519 100644
--- a/meta/lib/oeqa/runtime/cases/syslog.py
+++ b/meta/lib/oeqa/runtime/cases/syslog.py
@@ -34,6 +34,15 @@ class SyslogTestConfig(OERuntimeTestCase):
                ' Output: %s ' % output)
         self.assertEqual(status, 0, msg=msg)
 
+    @OETestID(1150)
+    @OETestDepends(['syslog.SyslogTest.test_syslog_running'])
+    def test_syslog_restart(self):
+        if "systemd" != self.tc.td.get("VIRTUAL-RUNTIME_init_manager", ""):
+            (_, _) = self.target.run('/etc/init.d/syslog restart')
+        else:
+            (_, _) = self.target.run('systemctl restart syslog.service')
+
+
     @OETestID(202)
     @OETestDepends(['syslog.SyslogTestConfig.test_syslog_logger'])
     @OEHasPackage(["!sysklogd", "busybox"])
-- 
2.6.6

[PATCHv3 2/5] runtime/cases/smart.py: Check for IMAGE_PKGTYPE instead of PACKAGE_CLASSES

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

smart test requires to build the image using rpm packages, this check was
included, but it checked for PACKAGE_CLASSES=='package_rpm', and this is
not true when building packages for rpm and deb/ipk. So this would check
IMAGE_PKGTYPE instead.

[YOCTO #10964]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/lib/oeqa/runtime/cases/smart.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/runtime/cases/smart.py b/meta/lib/oeqa/runtime/cases/smart.py
index 9b4d0d2..79bd9c8 100644
--- a/meta/lib/oeqa/runtime/cases/smart.py
+++ b/meta/lib/oeqa/runtime/cases/smart.py
@@ -23,7 +23,7 @@ class SmartBasicTest(SmartTest):
 
     @skipIfNotFeature('package-management',
                       'Test requires package-management to be in IMAGE_FEATURES')
-    @skipIfNotDataVar('PACKAGE_CLASSES', 'package_rpm',
+    @skipIfNotDataVar('IMAGE_PKGTYPE', 'rpm',
                       'RPM is not the primary package manager')
     @OEHasPackage(['smartpm'])
     @OETestID(716)
-- 
2.6.6

[PATCHv3 3/5] testimage.bbclass: Allow to run tests on autobuilder's images

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

With the change to the new framework data store dependecy was
removed, instead a new file is generated and used in testimage.
When testing builds from the autobuilders the test data values
are from the autobuilder, including the paths.

Some tests require paths to current environment in order to run,
this commit will update such paths and fix the error of running
images donwloaded from autobuilders.

[YOCTO #10964]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/testimage.bbclass   | 18 +++++++++++-------
 meta/lib/oeqa/core/utils/misc.py |  7 +++++++
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 73a5c1a..7495fe1 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -49,11 +49,11 @@ DEFAULT_TEST_SUITES_pn-core-image-x11 = "${MINTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-lsb = "${NETTESTSUITE} pam parselogs ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs ${RPMTESTSUITE} \
     ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python', '', d)}"
-DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} connman xorg perl python \
-    ${DEVTESTSUITE} parselogs ${RPMTESTSUITE}"
+DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} buildcvs buildiptables buildgalculator \
+    connman ${DEVTESTSUITE} logrotate perl parselogs python ${RPMTESTSUITE} xorg"
 DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcvs buildiptables buildgalculator \
-    connman ${DEVTESTSUITE} pam perl python parselogs ${RPMTESTSUITE}"
+    connman ${DEVTESTSUITE} logrotate pam parselogs perl python ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto"
 
 # aarch64 has no graphics
@@ -79,13 +79,15 @@ TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python-smartp
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'ipk', 'opkg-utils-native:do_populate_sysroot', '', d)}"
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'deb', 'apt-native:do_populate_sysroot', '', d)}"
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python-smartpm-native:do_populate_sysroot', '', d)}"
-
+TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-native:do_populate_sysroot', '', d)}"
 
 TESTIMAGELOCK = "${TMPDIR}/testimage.lock"
 TESTIMAGELOCK_qemuall = ""
 
 TESTIMAGE_DUMP_DIR ?= "/tmp/oe-saved-tests/"
 
+TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR"
+
 testimage_dump_target () {
     top -bn1
     ps
@@ -138,13 +140,12 @@ def testimage_sanity(d):
 
 def testimage_main(d):
     import os
-    import signal
     import json
-    import sys
+    import signal
     import logging
-    import time
 
     from bb.utils import export_proxies
+    from oeqa.core.utils.misc import updateTestData
     from oeqa.runtime.context import OERuntimeTestContext
     from oeqa.runtime.context import OERuntimeTestContextExecutor
     from oeqa.core.target.qemu import supported_fstypes
@@ -166,6 +167,9 @@ def testimage_main(d):
 
     tdname = "%s.testdata.json" % image_name
     td = json.load(open(tdname, "r"))
+    # Some variables need to be updates (mostly paths) with the
+    # ones of the current environment because some tests require them.
+    updateTestData(d, td, d.getVar('TESTIMAGE_UPDATE_VARS').split())
 
     image_manifest = "%s.manifest" % image_name
     image_packages = OERuntimeTestContextExecutor.readPackagesManifest(image_manifest)
diff --git a/meta/lib/oeqa/core/utils/misc.py b/meta/lib/oeqa/core/utils/misc.py
index 6ae58ad..d1eec13 100644
--- a/meta/lib/oeqa/core/utils/misc.py
+++ b/meta/lib/oeqa/core/utils/misc.py
@@ -35,3 +35,10 @@ def dataStoteToDict(d, variables):
         data[v] = d.getVar(v, True)
 
     return data
+
+def updateTestData(d, td, variables):
+    """
+    Updates variables with values of data store to test data.
+    """
+    for var in variables:
+        td[var] = d.getVar(var)
-- 
2.6.6

[PATCHv3 4/5] oeqa/core/context.py: Add validation for run-tests option

[mariano.lopez]

From: Aníbal Limón <anibal.limon@linux.intel.com>

The run-tests option is optional so if isn't specified set
to None instead of crash on split().

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/lib/oeqa/core/context.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index efed4e6..4476750 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -215,7 +215,11 @@ class OETestContextExecutor(object):
         else:
             self.tc_kwargs['init']['td'] = {}
 
-        self.tc_kwargs['load']['modules'] = args.run_tests.split()
+
+        if args.run_tests:
+            self.tc_kwargs['load']['modules'] = args.run_tests.split()
+        else:
+            self.tc_kwargs['load']['modules'] = None
 
         self.module_paths = args.CASES_PATHS
 
-- 
2.6.6

[PATCHv3 5/5] oeqa/runtime/cases: Rename syslog module to oe_syslog

[mariano.lopez]

From: Aníbal Limón <anibal.limon@linux.intel.com>

Debian based distros has a builtin syslog module so when
try to load tests using unittest it references the builtin
module instead of runtime/cases.

[YOCTO #10964]

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/testimage.bbclass                          | 2 +-
 meta/lib/oeqa/runtime/cases/{syslog.py => oe_syslog.py} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename meta/lib/oeqa/runtime/cases/{syslog.py => oe_syslog.py} (92%)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 7495fe1..c6e4cec 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -38,7 +38,7 @@ TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged"
 RPMTESTSUITE = "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'smart rpm', '', d)}"
 SYSTEMDSUITE = "${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'systemd', '', d)}"
 MINTESTSUITE = "ping"
-NETTESTSUITE = "${MINTESTSUITE} ssh df date scp syslog ${SYSTEMDSUITE}"
+NETTESTSUITE = "${MINTESTSUITE} ssh df date scp oe_syslog ${SYSTEMDSUITE}"
 DEVTESTSUITE = "gcc kernelmodule ldd"
 
 DEFAULT_TEST_SUITES = "${MINTESTSUITE} auto"
diff --git a/meta/lib/oeqa/runtime/cases/syslog.py b/meta/lib/oeqa/runtime/cases/oe_syslog.py
similarity index 92%
rename from meta/lib/oeqa/runtime/cases/syslog.py
rename to meta/lib/oeqa/runtime/cases/oe_syslog.py
index 537c519..005b697 100644
--- a/meta/lib/oeqa/runtime/cases/syslog.py
+++ b/meta/lib/oeqa/runtime/cases/oe_syslog.py
@@ -18,7 +18,7 @@ class SyslogTest(OERuntimeTestCase):
 class SyslogTestConfig(OERuntimeTestCase):
 
     @OETestID(1149)
-    @OETestDepends(['syslog.SyslogTest.test_syslog_running'])
+    @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running'])
     def test_syslog_logger(self):
         status, output = self.target.run('logger foobar')
         msg = "Can't log into syslog. Output: %s " % output
@@ -35,7 +35,7 @@ class SyslogTestConfig(OERuntimeTestCase):
         self.assertEqual(status, 0, msg=msg)
 
     @OETestID(1150)
-    @OETestDepends(['syslog.SyslogTest.test_syslog_running'])
+    @OETestDepends(['oe_syslog.SyslogTest.test_syslog_running'])
     def test_syslog_restart(self):
         if "systemd" != self.tc.td.get("VIRTUAL-RUNTIME_init_manager", ""):
             (_, _) = self.target.run('/etc/init.d/syslog restart')
@@ -44,7 +44,7 @@ class SyslogTestConfig(OERuntimeTestCase):
 
 
     @OETestID(202)
-    @OETestDepends(['syslog.SyslogTestConfig.test_syslog_logger'])
+    @OETestDepends(['oe_syslog.SyslogTestConfig.test_syslog_logger'])
     @OEHasPackage(["!sysklogd", "busybox"])
     @skipIfDataVar('VIRTUAL-RUNTIME_init_manager', 'systemd',
                    'Not appropiate for systemd image')
-- 
2.6.6