[PATCH v2] oeqa/runtime/cases: new image_upgrade test

[PATCH v2] oeqa/runtime/cases: new image_upgrade test

[michael.opdenacker]

From: Michael Opdenacker <michael.opdenacker@bootlin.com>

New oe-selftest and associated "testimage" test to check that generated
package feeds can be used to update the latest image built
by the Yocto Project autobuilder.

Currently, only the "core-image-full-cmdline" image with IPK packages
and the "poky-altcfg" distro is tested.

Test it by running:
oe-selftest -r image_upgrade

Signed-off-by: Michael Opdenacker <michael.opdenacker@bootlin.com>
Suggested-by: Richard Purdie <richard.purdie@linuxfoundation.org>
CC: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
CC: Bruce Ashfield <bruce.ashfield@gmail.com>
CC: Alexander Kanavin <alex.kanavin@gmail.com>

---
Changes in V2:

- Move meta/lib/oeqa/runtime/cases/opkg_sysupgrade.py
  to meta-selftest/lib/oeqa/runtime/cases/ipk_sysupgrade.py
  (notice that "ipk" is a new prefix, to match PACKAGE_CLASSES values)
  as it needs special setup via selftest, and not useful in standalone
  '-c testimage' runs.
  Suggested by Alexander Kanavin.

- Implement a more generic run_image_upgrade_test() which can be
  given any image type, image file path, image download URL,
  any PACKAGE_CLASSES value, and specific features.
  This way, the function running the tests is not supposed to be
  Yocto and Poky specific.

- Tested on the latest master against yocto-5.0_M3

Interested in further (specific) guidelines for making this test
more generic, depending on the environments calling it and their
configurations.
---
 .../lib/oeqa/runtime/cases/ipk_sysupgrade.py  |  68 +++++++++
 meta/lib/oeqa/selftest/cases/image_upgrade.py | 134 ++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 meta-selftest/lib/oeqa/runtime/cases/ipk_sysupgrade.py
 create mode 100755 meta/lib/oeqa/selftest/cases/image_upgrade.py

diff --git a/meta-selftest/lib/oeqa/runtime/cases/ipk_sysupgrade.py b/meta-selftest/lib/oeqa/runtime/cases/ipk_sysupgrade.py
new file mode 100644
index 0000000000..05b5847b4a
--- /dev/null
+++ b/meta-selftest/lib/oeqa/runtime/cases/ipk_sysupgrade.py
@@ -0,0 +1,68 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# Test that generated ipk packages can be used to upgrade
+# an older image version.
+#
+# This is done by the meta/lib/oeqa/selftest/cases/image_upgrade.py oe-selftest
+# replacing the newly generated image by an older image
+# generated by the Yocto Project autobuilder.
+#
+# Here, we replace the package feeds in our image by our own
+#
+# This test is not meant to be used as a regular "testimage" test
+# run on the fresh image.
+#
+# SPDX-License-Identifier: MIT
+#
+
+import os
+from oeqa.utils.httpserver import HTTPService
+from oeqa.runtime.case import OERuntimeTestCase
+from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature, skipIfFeature
+from oeqa.runtime.decorator.package import OEHasPackage
+
+class OpkgSysUpgradeTest(OERuntimeTestCase):
+
+    def pkg(self, command, expected = 0):
+        command = 'opkg %s' % command
+        status, output = self.target.run(command, 1500)
+        message = os.linesep.join([command, output])
+        self.assertEqual(status, expected, message)
+        return output
+
+class OpkgRepoTest(OpkgSysUpgradeTest):
+
+    @classmethod
+    def setUp(cls):
+        service_repo = os.path.join(cls.tc.td['DEPLOY_DIR_IPK'])
+        cls.repo_server = HTTPService(service_repo,
+                                      '0.0.0.0', port=cls.tc.target.server_port,
+                                      logger=cls.tc.logger)
+        cls.repo_server.start()
+
+    @classmethod
+    def tearDown(cls):
+        cls.repo_server.stop()
+
+    def setup_source_config_for_package_install(self):
+        source_server = 'http://%s:%s' % (self.tc.target.server_ip, self.repo_server.port)
+        sourceslist_dir = '/etc/opkg'
+        pkgarch = self.tc.td["TUNE_PKGARCH"]
+        machinedir = self.tc.td["MACHINE"].replace("-", "_")
+        self.target.run('cd %s; echo src/gz all %s/all > base-feeds.conf' % (sourceslist_dir, source_server))
+        self.target.run('cd %s; echo src/gz %s %s/%s >> base-feeds.conf' % (sourceslist_dir, pkgarch, source_server, pkgarch))
+        self.target.run('cd %s; echo src/gz %s %s/%s >> base-feeds.conf' % (sourceslist_dir, machinedir, source_server, machinedir))
+
+    @skipIfNotFeature('package-management',
+                      'Test requires package-management to be in IMAGE_FEATURES')
+    @skipIfNotDataVar('IMAGE_PKGTYPE', 'ipk',
+                      'IPK is not the primary package manager')
+    @skipIfFeature('read-only-rootfs',
+                   'Test does not work with read-only-rootfs in IMAGE_FEATURES')
+    @OEHasPackage(['opkg'])
+    def test_opkg_system_upgrade_from_repo(self):
+        self.setup_source_config_for_package_install()
+        self.pkg('update')
+        self.pkg('upgrade')
+
diff --git a/meta/lib/oeqa/selftest/cases/image_upgrade.py b/meta/lib/oeqa/selftest/cases/image_upgrade.py
new file mode 100755
index 0000000000..aed526483b
--- /dev/null
+++ b/meta/lib/oeqa/selftest/cases/image_upgrade.py
@@ -0,0 +1,134 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+import os
+import subprocess
+from oeqa.selftest.case import OESelftestTestCase
+from oeqa.utils.commands import bitbake, runCmd, get_bb_var
+from oeqa.core.decorator.data import skipIfNotQemu
+
+basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../')
+
+# Version string utilities copied from yocto-autobuilder-helper/scripts/utils.py
+
+def get_string_from_version(version, milestone=None, rc=None):
+    """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
+    those are major releases
+    """
+    if len(version) == 3 and version[-1] == 0:
+        version = version[:-1]
+
+    result = ".".join(list(map(str, version)))
+    if milestone:
+        result += "_M" + str(milestone)
+    if rc:
+        result += ".rc" + str(rc)
+    return result
+
+def get_tag_from_version(version, milestone):
+    if not milestone:
+        return "yocto-" + get_string_from_version(version, milestone)
+    return get_string_from_version(version, milestone)
+
+def get_version_from_string(raw_version):
+    """ Get version as list of int from raw_version.
+
+    Raw version _can_ be prefixed by "yocto-",
+    Raw version _can_ be suffixed by "_MX"
+    Raw version _can_ be suffixed by ".rcY"
+    """
+    version = None
+    milestone = None
+    rc = None
+    if raw_version[:6] == "yocto-":
+        raw_version = raw_version[6:]
+    raw_version = raw_version.split(".")
+    if raw_version[-1][:2] == "rc":
+        rc = int(raw_version[-1][-1])
+        raw_version = raw_version[:-1]
+    if raw_version[-1][-3:-1] == "_M":
+        milestone = int(raw_version[-1][-1])
+        raw_version = raw_version[:-1] + [raw_version[-1][:-3]]
+    version = list(map(int, raw_version))
+    """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
+    those are major releases
+    """
+    if len(version) == 3 and version[-1] == 0:
+        version = version[:-1]
+    return version, milestone, rc
+
+def get_poky_latest_image_url(machine, machine_variant, image_file):
+
+    """Returns the URL of the latest generated image for the current branch"""
+
+    baseversion, milestone, _ = get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=basepath).decode('utf-8').strip())
+    tag = get_tag_from_version(baseversion, milestone)
+    downloads_base = "https://downloads.yoctoproject.org/releases/yocto/"
+
+    if milestone is not None:
+        downloads_base += "milestones/yocto-%s" % tag
+    else:
+        downloads_base += tag
+
+    if machine_variant == "":
+        subdir = machine
+    else:
+        subdir = "%s-%s" % (machine, machine_variant)
+
+    return "%s/machines/qemu/%s/%s" % (downloads_base, subdir, image_file)
+
+class ImageIpkUpgrade(OESelftestTestCase):
+
+    def run_image_upgrade_test(self, image, image_path, image_url, pkg, features):
+        """
+        Summary: Test that generated packages can
+        be used to upgrade an older image version.
+        This is done by generating an image but then replacing it
+        by an older image available at a specified location.
+        We then run QEMU on the old image and replace the original
+        original package feeds by our own.
+        """
+
+        features += 'EXTRA_IMAGE_FEATURES += "package-management"\n'
+        features += 'PACKAGE_CLASSES = "package_%s"\n' % pkg
+        features += 'IMAGE_CLASSES += "testimage"\n'
+        features += 'TEST_SUITES="%s_sysupgrade"\n' % pkg
+        self.write_config(features)
+
+        # Need to build a full image to build the .json file needed by QEMU.
+        # Therefore, it is not sufficient to run only "package_write_ipk" for the image.
+
+        self.logger.info("Generating '%s' and package index for latest commit in this branch..." % image)
+        bitbake(image)
+        bitbake('package-index')
+
+        # Download previously generated image
+
+        image_full_path = '%s/%s' % (self.builddir, image_path)
+
+        os.remove(image_full_path)
+        self.logger.info("Downloading image: %s..." % image_url)
+        cmd = 'wget -O %s %s' % (image_full_path, image_url)
+        result = runCmd(cmd)
+        self.assertEqual(0, result.status, cmd + ' returned a non 0 status: %s' % result.output)
+
+        # Now run the upgrade tests on the old image
+
+        self.logger.info("Running upgrade tests on the downloaded image, using the package feeds generated here...")
+        bitbake(image + ' -c testimage')
+
+    def run_poky_image_upgrade_test(self, distro_variant, image, machine_variant, fstype, pkg):
+        machine = get_bb_var("MACHINE")
+        features = 'DISTRO = "poky%s"\n' % distro_variant
+        image_file = '%s-%s.rootfs.%s' % (image, machine, fstype)
+        image_path = 'tmp/deploy/images/%s/%s' % (machine, image_file)
+        image_url = get_poky_latest_image_url(machine, machine_variant, image_file)
+        self.run_image_upgrade_test(image, image_path, image_url, pkg, features)
+
+    @skipIfNotQemu()
+    def test_poky_altcfg_core_image_ext4_full_cmdline_ipk_upgrade(self):
+        self.run_poky_image_upgrade_test("-altcfg", "core-image-full-cmdline", "alt", "ext4", "ipk")
+
-- 
2.34.1

Re: [PATCH v2] oeqa/runtime/cases: new image_upgrade test

[Alexander Kanavin]

On Mon, 29 Apr 2024 at 17:22, <michael.opdenacker@bootlin.com> wrote:
> +# Version string utilities copied from yocto-autobuilder-helper/scripts/utils.py
> +
> +def get_string_from_version(version, milestone=None, rc=None):
> +    """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
> +    those are major releases
> +    """
> +    if len(version) == 3 and version[-1] == 0:
> +        version = version[:-1]
> +
> +    result = ".".join(list(map(str, version)))
> +    if milestone:
> +        result += "_M" + str(milestone)
> +    if rc:
> +        result += ".rc" + str(rc)
> +    return result
> +
> +def get_tag_from_version(version, milestone):
> +    if not milestone:
> +        return "yocto-" + get_string_from_version(version, milestone)
> +    return get_string_from_version(version, milestone)
> +
> +def get_version_from_string(raw_version):
> +    """ Get version as list of int from raw_version.
> +
> +    Raw version _can_ be prefixed by "yocto-",
> +    Raw version _can_ be suffixed by "_MX"
> +    Raw version _can_ be suffixed by ".rcY"
> +    """
> +    version = None
> +    milestone = None
> +    rc = None
> +    if raw_version[:6] == "yocto-":
> +        raw_version = raw_version[6:]
> +    raw_version = raw_version.split(".")
> +    if raw_version[-1][:2] == "rc":
> +        rc = int(raw_version[-1][-1])
> +        raw_version = raw_version[:-1]
> +    if raw_version[-1][-3:-1] == "_M":
> +        milestone = int(raw_version[-1][-1])
> +        raw_version = raw_version[:-1] + [raw_version[-1][:-3]]
> +    version = list(map(int, raw_version))
> +    """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
> +    those are major releases
> +    """
> +    if len(version) == 3 and version[-1] == 0:
> +        version = version[:-1]
> +    return version, milestone, rc

I understand that the above is merely a copy-paste, but these
functions would really benefit from having comments that contain
examples of input and output. I don't understand what they do.

> +def get_poky_latest_image_url(machine, machine_variant, image_file):
> +
> +    """Returns the URL of the latest generated image for the current branch"""
> +
> +    baseversion, milestone, _ = get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=basepath).decode('utf-8').strip())
> +    tag = get_tag_from_version(baseversion, milestone)
> +    downloads_base = "https://downloads.yoctoproject.org/releases/yocto/"
> +
> +    if milestone is not None:
> +        downloads_base += "milestones/yocto-%s" % tag
> +    else:
> +        downloads_base += tag
> +
> +    if machine_variant == "":
> +        subdir = machine
> +    else:
> +        subdir = "%s-%s" % (machine, machine_variant)
> +
> +    return "%s/machines/qemu/%s/%s" % (downloads_base, subdir, image_file)

Same here.  A comment with example input and output would help a lot.

> +class ImageIpkUpgrade(OESelftestTestCase):
> +
> +    def run_image_upgrade_test(self, image, image_path, image_url, pkg, features):
> +        """
> +        Summary: Test that generated packages can
> +        be used to upgrade an older image version.
> +        This is done by generating an image but then replacing it
> +        by an older image available at a specified location.
> +        We then run QEMU on the old image and replace the original
> +        original package feeds by our own.
> +        """
> +
> +        features += 'EXTRA_IMAGE_FEATURES += "package-management"\n'
> +        features += 'PACKAGE_CLASSES = "package_%s"\n' % pkg
> +        features += 'IMAGE_CLASSES += "testimage"\n'
> +        features += 'TEST_SUITES="%s_sysupgrade"\n' % pkg
> +        self.write_config(features)
> +
> +        # Need to build a full image to build the .json file needed by QEMU.
> +        # Therefore, it is not sufficient to run only "package_write_ipk" for the image.
> +
> +        self.logger.info("Generating '%s' and package index for latest commit in this branch..." % image)
> +        bitbake(image)
> +        bitbake('package-index')
> +
> +        # Download previously generated image
> +
> +        image_full_path = '%s/%s' % (self.builddir, image_path)
> +
> +        os.remove(image_full_path)
> +        self.logger.info("Downloading image: %s..." % image_url)
> +        cmd = 'wget -O %s %s' % (image_full_path, image_url)
> +        result = runCmd(cmd)
> +        self.assertEqual(0, result.status, cmd + ' returned a non 0 status: %s' % result.output)
> +
> +        # Now run the upgrade tests on the old image
> +
> +        self.logger.info("Running upgrade tests on the downloaded image, using the package feeds generated here...")
> +        bitbake(image + ' -c testimage')
> +
> +    def run_poky_image_upgrade_test(self, distro_variant, image, machine_variant, fstype, pkg):
> +        machine = get_bb_var("MACHINE")
> +        features = 'DISTRO = "poky%s"\n' % distro_variant
> +        image_file = '%s-%s.rootfs.%s' % (image, machine, fstype)
> +        image_path = 'tmp/deploy/images/%s/%s' % (machine, image_file)
> +        image_url = get_poky_latest_image_url(machine, machine_variant, image_file)
> +        self.run_image_upgrade_test(image, image_path, image_url, pkg, features)
> +
> +    @skipIfNotQemu()
> +    def test_poky_altcfg_core_image_ext4_full_cmdline_ipk_upgrade(self):
> +        self.run_poky_image_upgrade_test("-altcfg", "core-image-full-cmdline", "alt", "ext4", "ipk")

This is a much improved code structure that can scale easily, right? :)

Alex

Re: [PATCH v2] oeqa/runtime/cases: new image_upgrade test

[Richard Purdie]

On Mon, 2024-04-29 at 17:55 +0200, Alexander Kanavin wrote:
> On Mon, 29 Apr 2024 at 17:22, <michael.opdenacker@bootlin.com> wrote:
> > +# Version string utilities copied from yocto-autobuilder-helper/scripts/utils.py
> > +
> > +def get_string_from_version(version, milestone=None, rc=None):
> > +    """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
> > +    those are major releases
> > +    """
> > +    if len(version) == 3 and version[-1] == 0:
> > +        version = version[:-1]
> > +
> > +    result = ".".join(list(map(str, version)))
> > +    if milestone:
> > +        result += "_M" + str(milestone)
> > +    if rc:
> > +        result += ".rc" + str(rc)
> > +    return result
> > +
> > +def get_tag_from_version(version, milestone):
> > +    if not milestone:
> > +        return "yocto-" + get_string_from_version(version, milestone)
> > +    return get_string_from_version(version, milestone)
> > +
> > +def get_version_from_string(raw_version):
> > +    """ Get version as list of int from raw_version.
> > +
> > +    Raw version _can_ be prefixed by "yocto-",
> > +    Raw version _can_ be suffixed by "_MX"
> > +    Raw version _can_ be suffixed by ".rcY"
> > +    """
> > +    version = None
> > +    milestone = None
> > +    rc = None
> > +    if raw_version[:6] == "yocto-":
> > +        raw_version = raw_version[6:]
> > +    raw_version = raw_version.split(".")
> > +    if raw_version[-1][:2] == "rc":
> > +        rc = int(raw_version[-1][-1])
> > +        raw_version = raw_version[:-1]
> > +    if raw_version[-1][-3:-1] == "_M":
> > +        milestone = int(raw_version[-1][-1])
> > +        raw_version = raw_version[:-1] + [raw_version[-1][:-3]]
> > +    version = list(map(int, raw_version))
> > +    """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
> > +    those are major releases
> > +    """
> > +    if len(version) == 3 and version[-1] == 0:
> > +        version = version[:-1]
> > +    return version, milestone, rc
> 
> I understand that the above is merely a copy-paste, but these
> functions would really benefit from having comments that contain
> examples of input and output. I don't understand what they do.

They're coming from yocto-autobuilder-helper and there are even unit
tests for these functions there :/.


> > +def get_poky_latest_image_url(machine, machine_variant, image_file):
> > +
> > +    """Returns the URL of the latest generated image for the current branch"""
> > +
> > +    baseversion, milestone, _ = get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=basepath).decode('utf-8').strip())
> > +    tag = get_tag_from_version(baseversion, milestone)
> > +    downloads_base = "https://downloads.yoctoproject.org/releases/yocto/"
> > +
> > +    if milestone is not None:
> > +        downloads_base += "milestones/yocto-%s" % tag
> > +    else:
> > +        downloads_base += tag
> > +
> > +    if machine_variant == "":
> > +        subdir = machine
> > +    else:
> > +        subdir = "%s-%s" % (machine, machine_variant)
> > +
> > +    return "%s/machines/qemu/%s/%s" % (downloads_base, subdir, image_file)
> 
> Same here.  A comment with example input and output would help a lot.
> 
> > +class ImageIpkUpgrade(OESelftestTestCase):
> > +
> > +    def run_image_upgrade_test(self, image, image_path, image_url, pkg, features):
> > +        """
> > +        Summary: Test that generated packages can
> > +        be used to upgrade an older image version.
> > +        This is done by generating an image but then replacing it
> > +        by an older image available at a specified location.
> > +        We then run QEMU on the old image and replace the original
> > +        original package feeds by our own.
> > +        """
> > +
> > +        features += 'EXTRA_IMAGE_FEATURES += "package-management"\n'
> > +        features += 'PACKAGE_CLASSES = "package_%s"\n' % pkg
> > +        features += 'IMAGE_CLASSES += "testimage"\n'
> > +        features += 'TEST_SUITES="%s_sysupgrade"\n' % pkg
> > +        self.write_config(features)
> > +
> > +        # Need to build a full image to build the .json file needed by QEMU.
> > +        # Therefore, it is not sufficient to run only "package_write_ipk" for the image.
> > +
> > +        self.logger.info("Generating '%s' and package index for latest commit in this branch..." % image)
> > +        bitbake(image)
> > +        bitbake('package-index')
> > +
> > +        # Download previously generated image
> > +
> > +        image_full_path = '%s/%s' % (self.builddir, image_path)
> > +
> > +        os.remove(image_full_path)
> > +        self.logger.info("Downloading image: %s..." % image_url)
> > +        cmd = 'wget -O %s %s' % (image_full_path, image_url)
> > +        result = runCmd(cmd)
> > +        self.assertEqual(0, result.status, cmd + ' returned a non 0 status: %s' % result.output)
> > +
> > +        # Now run the upgrade tests on the old image
> > +
> > +        self.logger.info("Running upgrade tests on the downloaded image, using the package feeds generated here...")
> > +        bitbake(image + ' -c testimage')
> > +
> > +    def run_poky_image_upgrade_test(self, distro_variant, image, machine_variant, fstype, pkg):
> > +        machine = get_bb_var("MACHINE")
> > +        features = 'DISTRO = "poky%s"\n' % distro_variant
> > +        image_file = '%s-%s.rootfs.%s' % (image, machine, fstype)
> > +        image_path = 'tmp/deploy/images/%s/%s' % (machine, image_file)
> > +        image_url = get_poky_latest_image_url(machine, machine_variant, image_file)
> > +        self.run_image_upgrade_test(image, image_path, image_url, pkg, features)
> > +
> > +    @skipIfNotQemu()
> > +    def test_poky_altcfg_core_image_ext4_full_cmdline_ipk_upgrade(self):
> > +        self.run_poky_image_upgrade_test("-altcfg", "core-image-full-cmdline", "alt", "ext4", "ipk")
> 
> This is a much improved code structure that can scale easily, right? :)

Its getting better but the last bit of this 
(run_poky_image_upgrade_test and
test_poky_altcfg_core_image_ext4_full_cmdline_ipk_upgrade) should be in
meta-yocto/meta-poky/lib/oeqa/selftest.

Cheers,

Richard