[PATCH 1/5] overlayfs: all overlays unit

[PATCH 1/5] overlayfs: all overlays unit

[Vyacheslav Yurkov]

Application can depend on several overlayfs mount points. Provide a
systemd unit application can depend on to make sure all overlays are
mounted before it is started to avoid any race conditions

Signed-off-by: Bruno Knittel <Bruno.Knittel@bruker.com>
Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/classes/overlayfs.bbclass | 36 ++++++++++++++++++++++++++++++++--
 meta/lib/oe/overlayfs.py       |  5 +++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index 8d9b59c9bf..9397ab44f9 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -37,6 +37,8 @@ REQUIRED_DISTRO_FEATURES += "systemd overlayfs"
 inherit systemd features_check
 
 python do_create_overlayfs_units() {
+    from oe.overlayfs import mountUnitName
+
     CreateDirsUnitTemplate = """[Unit]
 Description=Overlayfs directories setup
 Requires={DATA_MOUNT_UNIT}
@@ -65,10 +67,23 @@ Options=lowerdir={LOWERDIR},upperdir={DATA_MOUNT_POINT}/upper{LOWERDIR},workdir=
 
 [Install]
 WantedBy=multi-user.target
+"""
+    AllOverlaysTemplate = """[Unit]
+Description=Groups all overlays required by {PN} in one unit
+After={ALL_OVERLAYFS_UNITS}
+Requires={ALL_OVERLAYFS_UNITS}
+
+[Service]
+Type=oneshot
+ExecStart=/bin/true
+RemainAfterExit=true
+
+[Install]
+WantedBy=local-fs.target
 """
 
     def prepareUnits(data, lower):
-        from oe.overlayfs import mountUnitName, helperUnitName
+        from oe.overlayfs import helperUnitName
 
         args = {
             'DATA_MOUNT_POINT': data,
@@ -83,10 +98,27 @@ WantedBy=multi-user.target
         with open(os.path.join(d.getVar('WORKDIR'), helperUnitName(lower)), 'w') as f:
             f.write(CreateDirsUnitTemplate.format(**args))
 
+    def prepareGlobalUnit(dependentUnits):
+        from oe.overlayfs import allOverlaysUnitName
+        args = {
+            'ALL_OVERLAYFS_UNITS': " ".join(dependentUnits),
+            'PN': d.getVar('PN')
+        }
+
+        with open(os.path.join(d.getVar('WORKDIR'), allOverlaysUnitName(d)), 'w') as f:
+            f.write(AllOverlaysTemplate.format(**args))
+
+    mountUnitList = []
     overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
     for mountPoint in overlayMountPoints:
         for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint).split():
             prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint), lower)
+            mountUnitList.append(mountUnitName(lower))
+
+    # set up one unit, which depends on all mount units, so users can set
+    # only one dependency in their units to make sure software starts
+    # when all overlays are mounted
+    prepareGlobalUnit(mountUnitList)
 }
 
 # we need to generate file names early during parsing stage
@@ -95,7 +127,7 @@ python () {
 
     unitList = unitFileList(d)
     for unit in unitList:
-        d.appendVar('SYSTEMD_SERVICE:' + d.getVar('PN'), ' ' + unit);
+        d.appendVar('SYSTEMD_SERVICE:' + d.getVar('PN'), ' ' + unit)
         d.appendVar('FILES:' + d.getVar('PN'), ' ' + strForBash(unit))
 
     d.setVar('OVERLAYFS_UNIT_LIST', ' '.join([strForBash(s) for s in unitList]))
diff --git a/meta/lib/oe/overlayfs.py b/meta/lib/oe/overlayfs.py
index 21ef710509..b5d5e88e80 100644
--- a/meta/lib/oe/overlayfs.py
+++ b/meta/lib/oe/overlayfs.py
@@ -15,6 +15,9 @@ def escapeSystemdUnitName(path):
 def strForBash(s):
     return s.replace('\\', '\\\\')
 
+def allOverlaysUnitName(d):
+    return d.getVar('PN') + '-overlays.service'
+
 def mountUnitName(unit):
     return escapeSystemdUnitName(unit) + '.mount'
 
@@ -39,5 +42,7 @@ def unitFileList(d):
             fileList.append(mountUnitName(path))
             fileList.append(helperUnitName(path))
 
+    fileList.append(allOverlaysUnitName(d))
+
     return fileList
 
-- 
2.28.0

[PATCH 2/5] oeqa/selftest: refactor common functions

[Vyacheslav Yurkov]

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/lib/oeqa/selftest/cases/overlayfs.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 0184d52494..f0c9860b48 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -8,11 +8,14 @@ from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu
 class OverlayFSTests(OESelftestTestCase):
     """Overlayfs class usage tests"""
 
-    def getline(self, res, line):
-        for l in res.output.split('\n'):
+    def getline_qemu(self, out, line):
+        for l in out.split('\n'):
             if line in l:
                 return l
 
+    def getline(self, res, line):
+        return self.getline_qemu(res.output, line)
+
     def add_overlay_conf_to_machine(self):
         machine_inc = """
 OVERLAYFS_MOUNT_POINT[mnt-overlay] = "/mnt/overlay"
@@ -154,18 +157,13 @@ EOT
 
         bitbake('core-image-minimal')
 
-        def getline_qemu(out, line):
-            for l in out.split('\n'):
-                if line in l:
-                    return l
-
         with runqemu('core-image-minimal') as qemu:
             # Check that we have /mnt/overlay fs mounted as tmpfs and
             # /usr/share/my-application as an overlay (see overlayfs-user recipe)
             status, output = qemu.run_serial("/bin/mount -t tmpfs,overlay")
 
-            line = getline_qemu(output, "on /mnt/overlay")
+            line = self.getline_qemu(output, "on /mnt/overlay")
             self.assertTrue(line and line.startswith("tmpfs"), msg=output)
 
-            line = getline_qemu(output, "upperdir=/mnt/overlay/upper/usr/share/my-application")
+            line = self.getline_qemu(output, "upperdir=/mnt/overlay/upper/usr/share/my-application")
             self.assertTrue(line and line.startswith("overlay"), msg=output)
-- 
2.28.0

[PATCH 3/5] overlayfs: meta-selftest recipe fix

[Vyacheslav Yurkov]

Avoid strict assignment in the recipe to allow overrides in .inc file

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb b/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
index 60405067de..913a4d1fdb 100644
--- a/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
+++ b/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
@@ -8,7 +8,7 @@ EXCLUDE_FROM_WORLD = "1"
 inherit ${@bb.utils.contains("DISTRO_FEATURES", "overlayfs", "overlayfs", "", d)}
 include test_recipe.inc
 
-OVERLAYFS_WRITABLE_PATHS[mnt-overlay] = "/usr/share/my-application"
+OVERLAYFS_WRITABLE_PATHS[mnt-overlay] += "/usr/share/my-application"
 
 do_install() {
     install -d ${D}/usr/share/my-application
-- 
2.28.0

[PATCH 4/5] oeqa/selftest: extend overlayfs test

[Vyacheslav Yurkov]

Test that overlayfs.bbclass generates one systemd unit, that
applications can set dependencies on, and that this unit mounts all
required overlays

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/lib/oeqa/selftest/cases/overlayfs.py | 42 +++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py b/meta/lib/oeqa/selftest/cases/overlayfs.py
index f0c9860b48..84242a1605 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -149,15 +149,54 @@ WantedBy=multi-user.target
 EOT
 }
 
+"""
+
+        overlayfs_recipe_append = """
+OVERLAYFS_WRITABLE_PATHS[mnt-overlay] += "/usr/share/another-overlay-mount"
+
+SYSTEMD_SERVICE:${PN} += " \
+    my-application.service \
+"
+
+do_install:append() {
+    install -d ${D}${systemd_system_unitdir}
+    cat <<EOT > ${D}${systemd_system_unitdir}/my-application.service
+[Unit]
+Description=Sample application start-up unit
+After=overlayfs-user-overlays.service
+Requires=overlayfs-user-overlays.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/true
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target
+EOT
+}
 """
 
         self.write_config(config)
         self.add_overlay_conf_to_machine()
         self.write_recipeinc('systemd-machine-units', systemd_machine_unit_append)
+        self.write_recipeinc('overlayfs-user', overlayfs_recipe_append)
 
         bitbake('core-image-minimal')
 
         with runqemu('core-image-minimal') as qemu:
+            # Check that application service started
+            status, output = qemu.run_serial("systemctl status my-application")
+            self.assertTrue("active (exited)" in output, msg=output)
+
+            # Check that overlay mounts are dependencies of our application unit
+            status, output = qemu.run_serial("systemctl list-dependencies my-application")
+            self.assertTrue("overlayfs-user-overlays.service" in output, msg=output)
+
+            status, output = qemu.run_serial("systemctl list-dependencies overlayfs-user-overlays")
+            self.assertTrue("usr-share-another\\x2doverlay\\x2dmount.mount" in output, msg=output)
+            self.assertTrue("usr-share-my\\x2dapplication.mount" in output, msg=output)
+
             # Check that we have /mnt/overlay fs mounted as tmpfs and
             # /usr/share/my-application as an overlay (see overlayfs-user recipe)
             status, output = qemu.run_serial("/bin/mount -t tmpfs,overlay")
@@ -167,3 +206,6 @@ EOT
 
             line = self.getline_qemu(output, "upperdir=/mnt/overlay/upper/usr/share/my-application")
             self.assertTrue(line and line.startswith("overlay"), msg=output)
+
+            line = self.getline_qemu(output, "upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
+            self.assertTrue(line and line.startswith("overlay"), msg=output)
-- 
2.28.0

[PATCH 5/5] overlayfs: add debug information

[Vyacheslav Yurkov]

Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com>
---
 meta/classes/overlayfs.bbclass | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index 9397ab44f9..3c0f4dc882 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -92,9 +92,11 @@ WantedBy=local-fs.target
             'LOWERDIR': lower,
         }
 
+        bb.debug(1, "Generate systemd unit %s" % mountUnitName(lower))
         with open(os.path.join(d.getVar('WORKDIR'), mountUnitName(lower)), 'w') as f:
             f.write(MountUnitTemplate.format(**args))
 
+        bb.debug(1, "Generate helper systemd unit %s" % helperUnitName(lower))
         with open(os.path.join(d.getVar('WORKDIR'), helperUnitName(lower)), 'w') as f:
             f.write(CreateDirsUnitTemplate.format(**args))
 
@@ -105,13 +107,17 @@ WantedBy=local-fs.target
             'PN': d.getVar('PN')
         }
 
+        bb.debug(1, "Generate systemd unit with all overlays %s" % allOverlaysUnitName(d))
         with open(os.path.join(d.getVar('WORKDIR'), allOverlaysUnitName(d)), 'w') as f:
             f.write(AllOverlaysTemplate.format(**args))
 
     mountUnitList = []
     overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
     for mountPoint in overlayMountPoints:
+        bb.debug(1, "Process variable flag %s" % mountPoint)
         for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint).split():
+            bb.debug(1, "Prepare mount unit for %s with data mount point %s" %
+                     (lower, d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint)))
             prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint), lower)
             mountUnitList.append(mountUnitName(lower))
 
-- 
2.28.0