[OE-core][PATCH v2 1/2] wic: Add --no-fstab-update part option

[OE-core][PATCH v2 1/2] wic: Add --no-fstab-update part option

[Daniel Gomez]

When embedding a rootfs image (e.g. 'rootfs-dir') as a partition we
might want to keep the stock fstab for that image. In such a case, use
this option to not update the fstab and use the stock one instead.

This option allows you to specify which partitions get the fstab
updated and which get the stock fstab.

The option matches the argument you can pass to wic itself where the
same action is performed but for all the partitions.

Example:
    part /export --source rootfs --rootfs-dir=hockeycam-image
--fstype=ext4 --label export --align 1024 --no-fstab-update

    part / --source rootfs --fstype=ext4 --label rootfs --align 1024

Signed-off-by: Daniel Gomez <daniel@qtec.com>
---
 scripts/lib/wic/help.py                  | 3 +++
 scripts/lib/wic/ksparser.py              | 1 +
 scripts/lib/wic/partition.py             | 5 +++--
 scripts/lib/wic/plugins/source/rootfs.py | 2 +-
 4 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py
index 991e5094bb..5c3d278d4e 100644
--- a/scripts/lib/wic/help.py
+++ b/scripts/lib/wic/help.py
@@ -991,6 +991,9 @@ DESCRIPTION
                              multiple partitions and we want to keep the right
                              permissions and usernames in all the partitions.
 
+         --no-fstab-update: This option is specific to wic. It does not update the
+                            '/etc/fstab' stock file for the given partition.
+
          --extra-space: This option is specific to wic. It adds extra
                         space after the space filled by the content
                         of the partition. The final size can go
diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py
index 7a4cc83af5..0df9eb0d05 100644
--- a/scripts/lib/wic/ksparser.py
+++ b/scripts/lib/wic/ksparser.py
@@ -185,6 +185,7 @@ class KickStart():
         part.add_argument('--use-uuid', action='store_true')
         part.add_argument('--uuid')
         part.add_argument('--fsuuid')
+        part.add_argument('--no-fstab-update', action='store_true')
 
         bootloader = subparsers.add_parser('bootloader')
         bootloader.add_argument('--append')
diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
index e0b2c5bdf2..ab304f1b2a 100644
--- a/scripts/lib/wic/partition.py
+++ b/scripts/lib/wic/partition.py
@@ -54,6 +54,7 @@ class Partition():
         self.uuid = args.uuid
         self.fsuuid = args.fsuuid
         self.type = args.type
+        self.no_fstab_update = args.no_fstab_update
         self.updated_fstab_path = None
         self.has_fstab = False
         self.update_fstab_in_rootfs = False
@@ -286,7 +287,7 @@ class Partition():
             (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir)
         exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
 
-        if self.updated_fstab_path and self.has_fstab:
+        if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
             debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
             with open(debugfs_script_path, "w") as f:
                 f.write("cd etc\n")
@@ -350,7 +351,7 @@ class Partition():
         mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir)
         exec_native_cmd(mcopy_cmd, native_sysroot)
 
-        if self.updated_fstab_path and self.has_fstab:
+        if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
             mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path)
             exec_native_cmd(mcopy_cmd, native_sysroot)
 
diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py
index 96d940a91d..2e34e715ca 100644
--- a/scripts/lib/wic/plugins/source/rootfs.py
+++ b/scripts/lib/wic/plugins/source/rootfs.py
@@ -218,7 +218,7 @@ class RootfsPlugin(SourcePlugin):
             # Update part.has_fstab here as fstab may have been added or
             # removed by the above modifications.
             part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab"))
-            if part.update_fstab_in_rootfs and part.has_fstab:
+            if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update:
                 fstab_path = os.path.join(new_rootfs, "etc/fstab")
                 # Assume that fstab should always be owned by root with fixed permissions
                 install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path)
-- 
2.30.2

[OE-core][PATCH v2 2/2] oeqa: wic: Add tests for --no-fstab-update

[Daniel Gomez]

Add tests for the --no-fstab-update wic part command.

Signed-off-by: Daniel Gomez <daniel@qtec.com>
---
 meta/lib/oeqa/selftest/cases/wic.py | 56 +++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
index 2efbe514c1..a58360851a 100644
--- a/meta/lib/oeqa/selftest/cases/wic.py
+++ b/meta/lib/oeqa/selftest/cases/wic.py
@@ -11,6 +11,7 @@
 import os
 import sys
 import unittest
+import hashlib
 
 from glob import glob
 from shutil import rmtree, copy
@@ -686,6 +687,61 @@ part /etc --source rootfs --fstype=ext4 --change-directory=etc
                                       % (wks_file, self.resultdir), ignore_status=True).status)
         os.remove(wks_file)
 
+    def test_no_fstab_update(self):
+        """Test --no-fstab-update wks option."""
+
+        oldpath = os.environ['PATH']
+        os.environ['PATH'] = get_bb_var("PATH", "wic-tools")
+
+        # Get stock fstab from base-files recipe
+        bitbake('base-files')
+        bf_fstab = os.path.join(get_bb_var('WORKDIR', 'base-files'),'image/etc/fstab')
+        bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
+
+        try:
+            no_fstab_update_path = os.path.join(self.resultdir, 'test-no-fstab-update')
+            os.makedirs(no_fstab_update_path)
+            wks_file = os.path.join(no_fstab_update_path, 'temp.wks')
+            with open(wks_file, 'w') as wks:
+                wks.writelines(['part / --source rootfs --fstype=ext4 --label rootfs\n',
+                                'part /mnt/p2 --source rootfs --rootfs-dir=core-image-minimal ',
+                                '--fstype=ext4 --label p2 --no-fstab-update\n'])
+            runCmd("wic create %s -e core-image-minimal -o %s" \
+                                       % (wks_file, self.resultdir))
+
+            part_fstab_md5sum = []
+            for i in range(1, 3):
+                part = glob(os.path.join(self.resultdir, 'temp-*.direct.p') + str(i))[0]
+                part_fstab = runCmd("debugfs -R 'cat etc/fstab' %s 2>/dev/null" % (part))
+                part_fstab_md5sum.append(hashlib.md5((part_fstab.output + "\n\n").encode('utf-8')).hexdigest())
+
+            # '/etc/fstab' in partition 2 should contain the same stock fstab file at base-file recipe.
+            self.assertEqual(bf_fstab_md5sum, part_fstab_md5sum[1])
+
+            # '/etc/fstab' in partition 1 should contain an updated fstab file.
+            self.assertNotEqual(bf_fstab_md5sum, part_fstab_md5sum[0])
+
+        finally:
+            os.environ['PATH'] = oldpath
+
+    def test_no_fstab_update_errors(self):
+        """Test --no-fstab-update wks option error handling."""
+        wks_file = 'temp.wks'
+
+        # Absolute argument.
+        with open(wks_file, 'w') as wks:
+            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update /etc")
+        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
+                                      % (wks_file, self.resultdir), ignore_status=True).status)
+        os.remove(wks_file)
+
+        # Argument pointing to parent directory.
+        with open(wks_file, 'w') as wks:
+            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update ././..")
+        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
+                                      % (wks_file, self.resultdir), ignore_status=True).status)
+        os.remove(wks_file)
+
 class Wic2(WicTestCase):
 
     def test_bmap_short(self):
-- 
2.30.2

Re: [OE-core][PATCH v2 2/2] oeqa: wic: Add tests for --no-fstab-update

[Alexandre Belloni]

Hello,

On 10/08/2021 22:11:01+0200, Daniel Gomez wrote:
> Add tests for the --no-fstab-update wic part command.
> 
> Signed-off-by: Daniel Gomez <daniel@qtec.com>
> ---
>  meta/lib/oeqa/selftest/cases/wic.py | 56 +++++++++++++++++++++++++++++
>  1 file changed, 56 insertions(+)
> 
> diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
> index 2efbe514c1..a58360851a 100644
> --- a/meta/lib/oeqa/selftest/cases/wic.py
> +++ b/meta/lib/oeqa/selftest/cases/wic.py
> @@ -11,6 +11,7 @@
>  import os
>  import sys
>  import unittest
> +import hashlib
>  
>  from glob import glob
>  from shutil import rmtree, copy
> @@ -686,6 +687,61 @@ part /etc --source rootfs --fstype=ext4 --change-directory=etc
>                                        % (wks_file, self.resultdir), ignore_status=True).status)
>          os.remove(wks_file)
>  
> +    def test_no_fstab_update(self):
> +        """Test --no-fstab-update wks option."""
> +
> +        oldpath = os.environ['PATH']
> +        os.environ['PATH'] = get_bb_var("PATH", "wic-tools")
> +
> +        # Get stock fstab from base-files recipe
> +        bitbake('base-files')
> +        bf_fstab = os.path.join(get_bb_var('WORKDIR', 'base-files'),'image/etc/fstab')
> +        bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> +

This failed on the autobuilders:

  File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/wic.py", line 699, in test_no_fstab_update
    bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
  File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 207, in runCmd
    raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
AssertionError: Command 'md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab 2>/dev/null' returned non-zero exit status 1:

Full log here:
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2374/steps/14/logs/stdio

> +        try:
> +            no_fstab_update_path = os.path.join(self.resultdir, 'test-no-fstab-update')
> +            os.makedirs(no_fstab_update_path)
> +            wks_file = os.path.join(no_fstab_update_path, 'temp.wks')
> +            with open(wks_file, 'w') as wks:
> +                wks.writelines(['part / --source rootfs --fstype=ext4 --label rootfs\n',
> +                                'part /mnt/p2 --source rootfs --rootfs-dir=core-image-minimal ',
> +                                '--fstype=ext4 --label p2 --no-fstab-update\n'])
> +            runCmd("wic create %s -e core-image-minimal -o %s" \
> +                                       % (wks_file, self.resultdir))
> +
> +            part_fstab_md5sum = []
> +            for i in range(1, 3):
> +                part = glob(os.path.join(self.resultdir, 'temp-*.direct.p') + str(i))[0]
> +                part_fstab = runCmd("debugfs -R 'cat etc/fstab' %s 2>/dev/null" % (part))
> +                part_fstab_md5sum.append(hashlib.md5((part_fstab.output + "\n\n").encode('utf-8')).hexdigest())
> +
> +            # '/etc/fstab' in partition 2 should contain the same stock fstab file at base-file recipe.
> +            self.assertEqual(bf_fstab_md5sum, part_fstab_md5sum[1])
> +
> +            # '/etc/fstab' in partition 1 should contain an updated fstab file.
> +            self.assertNotEqual(bf_fstab_md5sum, part_fstab_md5sum[0])
> +
> +        finally:
> +            os.environ['PATH'] = oldpath
> +
> +    def test_no_fstab_update_errors(self):
> +        """Test --no-fstab-update wks option error handling."""
> +        wks_file = 'temp.wks'
> +
> +        # Absolute argument.
> +        with open(wks_file, 'w') as wks:
> +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update /etc")
> +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> +        os.remove(wks_file)
> +
> +        # Argument pointing to parent directory.
> +        with open(wks_file, 'w') as wks:
> +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update ././..")
> +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> +        os.remove(wks_file)
> +
>  class Wic2(WicTestCase):
>  
>      def test_bmap_short(self):
> -- 
> 2.30.2
> 



-- 
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

Re: [OE-core][PATCH v2 2/2] oeqa: wic: Add tests for --no-fstab-update

[Daniel Gomez]

Hi Alexandre,

On Thu, 12 Aug 2021 at 18:34, Alexandre Belloni
<alexandre.belloni@bootlin.com> wrote:
>
> Hello,
>
> On 10/08/2021 22:11:01+0200, Daniel Gomez wrote:
> > Add tests for the --no-fstab-update wic part command.
> >
> > Signed-off-by: Daniel Gomez <daniel@qtec.com>
> > ---
> >  meta/lib/oeqa/selftest/cases/wic.py | 56 +++++++++++++++++++++++++++++
> >  1 file changed, 56 insertions(+)
> >
> > diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
> > index 2efbe514c1..a58360851a 100644
> > --- a/meta/lib/oeqa/selftest/cases/wic.py
> > +++ b/meta/lib/oeqa/selftest/cases/wic.py
> > @@ -11,6 +11,7 @@
> >  import os
> >  import sys
> >  import unittest
> > +import hashlib
> >
> >  from glob import glob
> >  from shutil import rmtree, copy
> > @@ -686,6 +687,61 @@ part /etc --source rootfs --fstype=ext4 --change-directory=etc
> >                                        % (wks_file, self.resultdir), ignore_status=True).status)
> >          os.remove(wks_file)
> >
> > +    def test_no_fstab_update(self):
> > +        """Test --no-fstab-update wks option."""
> > +
> > +        oldpath = os.environ['PATH']
> > +        os.environ['PATH'] = get_bb_var("PATH", "wic-tools")
> > +
> > +        # Get stock fstab from base-files recipe
> > +        bitbake('base-files')
> > +        bf_fstab = os.path.join(get_bb_var('WORKDIR', 'base-files'),'image/etc/fstab')
> > +        bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> > +
>
> This failed on the autobuilders:
>
>   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/wic.py", line 699, in test_no_fstab_update
>     bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
>   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 207, in runCmd
>     raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
> AssertionError: Command 'md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab 2>/dev/null' returned non-zero exit status 1:
>
> Full log here:
> https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2374/steps/14/logs/stdio

Thanks for letting me know. I'll take a look.

>
> > +        try:
> > +            no_fstab_update_path = os.path.join(self.resultdir, 'test-no-fstab-update')
> > +            os.makedirs(no_fstab_update_path)
> > +            wks_file = os.path.join(no_fstab_update_path, 'temp.wks')
> > +            with open(wks_file, 'w') as wks:
> > +                wks.writelines(['part / --source rootfs --fstype=ext4 --label rootfs\n',
> > +                                'part /mnt/p2 --source rootfs --rootfs-dir=core-image-minimal ',
> > +                                '--fstype=ext4 --label p2 --no-fstab-update\n'])
> > +            runCmd("wic create %s -e core-image-minimal -o %s" \
> > +                                       % (wks_file, self.resultdir))
> > +
> > +            part_fstab_md5sum = []
> > +            for i in range(1, 3):
> > +                part = glob(os.path.join(self.resultdir, 'temp-*.direct.p') + str(i))[0]
> > +                part_fstab = runCmd("debugfs -R 'cat etc/fstab' %s 2>/dev/null" % (part))
> > +                part_fstab_md5sum.append(hashlib.md5((part_fstab.output + "\n\n").encode('utf-8')).hexdigest())
> > +
> > +            # '/etc/fstab' in partition 2 should contain the same stock fstab file at base-file recipe.
> > +            self.assertEqual(bf_fstab_md5sum, part_fstab_md5sum[1])
> > +
> > +            # '/etc/fstab' in partition 1 should contain an updated fstab file.
> > +            self.assertNotEqual(bf_fstab_md5sum, part_fstab_md5sum[0])
> > +
> > +        finally:
> > +            os.environ['PATH'] = oldpath
> > +
> > +    def test_no_fstab_update_errors(self):
> > +        """Test --no-fstab-update wks option error handling."""
> > +        wks_file = 'temp.wks'
> > +
> > +        # Absolute argument.
> > +        with open(wks_file, 'w') as wks:
> > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update /etc")
> > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > +        os.remove(wks_file)
> > +
> > +        # Argument pointing to parent directory.
> > +        with open(wks_file, 'w') as wks:
> > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update ././..")
> > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > +        os.remove(wks_file)
> > +
> >  class Wic2(WicTestCase):
> >
> >      def test_bmap_short(self):
> > --
> > 2.30.2
> >
>
>
>
> --
> Alexandre Belloni, co-owner and COO, Bootlin
> Embedded Linux and Kernel engineering
> https://bootlin.com

Daniel

Re: [OE-core][PATCH v2 2/2] oeqa: wic: Add tests for --no-fstab-update

[Daniel Gomez]

Hi Alexandre,


On Fri, 13 Aug 2021 at 07:41, Daniel Gomez via lists.openembedded.org
<daniel=qtec.com@lists.openembedded.org> wrote:
>
> Hi Alexandre,
>
> On Thu, 12 Aug 2021 at 18:34, Alexandre Belloni
> <alexandre.belloni@bootlin.com> wrote:
> >
> > Hello,
> >
> > On 10/08/2021 22:11:01+0200, Daniel Gomez wrote:
> > > Add tests for the --no-fstab-update wic part command.
> > >
> > > Signed-off-by: Daniel Gomez <daniel@qtec.com>
> > > ---
> > >  meta/lib/oeqa/selftest/cases/wic.py | 56 +++++++++++++++++++++++++++++
> > >  1 file changed, 56 insertions(+)
> > >
> > > diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
> > > index 2efbe514c1..a58360851a 100644
> > > --- a/meta/lib/oeqa/selftest/cases/wic.py
> > > +++ b/meta/lib/oeqa/selftest/cases/wic.py
> > > @@ -11,6 +11,7 @@
> > >  import os
> > >  import sys
> > >  import unittest
> > > +import hashlib
> > >
> > >  from glob import glob
> > >  from shutil import rmtree, copy
> > > @@ -686,6 +687,61 @@ part /etc --source rootfs --fstype=ext4 --change-directory=etc
> > >                                        % (wks_file, self.resultdir), ignore_status=True).status)
> > >          os.remove(wks_file)
> > >
> > > +    def test_no_fstab_update(self):
> > > +        """Test --no-fstab-update wks option."""
> > > +
> > > +        oldpath = os.environ['PATH']
> > > +        os.environ['PATH'] = get_bb_var("PATH", "wic-tools")
> > > +
> > > +        # Get stock fstab from base-files recipe
> > > +        bitbake('base-files')
> > > +        bf_fstab = os.path.join(get_bb_var('WORKDIR', 'base-files'),'image/etc/fstab')
> > > +        bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> > > +
> >
> > This failed on the autobuilders:
> >
> >   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/wic.py", line 699, in test_no_fstab_update
> >     bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> >   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 207, in runCmd
> >     raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
> > AssertionError: Command 'md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab 2>/dev/null' returned non-zero exit status 1:
> >
> > Full log here:
> > https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2374/steps/14/logs/stdio
>
> Thanks for letting me know. I'll take a look.

I can't replicate the problem in my setup ('master' branch for bitbake
and openembedded core; 'qemux86-64' machine) . Could it be possible to
get the environment variables for the build?

The problem comes when executing the first md5sum command:

1. bitbake 'base-files' recipe.
2. Get the 'WORKDIR' for 'base-files' recipe.
3. Do 'md5sum' for the 'WORKDIR' + 'image/etc/fstab' file.

md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab

If the file does not exist, md5sum will return the same '1' status
(assuming that's the error). Could this be bause the '${WORKDIR} +
image' for the 'base-files' recipe is located under another directory?
Could we get access to that directory to know if the file was present
there?

If that's the case, I could replace the command to be executed with
one of the following environment variables:
D="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/image"
PKGD="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/package"
PKGDEST="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/packages-split"

Otherwise, any ideas or suggestions?

Thanks

>
> >
> > > +        try:
> > > +            no_fstab_update_path = os.path.join(self.resultdir, 'test-no-fstab-update')
> > > +            os.makedirs(no_fstab_update_path)
> > > +            wks_file = os.path.join(no_fstab_update_path, 'temp.wks')
> > > +            with open(wks_file, 'w') as wks:
> > > +                wks.writelines(['part / --source rootfs --fstype=ext4 --label rootfs\n',
> > > +                                'part /mnt/p2 --source rootfs --rootfs-dir=core-image-minimal ',
> > > +                                '--fstype=ext4 --label p2 --no-fstab-update\n'])
> > > +            runCmd("wic create %s -e core-image-minimal -o %s" \
> > > +                                       % (wks_file, self.resultdir))
> > > +
> > > +            part_fstab_md5sum = []
> > > +            for i in range(1, 3):
> > > +                part = glob(os.path.join(self.resultdir, 'temp-*.direct.p') + str(i))[0]
> > > +                part_fstab = runCmd("debugfs -R 'cat etc/fstab' %s 2>/dev/null" % (part))
> > > +                part_fstab_md5sum.append(hashlib.md5((part_fstab.output + "\n\n").encode('utf-8')).hexdigest())
> > > +
> > > +            # '/etc/fstab' in partition 2 should contain the same stock fstab file at base-file recipe.
> > > +            self.assertEqual(bf_fstab_md5sum, part_fstab_md5sum[1])
> > > +
> > > +            # '/etc/fstab' in partition 1 should contain an updated fstab file.
> > > +            self.assertNotEqual(bf_fstab_md5sum, part_fstab_md5sum[0])
> > > +
> > > +        finally:
> > > +            os.environ['PATH'] = oldpath
> > > +
> > > +    def test_no_fstab_update_errors(self):
> > > +        """Test --no-fstab-update wks option error handling."""
> > > +        wks_file = 'temp.wks'
> > > +
> > > +        # Absolute argument.
> > > +        with open(wks_file, 'w') as wks:
> > > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update /etc")
> > > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > > +        os.remove(wks_file)
> > > +
> > > +        # Argument pointing to parent directory.
> > > +        with open(wks_file, 'w') as wks:
> > > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update ././..")
> > > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > > +        os.remove(wks_file)
> > > +
> > >  class Wic2(WicTestCase):
> > >
> > >      def test_bmap_short(self):
> > > --
> > > 2.30.2
> > >
> >
> >
> >
> > --
> > Alexandre Belloni, co-owner and COO, Bootlin
> > Embedded Linux and Kernel engineering
> > https://bootlin.com
>
> Daniel
>
> 
>

Re: [OE-core][PATCH v2 2/2] oeqa: wic: Add tests for --no-fstab-update

[Alexandre Belloni]

Hi,

On 16/08/2021 10:05:08+0200, Daniel Gomez wrote:
> Hi Alexandre,
> 
> 
> On Fri, 13 Aug 2021 at 07:41, Daniel Gomez via lists.openembedded.org
> <daniel=qtec.com@lists.openembedded.org> wrote:
> >
> > Hi Alexandre,
> >
> > On Thu, 12 Aug 2021 at 18:34, Alexandre Belloni
> > <alexandre.belloni@bootlin.com> wrote:
> > >
> > > Hello,
> > >
> > > On 10/08/2021 22:11:01+0200, Daniel Gomez wrote:
> > > > Add tests for the --no-fstab-update wic part command.
> > > >
> > > > Signed-off-by: Daniel Gomez <daniel@qtec.com>
> > > > ---
> > > >  meta/lib/oeqa/selftest/cases/wic.py | 56 +++++++++++++++++++++++++++++
> > > >  1 file changed, 56 insertions(+)
> > > >
> > > > diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
> > > > index 2efbe514c1..a58360851a 100644
> > > > --- a/meta/lib/oeqa/selftest/cases/wic.py
> > > > +++ b/meta/lib/oeqa/selftest/cases/wic.py
> > > > @@ -11,6 +11,7 @@
> > > >  import os
> > > >  import sys
> > > >  import unittest
> > > > +import hashlib
> > > >
> > > >  from glob import glob
> > > >  from shutil import rmtree, copy
> > > > @@ -686,6 +687,61 @@ part /etc --source rootfs --fstype=ext4 --change-directory=etc
> > > >                                        % (wks_file, self.resultdir), ignore_status=True).status)
> > > >          os.remove(wks_file)
> > > >
> > > > +    def test_no_fstab_update(self):
> > > > +        """Test --no-fstab-update wks option."""
> > > > +
> > > > +        oldpath = os.environ['PATH']
> > > > +        os.environ['PATH'] = get_bb_var("PATH", "wic-tools")
> > > > +
> > > > +        # Get stock fstab from base-files recipe
> > > > +        bitbake('base-files')
> > > > +        bf_fstab = os.path.join(get_bb_var('WORKDIR', 'base-files'),'image/etc/fstab')
> > > > +        bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> > > > +
> > >
> > > This failed on the autobuilders:
> > >
> > >   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/wic.py", line 699, in test_no_fstab_update
> > >     bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> > >   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 207, in runCmd
> > >     raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
> > > AssertionError: Command 'md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab 2>/dev/null' returned non-zero exit status 1:
> > >
> > > Full log here:
> > > https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2374/steps/14/logs/stdio
> >
> > Thanks for letting me know. I'll take a look.
> 
> I can't replicate the problem in my setup ('master' branch for bitbake
> and openembedded core; 'qemux86-64' machine) . Could it be possible to
> get the environment variables for the build?
> 
> The problem comes when executing the first md5sum command:
> 
> 1. bitbake 'base-files' recipe.
> 2. Get the 'WORKDIR' for 'base-files' recipe.
> 3. Do 'md5sum' for the 'WORKDIR' + 'image/etc/fstab' file.
> 
> md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab
> 
> If the file does not exist, md5sum will return the same '1' status
> (assuming that's the error). Could this be bause the '${WORKDIR} +
> image' for the 'base-files' recipe is located under another directory?
> Could we get access to that directory to know if the file was present
> there?
> 

I did run the build again and:

debian10-ty-1:~$ md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-32220/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab
md5sum: /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-32220/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab: No such file or directory

debian10-ty-1:~$ ls /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-32220/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/
deploy-rpms               license-destdir      source-date-epoch                        sstate-install-package_qa         sstate-install-packagedata   sstate-install-populate_sysroot  temp
deploy-source-date-epoch  pkgdata-pdata-input  sstate-install-deploy_source_date_epoch  sstate-install-package_write_rpm  sstate-install-populate_lic  sysroot-destdir

I guess the issue is that we reuse the sstate cache and so the file is
not present at all.

> If that's the case, I could replace the command to be executed with
> one of the following environment variables:
> D="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/image"
> PKGD="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/package"
> PKGDEST="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/packages-split"
> 
> Otherwise, any ideas or suggestions?
> 
> Thanks
> 
> >
> > >
> > > > +        try:
> > > > +            no_fstab_update_path = os.path.join(self.resultdir, 'test-no-fstab-update')
> > > > +            os.makedirs(no_fstab_update_path)
> > > > +            wks_file = os.path.join(no_fstab_update_path, 'temp.wks')
> > > > +            with open(wks_file, 'w') as wks:
> > > > +                wks.writelines(['part / --source rootfs --fstype=ext4 --label rootfs\n',
> > > > +                                'part /mnt/p2 --source rootfs --rootfs-dir=core-image-minimal ',
> > > > +                                '--fstype=ext4 --label p2 --no-fstab-update\n'])
> > > > +            runCmd("wic create %s -e core-image-minimal -o %s" \
> > > > +                                       % (wks_file, self.resultdir))
> > > > +
> > > > +            part_fstab_md5sum = []
> > > > +            for i in range(1, 3):
> > > > +                part = glob(os.path.join(self.resultdir, 'temp-*.direct.p') + str(i))[0]
> > > > +                part_fstab = runCmd("debugfs -R 'cat etc/fstab' %s 2>/dev/null" % (part))
> > > > +                part_fstab_md5sum.append(hashlib.md5((part_fstab.output + "\n\n").encode('utf-8')).hexdigest())
> > > > +
> > > > +            # '/etc/fstab' in partition 2 should contain the same stock fstab file at base-file recipe.
> > > > +            self.assertEqual(bf_fstab_md5sum, part_fstab_md5sum[1])
> > > > +
> > > > +            # '/etc/fstab' in partition 1 should contain an updated fstab file.
> > > > +            self.assertNotEqual(bf_fstab_md5sum, part_fstab_md5sum[0])
> > > > +
> > > > +        finally:
> > > > +            os.environ['PATH'] = oldpath
> > > > +
> > > > +    def test_no_fstab_update_errors(self):
> > > > +        """Test --no-fstab-update wks option error handling."""
> > > > +        wks_file = 'temp.wks'
> > > > +
> > > > +        # Absolute argument.
> > > > +        with open(wks_file, 'w') as wks:
> > > > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update /etc")
> > > > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > > > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > > > +        os.remove(wks_file)
> > > > +
> > > > +        # Argument pointing to parent directory.
> > > > +        with open(wks_file, 'w') as wks:
> > > > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update ././..")
> > > > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > > > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > > > +        os.remove(wks_file)
> > > > +
> > > >  class Wic2(WicTestCase):
> > > >
> > > >      def test_bmap_short(self):
> > > > --
> > > > 2.30.2
> > > >
> > >
> > >
> > >
> > > --
> > > Alexandre Belloni, co-owner and COO, Bootlin
> > > Embedded Linux and Kernel engineering
> > > https://bootlin.com
> >
> > Daniel
> >
> > 
> >

-- 
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

Re: [OE-core][PATCH v2 2/2] oeqa: wic: Add tests for --no-fstab-update

[Daniel Gomez]

Hi,

On Tue, 17 Aug 2021 at 15:34, Alexandre Belloni
<alexandre.belloni@bootlin.com> wrote:
>
> Hi,
>
> On 16/08/2021 10:05:08+0200, Daniel Gomez wrote:
> > Hi Alexandre,
> >
> >
> > On Fri, 13 Aug 2021 at 07:41, Daniel Gomez via lists.openembedded.org
> > <daniel=qtec.com@lists.openembedded.org> wrote:
> > >
> > > Hi Alexandre,
> > >
> > > On Thu, 12 Aug 2021 at 18:34, Alexandre Belloni
> > > <alexandre.belloni@bootlin.com> wrote:
> > > >
> > > > Hello,
> > > >
> > > > On 10/08/2021 22:11:01+0200, Daniel Gomez wrote:
> > > > > Add tests for the --no-fstab-update wic part command.
> > > > >
> > > > > Signed-off-by: Daniel Gomez <daniel@qtec.com>
> > > > > ---
> > > > >  meta/lib/oeqa/selftest/cases/wic.py | 56 +++++++++++++++++++++++++++++
> > > > >  1 file changed, 56 insertions(+)
> > > > >
> > > > > diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
> > > > > index 2efbe514c1..a58360851a 100644
> > > > > --- a/meta/lib/oeqa/selftest/cases/wic.py
> > > > > +++ b/meta/lib/oeqa/selftest/cases/wic.py
> > > > > @@ -11,6 +11,7 @@
> > > > >  import os
> > > > >  import sys
> > > > >  import unittest
> > > > > +import hashlib
> > > > >
> > > > >  from glob import glob
> > > > >  from shutil import rmtree, copy
> > > > > @@ -686,6 +687,61 @@ part /etc --source rootfs --fstype=ext4 --change-directory=etc
> > > > >                                        % (wks_file, self.resultdir), ignore_status=True).status)
> > > > >          os.remove(wks_file)
> > > > >
> > > > > +    def test_no_fstab_update(self):
> > > > > +        """Test --no-fstab-update wks option."""
> > > > > +
> > > > > +        oldpath = os.environ['PATH']
> > > > > +        os.environ['PATH'] = get_bb_var("PATH", "wic-tools")
> > > > > +
> > > > > +        # Get stock fstab from base-files recipe
> > > > > +        bitbake('base-files')
> > > > > +        bf_fstab = os.path.join(get_bb_var('WORKDIR', 'base-files'),'image/etc/fstab')
> > > > > +        bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> > > > > +
> > > >
> > > > This failed on the autobuilders:
> > > >
> > > >   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/wic.py", line 699, in test_no_fstab_update
> > > >     bf_fstab_md5sum = runCmd('md5sum %s 2>/dev/null' % bf_fstab).output.split(" ")[0]
> > > >   File "/home/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 207, in runCmd
> > > >     raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
> > > > AssertionError: Command 'md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab 2>/dev/null' returned non-zero exit status 1:
> > > >
> > > > Full log here:
> > > > https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2374/steps/14/logs/stdio
> > >
> > > Thanks for letting me know. I'll take a look.
> >
> > I can't replicate the problem in my setup ('master' branch for bitbake
> > and openembedded core; 'qemux86-64' machine) . Could it be possible to
> > get the environment variables for the build?
> >
> > The problem comes when executing the first md5sum command:
> >
> > 1. bitbake 'base-files' recipe.
> > 2. Get the 'WORKDIR' for 'base-files' recipe.
> > 3. Do 'md5sum' for the 'WORKDIR' + 'image/etc/fstab' file.
> >
> > md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-15202/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab
> >
> > If the file does not exist, md5sum will return the same '1' status
> > (assuming that's the error). Could this be bause the '${WORKDIR} +
> > image' for the 'base-files' recipe is located under another directory?
> > Could we get access to that directory to know if the file was present
> > there?
> >
>
> I did run the build again and:
>
> debian10-ty-1:~$ md5sum /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-32220/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab
> md5sum: /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-32220/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/image/etc/fstab: No such file or directory
>
> debian10-ty-1:~$ ls /home/pokybuild/yocto-worker/oe-selftest-debian/build/build-st-32220/tmp/work/qemux86_64-poky-linux/base-files/3.0.14-r89/
> deploy-rpms               license-destdir      source-date-epoch                        sstate-install-package_qa         sstate-install-packagedata   sstate-install-populate_sysroot  temp
> deploy-source-date-epoch  pkgdata-pdata-input  sstate-install-deploy_source_date_epoch  sstate-install-package_write_rpm  sstate-install-populate_lic  sysroot-destdir
>
> I guess the issue is that we reuse the sstate cache and so the file is
> not present at all.

Thanks for the hint. I've just sent a v3 patch where I execute
'bitbake base-files -c do_install' instead of just 'bitbake
base-files' so, I make sure the file is being installed even though
you use the sstate cache.

>
> > If that's the case, I could replace the command to be executed with
> > one of the following environment variables:
> > D="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/image"
> > PKGD="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/package"
> > PKGDEST="/workdir/openembedded-core/build/tmp-glibc/work/qemux86_64-oe-linux/base-files/3.0.14-r89/packages-split"
> >
> > Otherwise, any ideas or suggestions?
> >
> > Thanks
> >
> > >
> > > >
> > > > > +        try:
> > > > > +            no_fstab_update_path = os.path.join(self.resultdir, 'test-no-fstab-update')
> > > > > +            os.makedirs(no_fstab_update_path)
> > > > > +            wks_file = os.path.join(no_fstab_update_path, 'temp.wks')
> > > > > +            with open(wks_file, 'w') as wks:
> > > > > +                wks.writelines(['part / --source rootfs --fstype=ext4 --label rootfs\n',
> > > > > +                                'part /mnt/p2 --source rootfs --rootfs-dir=core-image-minimal ',
> > > > > +                                '--fstype=ext4 --label p2 --no-fstab-update\n'])
> > > > > +            runCmd("wic create %s -e core-image-minimal -o %s" \
> > > > > +                                       % (wks_file, self.resultdir))
> > > > > +
> > > > > +            part_fstab_md5sum = []
> > > > > +            for i in range(1, 3):
> > > > > +                part = glob(os.path.join(self.resultdir, 'temp-*.direct.p') + str(i))[0]
> > > > > +                part_fstab = runCmd("debugfs -R 'cat etc/fstab' %s 2>/dev/null" % (part))
> > > > > +                part_fstab_md5sum.append(hashlib.md5((part_fstab.output + "\n\n").encode('utf-8')).hexdigest())
> > > > > +
> > > > > +            # '/etc/fstab' in partition 2 should contain the same stock fstab file at base-file recipe.
> > > > > +            self.assertEqual(bf_fstab_md5sum, part_fstab_md5sum[1])
> > > > > +
> > > > > +            # '/etc/fstab' in partition 1 should contain an updated fstab file.
> > > > > +            self.assertNotEqual(bf_fstab_md5sum, part_fstab_md5sum[0])
> > > > > +
> > > > > +        finally:
> > > > > +            os.environ['PATH'] = oldpath
> > > > > +
> > > > > +    def test_no_fstab_update_errors(self):
> > > > > +        """Test --no-fstab-update wks option error handling."""
> > > > > +        wks_file = 'temp.wks'
> > > > > +
> > > > > +        # Absolute argument.
> > > > > +        with open(wks_file, 'w') as wks:
> > > > > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update /etc")
> > > > > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > > > > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > > > > +        os.remove(wks_file)
> > > > > +
> > > > > +        # Argument pointing to parent directory.
> > > > > +        with open(wks_file, 'w') as wks:
> > > > > +            wks.write("part / --source rootfs --fstype=ext4 --no-fstab-update ././..")
> > > > > +        self.assertNotEqual(0, runCmd("wic create %s -e core-image-minimal -o %s" \
> > > > > +                                      % (wks_file, self.resultdir), ignore_status=True).status)
> > > > > +        os.remove(wks_file)
> > > > > +
> > > > >  class Wic2(WicTestCase):
> > > > >
> > > > >      def test_bmap_short(self):
> > > > > --
> > > > > 2.30.2
> > > > >
> > > >
> > > >
> > > >
> > > > --
> > > > Alexandre Belloni, co-owner and COO, Bootlin
> > > > Embedded Linux and Kernel engineering
> > > > https://bootlin.com
> > >
> > > Daniel
> > >
> > > 
> > >
>
> --
> Alexandre Belloni, co-owner and COO, Bootlin
> Embedded Linux and Kernel engineering
> https://bootlin.com