[PATCH 0/7] implement wic ls/cp/rm for ext partitions

[PATCH 0/7] implement wic ls/cp/rm for ext partitions

[Ed Bartosh]

Hi,

This patchset implements listing directory contents, copying and removing
files on ext[2,3,4] partitions of partitioned images.

The following changes since commit ee5bce2d11e783c0921df47b629025a6b67c44bf:

  kernel-dev: Removed bad reference for creating patches. (2017-09-02 00:52:49 +0100)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib ed/wip
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=ed/wip

Ed Bartosh (7):
  wic: implement ext fs support for 'wic ls'
  wic: implement ext fs support for 'wic cp'
  wic: implement ext fs support for 'wic rm'
  wic: update help content
  wic: selftest: add test_wic_ls_ext test case
  wic: selftest: add test_wic_cp_ext test case
  wic: selftest: add test_wic_rm_ext test case

 meta/lib/oeqa/selftest/cases/wic.py | 69 +++++++++++++++++++++++++++++++++++++
 scripts/lib/wic/engine.py           | 55 ++++++++++++++++++-----------
 scripts/lib/wic/help.py             | 64 +++++++++++++++++-----------------
 3 files changed, 136 insertions(+), 52 deletions(-)

--
Regards,
Ed

[PATCH 1/7] wic: implement ext fs support for 'wic ls'

[Ed Bartosh]

Implemented listing directory contents for ext file
system using debugfs tool.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 scripts/lib/wic/engine.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index eafc6c783ec..c6a63f2080a 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -234,7 +234,7 @@ def wic_list(args, scripts_path):
 
 
 class Disk:
-    def __init__(self, imagepath, native_sysroot, fstypes=('fat',)):
+    def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')):
         self.imagepath = imagepath
         self.native_sysroot = native_sysroot
         self.fstypes = fstypes
@@ -280,7 +280,7 @@ class Disk:
     def __getattr__(self, name):
         """Get path to the executable in a lazy way."""
         if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck",
-                    "resize2fs", "mkswap", "mkdosfs"):
+                    "resize2fs", "mkswap", "mkdosfs", "debugfs"):
             aname = "_%s" % name
             if aname not in self.__dict__:
                 setattr(self, aname, find_executable(name, self.paths))
@@ -314,9 +314,14 @@ class Disk:
                     seek=self.partitions[pnum].start)
 
     def dir(self, pnum, path):
-        return exec_cmd("{} -i {} ::{}".format(self.mdir,
-                                               self._get_part_image(pnum),
-                                               path))
+        if self.partitions[pnum].fstype.startswith('ext'):
+            return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs,
+                                                         self._get_part_image(pnum),
+                                                         path), as_shell=True)
+        else: # fat
+            return exec_cmd("{} -i {} ::{}".format(self.mdir,
+                                                   self._get_part_image(pnum),
+                                                   path))
 
     def copy(self, src, pnum, path):
         """Copy partition image into wic image."""
-- 
2.13.5

[PATCH 2/7] wic: implement ext fs support for 'wic cp'

[Ed Bartosh]

Implemented copying files to the ext partition
using debugfs tool.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 scripts/lib/wic/engine.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index c6a63f2080a..9ebd93ae278 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -325,10 +325,15 @@ class Disk:
 
     def copy(self, src, pnum, path):
         """Copy partition image into wic image."""
-        cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
-                                              self._get_part_image(pnum),
-                                              src, path)
-        exec_cmd(cmd)
+        if self.partitions[pnum].fstype.startswith('ext'):
+            cmd = "echo -e 'cd {}\nwrite {} {}' | {} -w {}".\
+                      format(path, src, os.path.basename(src),
+                             self.debugfs, self._get_part_image(pnum))
+        else: # fat
+            cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
+                                                  self._get_part_image(pnum),
+                                                  src, path)
+        exec_cmd(cmd, as_shell=True)
         self._put_part_image(pnum)
 
     def remove(self, pnum, path):
-- 
2.13.5

[PATCH 3/7] wic: implement ext fs support for 'wic rm'

[Ed Bartosh]

Implemented removing files or directories from the ext
partition using debugfs tool.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 scripts/lib/wic/engine.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index 9ebd93ae278..edcfab39ef5 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -339,18 +339,23 @@ class Disk:
     def remove(self, pnum, path):
         """Remove files/dirs from the partition."""
         partimg = self._get_part_image(pnum)
-        cmd = "{} -i {} ::{}".format(self.mdel, partimg, path)
-        try:
-            exec_cmd(cmd)
-        except WicError as err:
-            if "not found" in str(err) or "non empty" in str(err):
-                # mdel outputs 'File ... not found' or 'directory .. non empty"
-                # try to use mdeltree as path could be a directory
-                cmd = "{} -i {} ::{}".format(self.mdeltree,
-                                             partimg, path)
+        if self.partitions[pnum].fstype.startswith('ext'):
+            exec_cmd("{} {} -wR 'rm {}'".format(self.debugfs,
+                                                self._get_part_image(pnum),
+                                                path), as_shell=True)
+        else: # fat
+            cmd = "{} -i {} ::{}".format(self.mdel, partimg, path)
+            try:
                 exec_cmd(cmd)
-            else:
-                raise err
+            except WicError as err:
+                if "not found" in str(err) or "non empty" in str(err):
+                    # mdel outputs 'File ... not found' or 'directory .. non empty"
+                    # try to use mdeltree as path could be a directory
+                    cmd = "{} -i {} ::{}".format(self.mdeltree,
+                                                 partimg, path)
+                    exec_cmd(cmd)
+                else:
+                    raise err
         self._put_part_image(pnum)
 
     def write(self, target, expand):
-- 
2.13.5

[PATCH 4/7] wic: update help content

[Ed Bartosh]

Added ext* partitions to the description of 'wic ls',
'wic cp' and 'wic rm' commands.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 scripts/lib/wic/help.py | 64 ++++++++++++++++++++++++-------------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py
index ccd33823244..83bd86e7c51 100644
--- a/scripts/lib/wic/help.py
+++ b/scripts/lib/wic/help.py
@@ -288,10 +288,10 @@ wic_ls_usage = """
 
  List content of a partitioned image
 
- usage: wic ls <image>[:<vfat partition>[<path>]] [--native-sysroot <path>]
+ usage: wic ls <image>[:<partition>[<path>]] [--native-sysroot <path>]
 
  This command  outputs either list of image partitions or directory contents
- of vfat partitions.
+ of vfat and ext* partitions.
 
  See 'wic help ls' for more detailed instructions.
 
@@ -300,17 +300,17 @@ wic_ls_usage = """
 wic_ls_help = """
 
 NAME
-    wic ls - List contents of partitioned image or vfat partitions
+    wic ls - List contents of partitioned image or partition
 
 SYNOPSIS
     wic ls <image>
-    wic ls <image>:<vfat partition>
-    wic ls <image>:<vfat partition><path>
-    wic ls <image>:<vfat partition><path> --native-sysroot <path>
+    wic ls <image>:<vfat or ext* partition>
+    wic ls <image>:<vfat or ext* partition><path>
+    wic ls <image>:<vfat or ext* partition><path> --native-sysroot <path>
 
 DESCRIPTION
     This command lists either partitions of the image or directory contents
-    of vfat partitions.
+    of vfat or ext* partitions.
 
     The first form it lists partitions of the image.
     For example:
@@ -319,7 +319,7 @@ DESCRIPTION
         1        1048576     24438783     23390208  fat16
         2       25165824     50315263     25149440  ext4
 
-    Second and third form list directory content of vfat partition:
+    Second and third form list directory content of the partition:
         $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
         Volume in drive : is boot
          Volume Serial Number is 2DF2-5F02
@@ -351,12 +351,12 @@ DESCRIPTION
 
 wic_cp_usage = """
 
- Copy files and directories to the vfat partitions
+ Copy files and directories to the vfat or ext* partition
 
- usage: wic cp <src> <image>:<vfat partition>[<path>] [--native-sysroot <path>]
+ usage: wic cp <src> <image>:<partition>[<path>] [--native-sysroot <path>]
 
- This command  copies local files or directories to the vfat partitions of partitioned
- image.
+ This command  copies local files or directories to the vfat or ext* partitions
+of partitioned  image.
 
  See 'wic help cp' for more detailed instructions.
 
@@ -365,19 +365,19 @@ wic_cp_usage = """
 wic_cp_help = """
 
 NAME
-    wic cp - copy files and directories to the vfat partitions
+    wic cp - copy files and directories to the vfat or ext* partitions
 
 SYNOPSIS
-    wic cp <src> <image>:<vfat partition>
-    wic cp <src> <image>:<vfat partition><path>
-    wic cp <src> <image>:<vfat partition><path> --native-sysroot <path>
+    wic cp <src> <image>:<partition>
+    wic cp <src> <image>:<partition><path>
+    wic cp <src> <image>:<partition><path> --native-sysroot <path>
 
 DESCRIPTION
-    This command copies files and directories to the vfat partition of the
-    wic image.
+    This command copies files and directories to the vfat or ext* partition of
+    the partitioned image.
 
     The first form of it copies file or directory to the root directory of
-    the vfat partition:
+    the partition:
         $ wic cp test.wks tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
         $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
         Volume in drive : is boot
@@ -393,7 +393,7 @@ DESCRIPTION
                                  15 677 440 bytes free
 
     The second form of the command copies file or directory to the specified directory
-    on the vfat partition:
+    on the partition:
        $ wic cp test tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/
        $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/
        Volume in drive : is boot
@@ -413,12 +413,12 @@ DESCRIPTION
 
 wic_rm_usage = """
 
- Remove files or directories from the vfat partitions
+ Remove files or directories from the vfat or ext* partitions
 
- usage: wic rm <image>:<vfat partition><path> [--native-sysroot <path>]
+ usage: wic rm <image>:<partition><path> [--native-sysroot <path>]
 
- This command  removes files or directories from the vfat partitions of partitioned
- image.
+ This command  removes files or directories from the vfat or ext* partitions of
+ the partitioned image.
 
  See 'wic help rm' for more detailed instructions.
 
@@ -427,15 +427,15 @@ wic_rm_usage = """
 wic_rm_help = """
 
 NAME
-    wic rm - remove files or directories from the vfat partitions
+    wic rm - remove files or directories from the vfat or ext* partitions
 
 SYNOPSIS
-    wic rm <src> <image>:<vfat partition><path>
-    wic rm <src> <image>:<vfat partition><path> --native-sysroot <path>
+    wic rm <src> <image>:<partition><path>
+    wic rm <src> <image>:<partition><path> --native-sysroot <path>
 
 DESCRIPTION
-    This command removes files or directories from the vfat partition of the
-    wic image:
+    This command removes files or directories from the vfat or ext* partition of the
+    partitioned image:
 
         $ wic ls ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1
         Volume in drive : is boot
@@ -474,7 +474,7 @@ wic_write_usage = """
 
  usage: wic write <image> <target device> [--expand [rules]] [--native-sysroot <path>]
 
- This command writes wic image to a target device (USB stick, SD card etc).
+ This command writes partitioned image to a target device (USB stick, SD card etc).
 
  See 'wic help write' for more detailed instructions.
 
@@ -483,7 +483,7 @@ wic_write_usage = """
 wic_write_help = """
 
 NAME
-    wic write - write wic image to a device
+    wic write - write an image to a device
 
 SYNOPSIS
     wic write <image> <target>
@@ -492,7 +492,7 @@ SYNOPSIS
     wic write <image> <target> --native-sysroot <path>
 
 DESCRIPTION
-    This command writes wic image to a target device (USB stick, SD card etc)
+    This command writes an image to a target device (USB stick, SD card etc)
 
         $ wic write ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic /dev/sdb
 
-- 
2.13.5

[PATCH 5/7] wic: selftest: add test_wic_ls_ext test case

[Ed Bartosh]

Tested if 'wic ls' correctly lists directory contents
of the ext* partition.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/selftest/cases/wic.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
index aa73ba4f7ea..95170b946c6 100644
--- a/meta/lib/oeqa/selftest/cases/wic.py
+++ b/meta/lib/oeqa/selftest/cases/wic.py
@@ -991,3 +991,19 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r
                 os.unlink(new_image_path)
             if os.path.exists(image_path + '.bak'):
                 os.rename(image_path + '.bak', image_path)
+
+    def test_wic_ls_ext(self):
+        """Test listing content of the ext partition using 'wic ls'"""
+        self.assertEqual(0, runCmd("wic create wictestdisk "
+                                   "--image-name=core-image-minimal "
+                                   "-D -o %s" % self.resultdir).status)
+        images = glob(self.resultdir + "wictestdisk-*.direct")
+        self.assertEqual(1, len(images))
+
+        sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools')
+
+        # list directory content of the second ext4 partition
+        result = runCmd("wic ls %s:2/ -n %s" % (images[0], sysroot))
+        self.assertEqual(0, result.status)
+        self.assertTrue(set(['bin', 'home', 'proc', 'usr', 'var', 'dev', 'lib', 'sbin']).issubset(
+                            set(line.split()[-1] for line in result.output.split('\n') if line)))
-- 
2.13.5

[PATCH 6/7] wic: selftest: add test_wic_cp_ext test case

[Ed Bartosh]

Tested if 'wic cp' correctly copies files to the ext4 partition
of the wic image.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/selftest/cases/wic.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
index 95170b946c6..1f0383c091f 100644
--- a/meta/lib/oeqa/selftest/cases/wic.py
+++ b/meta/lib/oeqa/selftest/cases/wic.py
@@ -1007,3 +1007,33 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r
         self.assertEqual(0, result.status)
         self.assertTrue(set(['bin', 'home', 'proc', 'usr', 'var', 'dev', 'lib', 'sbin']).issubset(
                             set(line.split()[-1] for line in result.output.split('\n') if line)))
+
+    def test_wic_cp_ext(self):
+        """Test copy files and directories to the ext partition."""
+        self.assertEqual(0, runCmd("wic create wictestdisk "
+                                   "--image-name=core-image-minimal "
+                                   "-D -o %s" % self.resultdir).status)
+        images = glob(self.resultdir + "wictestdisk-*.direct")
+        self.assertEqual(1, len(images))
+
+        sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools')
+
+        # list directory content of the ext4 partition
+        result = runCmd("wic ls %s:2/ -n %s" % (images[0], sysroot))
+        self.assertEqual(0, result.status)
+        dirs = set(line.split()[-1] for line in result.output.split('\n') if line)
+        self.assertTrue(set(['bin', 'home', 'proc', 'usr', 'var', 'dev', 'lib', 'sbin']).issubset(dirs))
+
+        with NamedTemporaryFile("w", suffix=".wic-cp") as testfile:
+            testfile.write("test")
+
+            # copy file to the partition
+            result = runCmd("wic cp %s %s:2/ -n %s" % (testfile.name, images[0], sysroot))
+            self.assertEqual(0, result.status)
+
+            # check if file is there
+            result = runCmd("wic ls %s:2/ -n %s" % (images[0], sysroot))
+            self.assertEqual(0, result.status)
+            newdirs = set(line.split()[-1] for line in result.output.split('\n') if line)
+            self.assertEqual(newdirs.difference(dirs), set([os.path.basename(testfile.name)]))
+
-- 
2.13.5

[PATCH 7/7] wic: selftest: add test_wic_rm_ext test case

[Ed Bartosh]

Tested if 'wic rm' correctly removes files from the ext4 partition
of the wic image.

Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
---
 meta/lib/oeqa/selftest/cases/wic.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
index 1f0383c091f..e416a3724ab 100644
--- a/meta/lib/oeqa/selftest/cases/wic.py
+++ b/meta/lib/oeqa/selftest/cases/wic.py
@@ -1037,3 +1037,26 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r
             newdirs = set(line.split()[-1] for line in result.output.split('\n') if line)
             self.assertEqual(newdirs.difference(dirs), set([os.path.basename(testfile.name)]))
 
+    def test_wic_rm_ext(self):
+        """Test removing files from the ext partition."""
+        self.assertEqual(0, runCmd("wic create mkefidisk "
+                                   "--image-name=core-image-minimal "
+                                   "-D -o %s" % self.resultdir).status)
+        images = glob(self.resultdir + "mkefidisk-*.direct")
+        self.assertEqual(1, len(images))
+
+        sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools')
+
+        # list directory content of the /etc directory on ext4 partition
+        result = runCmd("wic ls %s:2/etc/ -n %s" % (images[0], sysroot))
+        self.assertEqual(0, result.status)
+        self.assertTrue('fstab' in [line.split()[-1] for line in result.output.split('\n') if line])
+
+        # remove file
+        result = runCmd("wic rm %s:2/etc/fstab -n %s" % (images[0], sysroot))
+        self.assertEqual(0, result.status)
+
+        # check if it's removed
+        result = runCmd("wic ls %s:2/etc/ -n %s" % (images[0], sysroot))
+        self.assertEqual(0, result.status)
+        self.assertTrue('fstab' not in [line.split()[-1] for line in result.output.split('\n') if line])
-- 
2.13.5