All of lore.kernel.org
 help / color / mirror / Atom feed
* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
@ 2017-03-02  9:06 Sébastien Szymanski
  2017-03-02 10:34 ` Thomas Petazzoni
  2017-03-18 14:30 ` Yann E. MORIN
  0 siblings, 2 replies; 10+ messages in thread
From: Sébastien Szymanski @ 2017-03-02  9:06 UTC (permalink / raw)
  To: buildroot

mkfs is now capable of generating rootfs images. Use mkfs intead of
genext2fs.

Signed-off-by: S?bastien Szymanski <sebastien.szymanski@armadeus.com>
---
 package/mke2img/Config.in.host |  1 -
 package/mke2img/mke2img        | 58 +++++++++++++++++-------------------------
 package/mke2img/mke2img.mk     |  2 +-
 3 files changed, 25 insertions(+), 36 deletions(-)

diff --git a/package/mke2img/Config.in.host b/package/mke2img/Config.in.host
index b5bcb84..f252715 100644
--- a/package/mke2img/Config.in.host
+++ b/package/mke2img/Config.in.host
@@ -1,7 +1,6 @@
 config BR2_PACKAGE_HOST_MKE2IMG
 	bool "host mke2img"
 	select BR2_PACKAGE_HOST_E2FSPROGS
-	select BR2_PACKAGE_HOST_GENEXT2FS
 	help
 	  Easily create filesystems of the extend familly: ext2/3/4.
 
diff --git a/package/mke2img/mke2img b/package/mke2img/mke2img
index c2e0d02..15b7246 100755
--- a/package/mke2img/mke2img
+++ b/package/mke2img/mke2img
@@ -10,9 +10,8 @@ set -e
 main() {
     local OPT OPTARG
     local nb_blocks nb_inodes nb_res_blocks root_dir image gen rev label uuid
-    local -a genext2fs_opts
-    local -a tune2fs_opts
-    local tune2fs_O_opts
+    local -a mkfs_opts
+    local mkfs_O_opts
 
     # Default values
     gen=2
@@ -82,55 +81,51 @@ main() {
 
     # Upgrade to rev1 if needed
     if [ ${rev} -ge 1 ]; then
-        tune2fs_O_opts+=",filetype,sparse_super"
+        mkfs_O_opts+=",filetype,sparse_super"
     fi
 
     # Add a journal for ext3 and above
+    # resize_inode for online resizing
     if [ ${gen} -ge 3 ]; then
-        tune2fs_opts+=( -j -J size=1 )
+        mkfs_opts+=( -j -J size=1 )
+        mkfs_O_opts+=",resize_inode"
     fi
 
     # Add ext4 specific features
     if [ ${gen} -ge 4 ]; then
-        tune2fs_O_opts+=",extents,uninit_bg,dir_index"
+        mkfs_O_opts+=",extents,uninit_bg,dir_index"
     fi
 
+    # Disable some defaults features
+    mkfs_O_opts+=",^ext_attr,^64bit,^flex_bg,^large_file,^huge_file,^dir_nlink,^extra_isize"
+
     # Add our -O options (there will be at most one leading comma, remove it)
-    if [ -n "${tune2fs_O_opts}" ]; then
-        tune2fs_opts+=( -O "${tune2fs_O_opts#,}" )
+    if [ -n "${mkfs_O_opts}" ]; then
+        mkfs_opts+=( -O "${mkfs_O_opts#,}" )
     fi
 
     # Add the label if specified
     if [ -n "${label}" ]; then
-        tune2fs_opts+=( -L "${label}" )
+        mkfs_opts+=( -L "${label}" )
     fi
 
-    # Generate the filesystem
-    genext2fs_opts=( -z -b ${nb_blocks} -N ${nb_inodes} -d "${root_dir}" )
-    if [ -n "${nb_res_blocks}" ]; then
-        genext2fs_opts+=( -m ${nb_res_blocks} )
-    fi
-    genext2fs "${genext2fs_opts[@]}" "${image}"
-
-    # genext2fs does not generate a UUID, but fsck will whine if one
-    # is missing, so we need to add a UUID.
-    # Of course, this has to happen _before_ we run fsck.
-    # Also, some ext4 metadata are based on the UUID, so we must
-    # set it before we can convert the filesystem to ext4.
-    # If the user did not specify a UUID, we generate a random one.
+    # If the user did not specify a UUID, mkfs will generate a random one.
     # Although a random UUID may seem bad for reproducibility, there
     # already are so many things that are not reproducible in a
     # filesystem: file dates, file ordering, content of the files...
-    tune2fs -U "${uuid:-random}" "${image}"
+    if [ -n "${UUID}" ]; then
+        mkfs_opts+=( -U "${UUID}" )
+    fi
 
-    # Upgrade the filesystem
-    if [ ${#tune2fs_opts[@]} -ne 0 ]; then
-        tune2fs "${tune2fs_opts[@]}" "${image}"
+    # Generate the filesystem
+    mkfs_opts+=( -d "${root_dir}" -N ${nb_inodes} -T small -F )
+    if [ -n "${nb_res_block}" ]; then
+        mkfs_opts+=( -m ${nb_res_blocks} )
     fi
+    mkfs.ext${gen} "${mkfs_opts[@]}" "${image}" "${nb_blocks}"
 
-    # After changing filesystem options, running fsck is required
-    # (see: man tune2fs). Running e2fsck in other cases will ensure
-    # coherency of the filesystem, although it is not required.
+    # Running e2fsck will ensure coherency of the filesystem,
+    # although it is not required.
     # 'e2fsck -pDf' means:
     #  - automatically repair
     #  - optimise and check for duplicate entries
@@ -149,11 +144,6 @@ main() {
     printf "\n"
     trace "e2fsck was successfully run on '%s' (ext%d)\n" "${image}" ${gen}
     printf "\n"
-
-    # Remove count- and time-based checks, they are not welcome
-    # on embedded devices, where they can cause serious boot-time
-    # issues by tremendously slowing down the boot.
-    tune2fs -c 0 -i 0 "${image}"
 }
 
 help() {
diff --git a/package/mke2img/mke2img.mk b/package/mke2img/mke2img.mk
index 9de387a..ead9d70 100644
--- a/package/mke2img/mke2img.mk
+++ b/package/mke2img/mke2img.mk
@@ -4,7 +4,7 @@
 #
 ################################################################################
 
-HOST_MKE2IMG_DEPENDENCIES = host-genext2fs host-e2fsprogs
+HOST_MKE2IMG_DEPENDENCIES = host-e2fsprogs
 
 define HOST_MKE2IMG_INSTALL_CMDS
 	$(INSTALL) -D -m 0755 package/mke2img/mke2img $(HOST_DIR)/usr/bin/mke2img
-- 
2.7.3

^ permalink raw reply related	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-02  9:06 [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image Sébastien Szymanski
@ 2017-03-02 10:34 ` Thomas Petazzoni
  2017-03-02 12:45   ` Sébastien Szymanski
  2017-03-18 14:30 ` Yann E. MORIN
  1 sibling, 1 reply; 10+ messages in thread
From: Thomas Petazzoni @ 2017-03-02 10:34 UTC (permalink / raw)
  To: buildroot

Hello,

On Thu,  2 Mar 2017 10:06:16 +0100, S?bastien Szymanski wrote:
> mkfs is now capable of generating rootfs images. Use mkfs intead of
> genext2fs.
> 
> Signed-off-by: S?bastien Szymanski <sebastien.szymanski@armadeus.com>
> ---
>  package/mke2img/Config.in.host |  1 -
>  package/mke2img/mke2img        | 58 +++++++++++++++++-------------------------
>  package/mke2img/mke2img.mk     |  2 +-
>  3 files changed, 25 insertions(+), 36 deletions(-)

Thanks for working on this, definitely very useful.

Do we still need the mke2img wrapper script? The only reason why this
wrapper script was created is because genext2fs was too stupid, and
many things had to be done "by hand" (like calling tune2fs, etc.) and
it became too nasty to do in fs/ext2/ext2.mk.

Now that we use mkfs, is it possible to get rid of mke2img entirely?

> +    # Disable some defaults features
> +    mkfs_O_opts+=",^ext_attr,^64bit,^flex_bg,^large_file,^huge_file,^dir_nlink,^extra_isize"

Why would we disable these default features

> +    # Running e2fsck will ensure coherency of the filesystem,
> +    # although it is not required.

If we use mkfs, I don't think calling e2fsck. It was really needed
between genext2fs and tune2fs, but I don't think it's needed anymore
now.

Thanks,

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux and Kernel engineering
http://free-electrons.com

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-02 10:34 ` Thomas Petazzoni
@ 2017-03-02 12:45   ` Sébastien Szymanski
  2017-03-02 12:47     ` Thomas Petazzoni
  0 siblings, 1 reply; 10+ messages in thread
From: Sébastien Szymanski @ 2017-03-02 12:45 UTC (permalink / raw)
  To: buildroot

On 03/02/2017 11:34 AM, Thomas Petazzoni wrote:
> Hello,
> 
> On Thu,  2 Mar 2017 10:06:16 +0100, S?bastien Szymanski wrote:
>> mkfs is now capable of generating rootfs images. Use mkfs intead of
>> genext2fs.
>>
>> Signed-off-by: S?bastien Szymanski <sebastien.szymanski@armadeus.com>
>> ---
>>  package/mke2img/Config.in.host |  1 -
>>  package/mke2img/mke2img        | 58 +++++++++++++++++-------------------------
>>  package/mke2img/mke2img.mk     |  2 +-
>>  3 files changed, 25 insertions(+), 36 deletions(-)
> 
> Thanks for working on this, definitely very useful.
> 
> Do we still need the mke2img wrapper script? The only reason why this
> wrapper script was created is because genext2fs was too stupid, and
> many things had to be done "by hand" (like calling tune2fs, etc.) and
> it became too nasty to do in fs/ext2/ext2.mk.
> 
> Now that we use mkfs, is it possible to get rid of mke2img entirely?

We still have to calculate the number of blocks needed. I guess we can
do that in the .mk file, can't we?

> 
>> +    # Disable some defaults features
>> +    mkfs_O_opts+=",^ext_attr,^64bit,^flex_bg,^large_file,^huge_file,^dir_nlink,^extra_isize"
> 
> Why would we disable these default features

I disable these default features, to generate a rootfs as similar as
possible the one generated by genext2fs.
U-boot had some issues with rootfs with the 64bit flags, but this has
been fixed

http://git.denx.de/?p=u-boot.git;a=commit;h=b4976b49a01ac68f7dbb33657a44dcffe584fa4a

> 
>> +    # Running e2fsck will ensure coherency of the filesystem,
>> +    # although it is not required.
> 
> If we use mkfs, I don't think calling e2fsck. It was really needed
> between genext2fs and tune2fs, but I don't think it's needed anymore
> now.

I agree. e2fsck isn't needed anymore now.

Regards,

> 
> Thanks,
> 
> Thomas
> 


-- 
S?bastien Szymanski
Software Engineer, Armadeus Systems
sebastien.szymanski at armadeus.com
Tel: +33 (0)9 72 29 41 44
Fax: +33 (0)9 72 28 79 26

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-02 12:45   ` Sébastien Szymanski
@ 2017-03-02 12:47     ` Thomas Petazzoni
  2017-03-06 13:20       ` Sébastien Szymanski
  0 siblings, 1 reply; 10+ messages in thread
From: Thomas Petazzoni @ 2017-03-02 12:47 UTC (permalink / raw)
  To: buildroot

Hello,

On Thu, 2 Mar 2017 13:45:06 +0100, S?bastien Szymanski wrote:

> > Now that we use mkfs, is it possible to get rid of mke2img entirely?  
> 
> We still have to calculate the number of blocks needed. I guess we can
> do that in the .mk file, can't we?

Do we still need to do this calculation? Can't mkfs create an image of
an arbitrary size in MBs ?

Indeed, the current code that calculates the number of blocks needed is
completely bogus, and breaks as soon as the host filesystem is not
ext2/3/4. We have two bug reports that says it doesn't work, one one
ZFS, another on NTFS, and I remember it also broke on XFS.

See:

  https://bugs.busybox.net/show_bug.cgi?id=8831 (ZFS)
  https://bugs.busybox.net/show_bug.cgi?id=9496 (NTFS)
  https://github.com/buildroot/buildroot-defconfig-testing/blob/master/.travis.yml#L158 (XFS)

So the whole "let's calculate how many blocks are needed" logic we have
simply cannot work, because the number of blocks needed completely
depends on the internal architecture of the filesystem.

Therefore, I would suggest that we get rid of this, and instead add an
option "Filesystem size" in Config.in, default to some reasonable value
like 64M or 128M, and it would be up to the user to define it to a
value that is large enough to host the package selection he has done.

> >> +    # Disable some defaults features
> >> +    mkfs_O_opts+=",^ext_attr,^64bit,^flex_bg,^large_file,^huge_file,^dir_nlink,^extra_isize"  
> > 
> > Why would we disable these default features  
> 
> I disable these default features, to generate a rootfs as similar as
> possible the one generated by genext2fs.

OK.

> U-boot had some issues with rootfs with the 64bit flags, but this has
> been fixed
> 
> http://git.denx.de/?p=u-boot.git;a=commit;h=b4976b49a01ac68f7dbb33657a44dcffe584fa4a

Then, I think we want to disable it by default indeed, since we want to
remain compatible with old U-Boot versions that may not have this fix.

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux and Kernel engineering
http://free-electrons.com

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-02 12:47     ` Thomas Petazzoni
@ 2017-03-06 13:20       ` Sébastien Szymanski
  2017-03-06 13:24         ` Thomas Petazzoni
  0 siblings, 1 reply; 10+ messages in thread
From: Sébastien Szymanski @ 2017-03-06 13:20 UTC (permalink / raw)
  To: buildroot

Hi,

On 03/02/2017 01:47 PM, Thomas Petazzoni wrote:
> Hello,
> 
> On Thu, 2 Mar 2017 13:45:06 +0100, S?bastien Szymanski wrote:
> 
>>> Now that we use mkfs, is it possible to get rid of mke2img entirely?  
>>
>> We still have to calculate the number of blocks needed. I guess we can
>> do that in the .mk file, can't we?
> 
> Do we still need to do this calculation? Can't mkfs create an image of
> an arbitrary size in MBs ?
> 
> Indeed, the current code that calculates the number of blocks needed is
> completely bogus, and breaks as soon as the host filesystem is not
> ext2/3/4. We have two bug reports that says it doesn't work, one one
> ZFS, another on NTFS, and I remember it also broke on XFS.
> 
> See:
> 
>   https://bugs.busybox.net/show_bug.cgi?id=8831 (ZFS)
>   https://bugs.busybox.net/show_bug.cgi?id=9496 (NTFS)
>   https://github.com/buildroot/buildroot-defconfig-testing/blob/master/.travis.yml#L158 (XFS)
> 
> So the whole "let's calculate how many blocks are needed" logic we have
> simply cannot work, because the number of blocks needed completely
> depends on the internal architecture of the filesystem.

I copied the target/ directory on a xfs filesystem and looked the
differences. On ext4, the size of a directory is at least the size of
block. With find, we can get the number of directories with a size less
than the size of a block:

XFS fs
$ du -s -k /mnt/target
16032

Ext4 fs (4K block size):
$ du -s -k output/target
16320

$ find /mnt/target/ -type d -size -1024c | wc -l
72

16032 + 72*4 = 16320

I didn't check if this is true with other filesystems.

> 
> Therefore, I would suggest that we get rid of this, and instead add an
> option "Filesystem size" in Config.in, default to some reasonable value
> like 64M or 128M, and it would be up to the user to define it to a
> value that is large enough to host the package selection he has done.

Can't we automate this? I mean depending of the size of the target
directory, we use one of the following size: 64M, 128M, 256M,
and then we shrink the filesystem with resize2fs -M ?

Regards,

> 
>>>> +    # Disable some defaults features
>>>> +    mkfs_O_opts+=",^ext_attr,^64bit,^flex_bg,^large_file,^huge_file,^dir_nlink,^extra_isize"  
>>>
>>> Why would we disable these default features  
>>
>> I disable these default features, to generate a rootfs as similar as
>> possible the one generated by genext2fs.
> 
> OK.
> 
>> U-boot had some issues with rootfs with the 64bit flags, but this has
>> been fixed
>>
>> http://git.denx.de/?p=u-boot.git;a=commit;h=b4976b49a01ac68f7dbb33657a44dcffe584fa4a
> 
> Then, I think we want to disable it by default indeed, since we want to
> remain compatible with old U-Boot versions that may not have this fix.
> 
> Thomas
> 


-- 
S?bastien Szymanski
Software Engineer, Armadeus Systems
sebastien.szymanski at armadeus.com
Tel: +33 (0)9 72 29 41 44
Fax: +33 (0)9 72 28 79 26

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-06 13:20       ` Sébastien Szymanski
@ 2017-03-06 13:24         ` Thomas Petazzoni
  2017-03-07  8:18           ` Arnout Vandecappelle
  0 siblings, 1 reply; 10+ messages in thread
From: Thomas Petazzoni @ 2017-03-06 13:24 UTC (permalink / raw)
  To: buildroot

Hello,

On Mon, 6 Mar 2017 14:20:56 +0100, S?bastien Szymanski wrote:

> I copied the target/ directory on a xfs filesystem and looked the
> differences. On ext4, the size of a directory is at least the size of
> block. With find, we can get the number of directories with a size less
> than the size of a block:
> 
> XFS fs
> $ du -s -k /mnt/target
> 16032
> 
> Ext4 fs (4K block size):
> $ du -s -k output/target
> 16320
> 
> $ find /mnt/target/ -type d -size -1024c | wc -l
> 72
> 
> 16032 + 72*4 = 16320
> 
> I didn't check if this is true with other filesystems.

I'm wondering if XFS also doesn't store very small files, and possibly
symbolic links, in a different way.

In any case, I believe we agree that looking at the size of files
stored in filesystem A doesn't give an accurate idea of how much space
they will consume once stored with filesystem B.

> > Therefore, I would suggest that we get rid of this, and instead add an
> > option "Filesystem size" in Config.in, default to some reasonable value
> > like 64M or 128M, and it would be up to the user to define it to a
> > value that is large enough to host the package selection he has done.  
> 
> Can't we automate this? I mean depending of the size of the target
> directory, we use one of the following size: 64M, 128M, 256M,

How do you know which one to choose? If you look at the target
directory size, then you end up doing what you do today, and which is
broken. Or you have to take some really really big margin. But OK, we
can try with even margin (like 20% or 30% margin).

> and then we shrink the filesystem with resize2fs -M ?

Why would you shrink it down? The fact that our filesystem images today
are really just the size of the files they contain is most of time
annoying, and you have to increase the size of the filesystem
afterwards to make the system usable. So it'd be great to instead have
images of 64 MB, 128 MB, 256 MB, with some free space.

Of course, we should have options to allow the user to indicate the
size he wants, and comply with this size exactly.

Best regards,

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux and Kernel engineering
http://free-electrons.com

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-06 13:24         ` Thomas Petazzoni
@ 2017-03-07  8:18           ` Arnout Vandecappelle
  0 siblings, 0 replies; 10+ messages in thread
From: Arnout Vandecappelle @ 2017-03-07  8:18 UTC (permalink / raw)
  To: buildroot



On 06-03-17 14:24, Thomas Petazzoni wrote:
> Hello,
> 
> On Mon, 6 Mar 2017 14:20:56 +0100, S?bastien Szymanski wrote:
> 
>> I copied the target/ directory on a xfs filesystem and looked the
>> differences. On ext4, the size of a directory is at least the size of
>> block. With find, we can get the number of directories with a size less
>> than the size of a block:
>>
>> XFS fs
>> $ du -s -k /mnt/target
>> 16032
>>
>> Ext4 fs (4K block size):
>> $ du -s -k output/target
>> 16320
>>
>> $ find /mnt/target/ -type d -size -1024c | wc -l
>> 72
>>
>> 16032 + 72*4 = 16320
>>
>> I didn't check if this is true with other filesystems.
> 
> I'm wondering if XFS also doesn't store very small files, and possibly
> symbolic links, in a different way.
> 
> In any case, I believe we agree that looking at the size of files
> stored in filesystem A doesn't give an accurate idea of how much space
> they will consume once stored with filesystem B.

 As an approximation, however, we could look at the apparent file size, and add
e.g. 1KB per inode. So (du --apparent-size) + 1K * (du --inodes). AFAICS this
should always be an overestimate. Well, that is, assuming that mkfs.ext2 retains
hardlinks.

>>> Therefore, I would suggest that we get rid of this, and instead add an
>>> option "Filesystem size" in Config.in, default to some reasonable value
>>> like 64M or 128M, and it would be up to the user to define it to a
>>> value that is large enough to host the package selection he has done.  
>>
>> Can't we automate this? I mean depending of the size of the target
>> directory, we use one of the following size: 64M, 128M, 256M,
> 
> How do you know which one to choose? If you look at the target
> directory size, then you end up doing what you do today, and which is
> broken. Or you have to take some really really big margin. But OK, we
> can try with even margin (like 20% or 30% margin).

 If we use "reasonable sizes", do take into account that a typical SD card is
slightly smaller than the corresponding power-of-two (or sometimes even SI)
value, and that you typically need a little bit of extra space for a boot
partition. So the "reasonable sizes" should be something like 60M, 122M, ...

 Also, the list is going to be pretty long: we have enough packages in Buildroot
to go over 1GB rootfs...


>> and then we shrink the filesystem with resize2fs -M ?
> 
> Why would you shrink it down? 

 Because resize2fs will calculate the exact required size. It can do that
because the fs is already created so all the information is there to calculate
the size.

 With this approach, we could do a serious overestimate in the first pass (e.g.
double of the apparent size), then resize to the actual required size, and then
(if BR2_TARGET_ROOTFS_EXT2_EXTRA_BLOCKS is specified) resize again to add the
extra space.

 This could work, but sounds pretty inefficient...


> The fact that our filesystem images today
> are really just the size of the files they contain is most of time
> annoying, and you have to increase the size of the filesystem
> afterwards to make the system usable. So it'd be great to instead have
> images of 64 MB, 128 MB, 256 MB, with some free space.
> 
> Of course, we should have options to allow the user to indicate the
> size he wants, and comply with this size exactly.

 We have that: BR2_TARGET_ROOTFS_EXT2_BLOCKS. Before that was introduced (in
2005), the size was always auto-calculated.

 Regards,
 Arnout


-- 
Arnout Vandecappelle                          arnout at mind be
Senior Embedded Software Architect            +32-16-286500
Essensium/Mind                                http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium           BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  7493 020B C7E3 8618 8DEC 222C 82EB F404 F9AC 0DDF

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-02  9:06 [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image Sébastien Szymanski
  2017-03-02 10:34 ` Thomas Petazzoni
@ 2017-03-18 14:30 ` Yann E. MORIN
  2017-03-18 14:42   ` Arnout Vandecappelle
  1 sibling, 1 reply; 10+ messages in thread
From: Yann E. MORIN @ 2017-03-18 14:30 UTC (permalink / raw)
  To: buildroot

S?bastien, All,

On 2017-03-02 10:06 +0100, S?bastien Szymanski spake thusly:
> mkfs is now capable of generating rootfs images. Use mkfs intead of
> genext2fs.
> 
> Signed-off-by: S?bastien Szymanski <sebastien.szymanski@armadeus.com>

So, to sumarize the discussion in this thread, and a discussion Thomas,
Arnout and I just had on IRC:

 1. mke2fs can't auto-calculate the size;

 2. auto-calculation is inherently flawed, because it relies on the host
    filesystem specifics;

 3. so we want the user to be responsible for specifying the exact size
    he wants for his extfs;

 4. we don't care much about generating filesystems that are bckward
    identical (i.e. we don;t care if the enerated filesystem does not
    have the same feature set as was previously done);

 5. except we want to drop features that make it incompatible with older
    U-Boot or other bootlaoders, or older kernels, but that should be
    optional.

 6. can we kill most of mke2img, or can we even kill it altogether? ;-)

Thanks!

Regards,
Yann E. MORIN.

> ---
>  package/mke2img/Config.in.host |  1 -
>  package/mke2img/mke2img        | 58 +++++++++++++++++-------------------------
>  package/mke2img/mke2img.mk     |  2 +-
>  3 files changed, 25 insertions(+), 36 deletions(-)
> 
> diff --git a/package/mke2img/Config.in.host b/package/mke2img/Config.in.host
> index b5bcb84..f252715 100644
> --- a/package/mke2img/Config.in.host
> +++ b/package/mke2img/Config.in.host
> @@ -1,7 +1,6 @@
>  config BR2_PACKAGE_HOST_MKE2IMG
>  	bool "host mke2img"
>  	select BR2_PACKAGE_HOST_E2FSPROGS
> -	select BR2_PACKAGE_HOST_GENEXT2FS
>  	help
>  	  Easily create filesystems of the extend familly: ext2/3/4.
>  
> diff --git a/package/mke2img/mke2img b/package/mke2img/mke2img
> index c2e0d02..15b7246 100755
> --- a/package/mke2img/mke2img
> +++ b/package/mke2img/mke2img
> @@ -10,9 +10,8 @@ set -e
>  main() {
>      local OPT OPTARG
>      local nb_blocks nb_inodes nb_res_blocks root_dir image gen rev label uuid
> -    local -a genext2fs_opts
> -    local -a tune2fs_opts
> -    local tune2fs_O_opts
> +    local -a mkfs_opts
> +    local mkfs_O_opts
>  
>      # Default values
>      gen=2
> @@ -82,55 +81,51 @@ main() {
>  
>      # Upgrade to rev1 if needed
>      if [ ${rev} -ge 1 ]; then
> -        tune2fs_O_opts+=",filetype,sparse_super"
> +        mkfs_O_opts+=",filetype,sparse_super"
>      fi
>  
>      # Add a journal for ext3 and above
> +    # resize_inode for online resizing
>      if [ ${gen} -ge 3 ]; then
> -        tune2fs_opts+=( -j -J size=1 )
> +        mkfs_opts+=( -j -J size=1 )
> +        mkfs_O_opts+=",resize_inode"
>      fi
>  
>      # Add ext4 specific features
>      if [ ${gen} -ge 4 ]; then
> -        tune2fs_O_opts+=",extents,uninit_bg,dir_index"
> +        mkfs_O_opts+=",extents,uninit_bg,dir_index"
>      fi
>  
> +    # Disable some defaults features
> +    mkfs_O_opts+=",^ext_attr,^64bit,^flex_bg,^large_file,^huge_file,^dir_nlink,^extra_isize"
> +
>      # Add our -O options (there will be at most one leading comma, remove it)
> -    if [ -n "${tune2fs_O_opts}" ]; then
> -        tune2fs_opts+=( -O "${tune2fs_O_opts#,}" )
> +    if [ -n "${mkfs_O_opts}" ]; then
> +        mkfs_opts+=( -O "${mkfs_O_opts#,}" )
>      fi
>  
>      # Add the label if specified
>      if [ -n "${label}" ]; then
> -        tune2fs_opts+=( -L "${label}" )
> +        mkfs_opts+=( -L "${label}" )
>      fi
>  
> -    # Generate the filesystem
> -    genext2fs_opts=( -z -b ${nb_blocks} -N ${nb_inodes} -d "${root_dir}" )
> -    if [ -n "${nb_res_blocks}" ]; then
> -        genext2fs_opts+=( -m ${nb_res_blocks} )
> -    fi
> -    genext2fs "${genext2fs_opts[@]}" "${image}"
> -
> -    # genext2fs does not generate a UUID, but fsck will whine if one
> -    # is missing, so we need to add a UUID.
> -    # Of course, this has to happen _before_ we run fsck.
> -    # Also, some ext4 metadata are based on the UUID, so we must
> -    # set it before we can convert the filesystem to ext4.
> -    # If the user did not specify a UUID, we generate a random one.
> +    # If the user did not specify a UUID, mkfs will generate a random one.
>      # Although a random UUID may seem bad for reproducibility, there
>      # already are so many things that are not reproducible in a
>      # filesystem: file dates, file ordering, content of the files...
> -    tune2fs -U "${uuid:-random}" "${image}"
> +    if [ -n "${UUID}" ]; then
> +        mkfs_opts+=( -U "${UUID}" )
> +    fi
>  
> -    # Upgrade the filesystem
> -    if [ ${#tune2fs_opts[@]} -ne 0 ]; then
> -        tune2fs "${tune2fs_opts[@]}" "${image}"
> +    # Generate the filesystem
> +    mkfs_opts+=( -d "${root_dir}" -N ${nb_inodes} -T small -F )
> +    if [ -n "${nb_res_block}" ]; then
> +        mkfs_opts+=( -m ${nb_res_blocks} )
>      fi
> +    mkfs.ext${gen} "${mkfs_opts[@]}" "${image}" "${nb_blocks}"
>  
> -    # After changing filesystem options, running fsck is required
> -    # (see: man tune2fs). Running e2fsck in other cases will ensure
> -    # coherency of the filesystem, although it is not required.
> +    # Running e2fsck will ensure coherency of the filesystem,
> +    # although it is not required.
>      # 'e2fsck -pDf' means:
>      #  - automatically repair
>      #  - optimise and check for duplicate entries
> @@ -149,11 +144,6 @@ main() {
>      printf "\n"
>      trace "e2fsck was successfully run on '%s' (ext%d)\n" "${image}" ${gen}
>      printf "\n"
> -
> -    # Remove count- and time-based checks, they are not welcome
> -    # on embedded devices, where they can cause serious boot-time
> -    # issues by tremendously slowing down the boot.
> -    tune2fs -c 0 -i 0 "${image}"
>  }
>  
>  help() {
> diff --git a/package/mke2img/mke2img.mk b/package/mke2img/mke2img.mk
> index 9de387a..ead9d70 100644
> --- a/package/mke2img/mke2img.mk
> +++ b/package/mke2img/mke2img.mk
> @@ -4,7 +4,7 @@
>  #
>  ################################################################################
>  
> -HOST_MKE2IMG_DEPENDENCIES = host-genext2fs host-e2fsprogs
> +HOST_MKE2IMG_DEPENDENCIES = host-e2fsprogs
>  
>  define HOST_MKE2IMG_INSTALL_CMDS
>  	$(INSTALL) -D -m 0755 package/mke2img/mke2img $(HOST_DIR)/usr/bin/mke2img
> -- 
> 2.7.3
> 
> _______________________________________________
> buildroot mailing list
> buildroot at busybox.net
> http://lists.busybox.net/mailman/listinfo/buildroot

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 223 225 172 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-18 14:30 ` Yann E. MORIN
@ 2017-03-18 14:42   ` Arnout Vandecappelle
  2017-03-20 15:24     ` Sébastien Szymanski
  0 siblings, 1 reply; 10+ messages in thread
From: Arnout Vandecappelle @ 2017-03-18 14:42 UTC (permalink / raw)
  To: buildroot



On 18-03-17 15:30, Yann E. MORIN wrote:
> S?bastien, All,
> 
> On 2017-03-02 10:06 +0100, S?bastien Szymanski spake thusly:
>> mkfs is now capable of generating rootfs images. Use mkfs intead of
>> genext2fs.
>>
>> Signed-off-by: S?bastien Szymanski <sebastien.szymanski@armadeus.com>
> 
> So, to sumarize the discussion in this thread, and a discussion Thomas,
> Arnout and I just had on IRC:
> 
>  1. mke2fs can't auto-calculate the size;
> 
>  2. auto-calculation is inherently flawed, because it relies on the host
>     filesystem specifics;
> 
>  3. so we want the user to be responsible for specifying the exact size
>     he wants for his extfs;
> 
>  4. we don't care much about generating filesystems that are bckward
>     identical (i.e. we don;t care if the enerated filesystem does not
>     have the same feature set as was previously done);
> 
>  5. except we want to drop features that make it incompatible with older
>     U-Boot or other bootlaoders, or older kernels, but that should be
>     optional.
> 
>  6. can we kill most of mke2img, or can we even kill it altogether? ;-)

 So what we would like to see is something like the following patches:

1. Remove support for auto-calculation of BR2_TARGET_ROOTFS_EXT2_BLOCKS
   (move BR2_TARGET_ROOTFS_EXT2_EXTRA_BLOCKS to Config.in.legacy).
2. Switch to mkfs to generate the rootfs image (i.e. this patch).
3. Remove the mke2img package altogether, instead call mke2fs directly from
fs/ext2/ext2.mk
4. Switch BR2_TARGET_ROOTFS_EXT2_BLOCKS to a string so that you can specify
   120M instead of 120000000 - and also give it a reasonable default value.

 Do you feel up to it?

 Regards,
 Arnout

[snip]

-- 
Arnout Vandecappelle                          arnout at mind be
Senior Embedded Software Architect            +32-16-286500
Essensium/Mind                                http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium           BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  7493 020B C7E3 8618 8DEC 222C 82EB F404 F9AC 0DDF

^ permalink raw reply	[flat|nested] 10+ messages in thread

* [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image
  2017-03-18 14:42   ` Arnout Vandecappelle
@ 2017-03-20 15:24     ` Sébastien Szymanski
  0 siblings, 0 replies; 10+ messages in thread
From: Sébastien Szymanski @ 2017-03-20 15:24 UTC (permalink / raw)
  To: buildroot

Hello,

On 03/18/2017 03:42 PM, Arnout Vandecappelle wrote:
> 
> 
> On 18-03-17 15:30, Yann E. MORIN wrote:
>> S?bastien, All,
>>
>> On 2017-03-02 10:06 +0100, S?bastien Szymanski spake thusly:
>>> mkfs is now capable of generating rootfs images. Use mkfs intead of
>>> genext2fs.
>>>
>>> Signed-off-by: S?bastien Szymanski <sebastien.szymanski@armadeus.com>
>>
>> So, to sumarize the discussion in this thread, and a discussion Thomas,
>> Arnout and I just had on IRC:
>>
>>  1. mke2fs can't auto-calculate the size;
>>
>>  2. auto-calculation is inherently flawed, because it relies on the host
>>     filesystem specifics;
>>
>>  3. so we want the user to be responsible for specifying the exact size
>>     he wants for his extfs;
>>
>>  4. we don't care much about generating filesystems that are bckward
>>     identical (i.e. we don;t care if the enerated filesystem does not
>>     have the same feature set as was previously done);
>>
>>  5. except we want to drop features that make it incompatible with older
>>     U-Boot or other bootlaoders, or older kernels, but that should be
>>     optional.
>>
>>  6. can we kill most of mke2img, or can we even kill it altogether? ;-)
> 
>  So what we would like to see is something like the following patches:
> 
> 1. Remove support for auto-calculation of BR2_TARGET_ROOTFS_EXT2_BLOCKS
>    (move BR2_TARGET_ROOTFS_EXT2_EXTRA_BLOCKS to Config.in.legacy).
> 2. Switch to mkfs to generate the rootfs image (i.e. this patch).
> 3. Remove the mke2img package altogether, instead call mke2fs directly from
> fs/ext2/ext2.mk
> 4. Switch BR2_TARGET_ROOTFS_EXT2_BLOCKS to a string so that you can specify
>    120M instead of 120000000 - and also give it a reasonable default value.
> 
>  Do you feel up to it?

Yes, I will try to send a new version this week.

Regards,

> 
>  Regards,
>  Arnout
> 
> [snip]
> 


-- 
S?bastien Szymanski
Software Engineer, Armadeus Systems
sebastien.szymanski at armadeus.com
Tel: +33 (0)9 72 29 41 44
Fax: +33 (0)9 72 28 79 26

^ permalink raw reply	[flat|nested] 10+ messages in thread

end of thread, other threads:[~2017-03-20 15:24 UTC | newest]

Thread overview: 10+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2017-03-02  9:06 [Buildroot] [PATCH 1/1] package/mke2img: use mkfs to generate rootfs image Sébastien Szymanski
2017-03-02 10:34 ` Thomas Petazzoni
2017-03-02 12:45   ` Sébastien Szymanski
2017-03-02 12:47     ` Thomas Petazzoni
2017-03-06 13:20       ` Sébastien Szymanski
2017-03-06 13:24         ` Thomas Petazzoni
2017-03-07  8:18           ` Arnout Vandecappelle
2017-03-18 14:30 ` Yann E. MORIN
2017-03-18 14:42   ` Arnout Vandecappelle
2017-03-20 15:24     ` Sébastien Szymanski

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.