[Buildroot] [PATCH 1/1] fs/squashfs: add more options to customize creation

[Buildroot] [PATCH 1/1] fs/squashfs: add more options to customize creation

[Linus Kaschulla via buildroot]

Squashfs seems to be the best way to shipping a highly compressed
rootfs that can easily be mounted without any extraction overhead.
While trying to get the squashfs as small as possible, I found out
that buildroot only allows rather basic customizations and many
options to improve the compression ratio are not available.

Notably, squashfs doesn't use the maximum available blocksize by
default and compression options offered by `mksquashfs` are not
changable from buildroot without patching the source code.

This patch adds 2 new options to fs/squashfs:

 - Changing the blocksize (e.g. to improve compression ratio)
 - Appending any custom arguments (e.g. for adding compression options)

Using these options, specifying the max blocksize of 1M and adding
the arguments `-Xbcj arm` allows for a notably smaller rootfs when
using xz compression and building for an arm target.

Signed-off-by: Linus Kaschulla <linus@cosmos-ink.net>
---
 fs/squashfs/Config.in   | 20 ++++++++++++++++++++
 fs/squashfs/squashfs.mk |  8 ++++++++
 2 files changed, 28 insertions(+)

diff --git a/fs/squashfs/Config.in b/fs/squashfs/Config.in
index 70d4a20cf0..7ae2743961 100644
--- a/fs/squashfs/Config.in
+++ b/fs/squashfs/Config.in
@@ -12,6 +12,16 @@ config BR2_TARGET_ROOTFS_SQUASHFS_PAD
 	  Say 'y' here (the default) to pad the the filesystem image
 	  to a 4K boundary. Say 'n' to disable padding.
 
+config BR2_TARGET_ROOTFS_SQUASHFS4_BLOCKSIZE
+	string "block size"
+	help
+	  Define a specific block size to use when creating the
+	  squashfs. Higher values can improve compression ratios.
+	  The suffixes K and M are supported. The value should not
+	  be above 1 MiB.
+
+	  If unsure, leave this empty to use the default blocksize.
+
 choice
 	prompt "Compression algorithm"
 	default BR2_TARGET_ROOTFS_SQUASHFS4_GZIP
@@ -39,4 +49,14 @@ config BR2_TARGET_ROOTFS_SQUASHFS4_ZSTD
 
 endchoice
 
+config BR2_TARGET_ROOTFS_SQUASHFS4_CUSTOM_ARGS
+	string "additional arguments to pass to the squashfs creation"
+	help
+	  Append extra commandline arguments to the mksquashfs command.
+	  This can be useful for finetuning a selected compression or
+	  enable/disable other features of squashfs.
+	  See "$HOST_DIR/bin/mksquashfs -h" for available arguments.
+
+	  If unsure, leave empty.
+
 endif
diff --git a/fs/squashfs/squashfs.mk b/fs/squashfs/squashfs.mk
index 7a5e3e313e..42343b1b40 100644
--- a/fs/squashfs/squashfs.mk
+++ b/fs/squashfs/squashfs.mk
@@ -12,6 +12,10 @@ ifeq ($(BR2_TARGET_ROOTFS_SQUASHFS_PAD),)
 ROOTFS_SQUASHFS_ARGS += -nopad
 endif
 
+ifneq ($(BR2_TARGET_ROOTFS_SQUASHFS4_BLOCKSIZE),"")
+ROOTFS_SQUASHFS_ARGS += -b "$(BR2_TARGET_ROOTFS_SQUASHFS4_BLOCKSIZE)"
+endif
+
 ifeq ($(BR2_TARGET_ROOTFS_SQUASHFS4_LZ4),y)
 ROOTFS_SQUASHFS_ARGS += -comp lz4 -Xhc
 else ifeq ($(BR2_TARGET_ROOTFS_SQUASHFS4_LZO),y)
@@ -26,6 +30,10 @@ else
 ROOTFS_SQUASHFS_ARGS += -comp gzip
 endif
 
+ifneq ($(BR2_TARGET_ROOTFS_SQUASHFS4_CUSTOM_ARGS),"")
+ROOTFS_SQUASHFS_ARGS += $(call qstrip,$(BR2_TARGET_ROOTFS_SQUASHFS4_CUSTOM_ARGS))
+endif
+
 define ROOTFS_SQUASHFS_CMD
 	$(HOST_DIR)/bin/mksquashfs $(TARGET_DIR) $@ $(ROOTFS_SQUASHFS_ARGS)
 endef
-- 
2.35.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/1] fs/squashfs: add more options to customize creation

[Yann E. MORIN]

Linus, All,

On 2022-05-15 14:54 +0000, Linus Kaschulla via buildroot spake thusly:
> Squashfs seems to be the best way to shipping a highly compressed
> rootfs that can easily be mounted without any extraction overhead.
> While trying to get the squashfs as small as possible, I found out
> that buildroot only allows rather basic customizations and many
> options to improve the compression ratio are not available.
> 
> Notably, squashfs doesn't use the maximum available blocksize by
> default and compression options offered by `mksquashfs` are not
> changable from buildroot without patching the source code.
> 
> This patch adds 2 new options to fs/squashfs:
> 
>  - Changing the blocksize (e.g. to improve compression ratio)
>  - Appending any custom arguments (e.g. for adding compression options)

Two changes, two patches. ;-)

> Using these options, specifying the max blocksize of 1M and adding
> the arguments `-Xbcj arm` allows for a notably smaller rootfs when
> using xz compression and building for an arm target.

So, I think this is good to have those additional options, however I
think we should restrict them. For example, arbitrary string options is
not very nice.

Instead, here's what I suggest:

  - for the blocksize: add a choice, for the only possible and
    meaningful values (1L is probably too low, maybe 4K should be the
    minimum?)

        choice
            bool "blocksize"
            default BR2_TARGET_ROOTFS_BS_128K

        config BR2_TARGET_ROOTFS_BS_1K
            bool "1k"

        config BR2_TARGET_ROOTFS_BS_2K
            bool "1k"

        [...]

        config BR2_TARGET_ROOTFS_BS_1M
            bool "1k"

        endchoice

        config BR2_TARGET_ROOTFS_BS
            string
            default "1K"   if BR2_TARGET_ROOTFS_BS_1K
            default "2K"   if BR2_TARGET_ROOTFS_BS_2K
            ...
            default "1M"   if BR2_TARGET_ROOTFS_BS_1M

    and in squashfs.mk:

        ROOTFS_SQUASHFS_ARGS += -b $(call qstrip,$(BR2_TARGET_ROOTFS_SQUASHFS4_BLOCKSIZE))

  - for the comnpression options, I think a generic boolean would be
    better:

        config BR2_TARGET_ROOTFS_EXTREME_COMP
            bool "extreme compression if possible"
            help
              Use options to increase compression ration as much as
              possible, like using architecture-specific options, at
              the cost of time when assmebline the filesystem image.

              For example:
                - with gzip, use -Xcompression-level 9
                - with lzo, use -Xcompression-level 9
                - with xz on ARM, use -Xbcj arm.thumb

        config BR2_TARGET_ROOTFS_COMP_OPTS
            string
            default "-Xcompression-level 9" if BR2_TARGET_ROOTFS_SQUASHFS4_GZIP
            default "-Xcompression-level 9" if BR2_TARGET_ROOTFS_SQUASHFS4_LZO
            default "-Xbcj arm,armthumb" if BR2_TARGET_ROOTFS_SQUASHFS4_XZ && (BR2_arm || BR2_armeb)
            default "-Xbcj x86" if BR2_TARGET_ROOTFS_SQUASHFS4_XZ && (BR2_i386 || BR2_x86_64)
            ...

    and in squashfs.mk:

        ROOTFS_SQUASHFS_ARGS += $(call qstrip,$(BR2_TARGET_ROOTFS_COMP_OPTS))

    Also note that lz4 already uses -Xhc unconditionally, and commit
    07e37bcc42f notes that an extra option was dropped, so maybe we
    should just unconditionally enable all those options when possible?

And of course, this should be in two patches: one to introudce the
blocksize, one to introduce the extreme compression optiopns

Regards,
Yann E. MORIN.

> Signed-off-by: Linus Kaschulla <linus@cosmos-ink.net>
> ---
>  fs/squashfs/Config.in   | 20 ++++++++++++++++++++
>  fs/squashfs/squashfs.mk |  8 ++++++++
>  2 files changed, 28 insertions(+)
> 
> diff --git a/fs/squashfs/Config.in b/fs/squashfs/Config.in
> index 70d4a20cf0..7ae2743961 100644
> --- a/fs/squashfs/Config.in
> +++ b/fs/squashfs/Config.in
> @@ -12,6 +12,16 @@ config BR2_TARGET_ROOTFS_SQUASHFS_PAD
>  	  Say 'y' here (the default) to pad the the filesystem image
>  	  to a 4K boundary. Say 'n' to disable padding.
>  
> +config BR2_TARGET_ROOTFS_SQUASHFS4_BLOCKSIZE
> +	string "block size"
> +	help
> +	  Define a specific block size to use when creating the
> +	  squashfs. Higher values can improve compression ratios.
> +	  The suffixes K and M are supported. The value should not
> +	  be above 1 MiB.
> +
> +	  If unsure, leave this empty to use the default blocksize.
> +
>  choice
>  	prompt "Compression algorithm"
>  	default BR2_TARGET_ROOTFS_SQUASHFS4_GZIP
> @@ -39,4 +49,14 @@ config BR2_TARGET_ROOTFS_SQUASHFS4_ZSTD
>  
>  endchoice
>  
> +config BR2_TARGET_ROOTFS_SQUASHFS4_CUSTOM_ARGS
> +	string "additional arguments to pass to the squashfs creation"
> +	help
> +	  Append extra commandline arguments to the mksquashfs command.
> +	  This can be useful for finetuning a selected compression or
> +	  enable/disable other features of squashfs.
> +	  See "$HOST_DIR/bin/mksquashfs -h" for available arguments.
> +
> +	  If unsure, leave empty.
> +
>  endif
> diff --git a/fs/squashfs/squashfs.mk b/fs/squashfs/squashfs.mk
> index 7a5e3e313e..42343b1b40 100644
> --- a/fs/squashfs/squashfs.mk
> +++ b/fs/squashfs/squashfs.mk
> @@ -12,6 +12,10 @@ ifeq ($(BR2_TARGET_ROOTFS_SQUASHFS_PAD),)
>  ROOTFS_SQUASHFS_ARGS += -nopad
>  endif
>  
> +ifneq ($(BR2_TARGET_ROOTFS_SQUASHFS4_BLOCKSIZE),"")
> +ROOTFS_SQUASHFS_ARGS += -b "$(BR2_TARGET_ROOTFS_SQUASHFS4_BLOCKSIZE)"
> +endif
> +
>  ifeq ($(BR2_TARGET_ROOTFS_SQUASHFS4_LZ4),y)
>  ROOTFS_SQUASHFS_ARGS += -comp lz4 -Xhc
>  else ifeq ($(BR2_TARGET_ROOTFS_SQUASHFS4_LZO),y)
> @@ -26,6 +30,10 @@ else
>  ROOTFS_SQUASHFS_ARGS += -comp gzip
>  endif
>  
> +ifneq ($(BR2_TARGET_ROOTFS_SQUASHFS4_CUSTOM_ARGS),"")
> +ROOTFS_SQUASHFS_ARGS += $(call qstrip,$(BR2_TARGET_ROOTFS_SQUASHFS4_CUSTOM_ARGS))
> +endif
> +
>  define ROOTFS_SQUASHFS_CMD
>  	$(HOST_DIR)/bin/mksquashfs $(TARGET_DIR) $@ $(ROOTFS_SQUASHFS_ARGS)
>  endef
> -- 
> 2.35.1
> 
> _______________________________________________
> buildroot mailing list
> buildroot@buildroot.org
> https://lists.buildroot.org/mailman/listinfo/buildroot

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/1] fs/squashfs: add more options to customize creation

[Linus via buildroot]

[-- Attachment #1.1.1: Type: text/plain, Size: 1924 bytes --]

Hi Yann, All,

> Two changes, two patches. ;-)
No problem. Thought the two changes would be fine as one.
Will split them.

## Blocksize

> So, I think this is good to have those additional options, however I
> think we should restrict them. For example, arbitrary string options is
> not very nice.
>
> Instead, here's what I suggest:
> [...]
I have already prepared the patch for blocksize, using choices
from 4k, 8k, ..., 512k, 1024k. That one should be pretty straight
forward.

I also find 1k to be pretty ridiculous. Not even sure if it would
work. This size seems to be pretty old and also only start at 4k:
https://tldp.org/HOWTO/SquashFS-HOWTO/mksqoverview.html

I've used the same style that was used for specifiying the compression.
Not sure whether the later string without a name is hidden, but I've
so far never encountered that. I can add this of course if this has
some benefit (maybe being able to specify a specific size in raw
config values??).

## Compression

For the compression arguments, I'm up to anything here as well.
I'm not sure if we should use all extreme options by default

> Also note that lz4 already uses -Xhc unconditionally, and commit
>      07e37bcc42f notes that an extra option was dropped, so maybe we
>      should just unconditionally enable all those options when possible?

Seems like a easy way to confuse people or cause errors should
a option somehow make problems. Or is it a major problem to break
the assumption that only lz4 is already optimized?

>    - for the comnpression options, I think a generic boolean would be
>      better:
I agree with that. Will try to get this done tomorrow and do a couple
of tests to ensure there are no obvious problems.

## In conclusion

If that's okay with you I can prepare a patch-set tomorrow and
send them in for review (hopefully also tomorrow).

Greetings,
Linus Kaschulla


[-- Attachment #1.2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

[-- Attachment #2: Type: text/plain, Size: 150 bytes --]

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/1] fs/squashfs: add more options to customize creation

[Yann E. MORIN]

Linus, Al,

On 2022-05-16 01:56 +0200, Linus spake thusly:
> I have already prepared the patch for blocksize, using choices
> from 4k, 8k, ..., 512k, 1024k. That one should be pretty straight
> forward.
> 
> I also find 1k to be pretty ridiculous. Not even sure if it would
> work. This size seems to be pretty old and also only start at 4k:
> https://tldp.org/HOWTO/SquashFS-HOWTO/mksqoverview.html

Yes, 1K and 2K are not suported, but that page is about mksquashfs 3.x,
which supported from 4K to 128K, but we are now packagng 4.5, which
supports from 4K to 1M.

> I've used the same style that was used for specifiying the compression.
> Not sure whether the later string without a name is hidden,

Yes, it should be hidden, i.e. have no prompt, because the actual user
input is the entry in the choice; the hidden string is just a
translation from a multi-boolean to a the actual size. We have such
constructs in a few opther places. For example, see the version choice
for the kernel version;

      129 config BR2_LINUX_KERNEL_VERSION
      130     string
      131     default "5.17.7" if BR2_LINUX_KERNEL_LATEST_VERSION
      132     default "5.10.104-cip3" if BR2_LINUX_KERNEL_LATEST_CIP_VERSION
      133     default "5.10.104-cip3-rt3" if BR2_LINUX_KERNEL_LATEST_CIP_RT_VERSION
      134 ...

> but I've
> so far never encountered that. I can add this of course if this has
> some benefit (maybe being able to specify a specific size in raw
> config values??).

Nope, the choice is the only way to set it.

> ## Compression
> 
> For the compression arguments, I'm up to anything here as well.
> I'm not sure if we should use all extreme options by default
> 
> >Also note that lz4 already uses -Xhc unconditionally, and commit
> >     07e37bcc42f notes that an extra option was dropped, so maybe we
> >     should just unconditionally enable all those options when possible?
> Seems like a easy way to confuse people or cause errors should
> a option somehow make problems.

In that case, people would not enable this option if it is causing issue
for them.

> Or is it a major problem to break
> the assumption that only lz4 is already optimized?

Well, to keep the existing behaviour we could do something like:

    config BR2_TARGET_ROOTFS_EXTREME_COMP
        bool "extreme compression if possible"
        default y if BR2_TARGET_ROOTFS_SQUASHFS4_LZH

> >   - for the comnpression options, I think a generic boolean would be
> >     better:
> I agree with that. Will try to get this done tomorrow and do a couple
> of tests to ensure there are no obvious problems.
> 
> ## In conclusion
> 
> If that's okay with you I can prepare a patch-set tomorrow and
> send them in for review (hopefully also tomorrow).

Yes, no worries, as you can make it. Thanks!

Regards,
Yann E. MORIN.


-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/1] fs/squashfs: add more options to customize creation

[Linus via buildroot]

Hi Yann, All,

> Yes, 1K and 2K are not suported, but that page is about mksquashfs 3.x,
> which supported from 4K to 128K, but we are now packagng 4.5, which
> supports from 4K to 1M.
Good to know! Didn't know the details on this and just guessed lucky.

> Yes, it should be hidden, i.e. have no prompt, because the actual user
> input is the entry in the choice; [...]

Cool. Wasn't aware of that. Seems the options appear in the .config,
but are not exported to a defconfig.

>> but I've
>> so far never encountered that. I can add this of course if this has
>> some benefit (maybe being able to specify a specific size in raw
>> config values??).
> Nope, the choice is the only way to set it.
Technically, patching the generated .config would be a way As I found
out above. But I agree that it is not changeable if one does everything
correctly.

## Patches

I've done the patches now. I used the "config default" thing for
both options. I was a bit confused about some envs using
SQUASH4 and some only SQUASH. I'm assuming that the "4" is some
baggage and left it out of the patches. Feel free to correct me
if I assume incorrectly here.

Regarding some compression options, seems that `xz --long-help`
explicitly states that big endian is not supported for arm.
While x86 explicitly states 64-bit support I'm not sure whether
arm (and some similar architectures) are supposed to work for
64 bit as well. For now I'm assuming it. Worst case it'll not
do any savings there I guess.

Looking forward for feedback on the new patches,
Linus Kaschulla

Am 16.05.22 um 18:47 schrieb Yann E. MORIN:
> Linus, Al,
>
> On 2022-05-16 01:56 +0200, Linus spake thusly:
>> I have already prepared the patch for blocksize, using choices
>> from 4k, 8k, ..., 512k, 1024k. That one should be pretty straight
>> forward.
>>
>> I also find 1k to be pretty ridiculous. Not even sure if it would
>> work. This size seems to be pretty old and also only start at 4k:
>> https://tldp.org/HOWTO/SquashFS-HOWTO/mksqoverview.html
> Yes, 1K and 2K are not suported, but that page is about mksquashfs 3.x,
> which supported from 4K to 128K, but we are now packagng 4.5, which
> supports from 4K to 1M.
>
>> I've used the same style that was used for specifiying the compression.
>> Not sure whether the later string without a name is hidden,
> Yes, it should be hidden, i.e. have no prompt, because the actual user
> input is the entry in the choice; the hidden string is just a
> translation from a multi-boolean to a the actual size. We have such
> constructs in a few opther places. For example, see the version choice
> for the kernel version;
>
>        129 config BR2_LINUX_KERNEL_VERSION
>        130     string
>        131     default "5.17.7" if BR2_LINUX_KERNEL_LATEST_VERSION
>        132     default "5.10.104-cip3" if BR2_LINUX_KERNEL_LATEST_CIP_VERSION
>        133     default "5.10.104-cip3-rt3" if BR2_LINUX_KERNEL_LATEST_CIP_RT_VERSION
>        134 ...
>
>> but I've
>> so far never encountered that. I can add this of course if this has
>> some benefit (maybe being able to specify a specific size in raw
>> config values??).
> Nope, the choice is the only way to set it.
>
>> ## Compression
>>
>> For the compression arguments, I'm up to anything here as well.
>> I'm not sure if we should use all extreme options by default
>>
>>> Also note that lz4 already uses -Xhc unconditionally, and commit
>>>      07e37bcc42f notes that an extra option was dropped, so maybe we
>>>      should just unconditionally enable all those options when possible?
>> Seems like a easy way to confuse people or cause errors should
>> a option somehow make problems.
> In that case, people would not enable this option if it is causing issue
> for them.
>
>> Or is it a major problem to break
>> the assumption that only lz4 is already optimized?
> Well, to keep the existing behaviour we could do something like:
>
>      config BR2_TARGET_ROOTFS_EXTREME_COMP
>          bool "extreme compression if possible"
>          default y if BR2_TARGET_ROOTFS_SQUASHFS4_LZH
>
>>>    - for the comnpression options, I think a generic boolean would be
>>>      better:
>> I agree with that. Will try to get this done tomorrow and do a couple
>> of tests to ensure there are no obvious problems.
>>
>> ## In conclusion
>>
>> If that's okay with you I can prepare a patch-set tomorrow and
>> send them in for review (hopefully also tomorrow).
> Yes, no worries, as you can make it. Thanks!
>
> Regards,
> Yann E. MORIN.
>
>
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot