* [PATCH v4 1/2] zstd: move recipe to oe-core from meta-oe
@ 2019-11-20 9:12 Stefan Agner
2019-11-20 9:12 ` [PATCH v4 2/2] image_types: add Zstandard conversion support Stefan Agner
0 siblings, 1 reply; 6+ messages in thread
From: Stefan Agner @ 2019-11-20 9:12 UTC (permalink / raw)
To: openembedded-core; +Cc: Stefan Agner
From: Stefan Agner <stefan.agner@toradex.com>
Zstd is a very fast compression algorithm and has gained quite some
popularity recently. An upcoming patch allows to compress rootfs
using Zstd. Moving Zstd to oe-core allows this new conversion formats
to be automatically tested by the oe-selftest
imagefeatures.ImageFeatures.test_image_fstypes test.
Furthermore there are three packages in oe-core which allow to enable
Zstd support through packageconfig currently.
Signed-off-by: Stefan Agner <stefan.agner@toradex.com>
---
meta/conf/distro/include/maintainers.inc | 1 +
meta/recipes-extended/zstd/zstd_1.4.4.bb | 35 ++++++++++++++++++++++++
2 files changed, 36 insertions(+)
create mode 100644 meta/recipes-extended/zstd/zstd_1.4.4.bb
diff --git a/meta/conf/distro/include/maintainers.inc b/meta/conf/distro/include/maintainers.inc
index d85e5b697f..dff7ce28f8 100644
--- a/meta/conf/distro/include/maintainers.inc
+++ b/meta/conf/distro/include/maintainers.inc
@@ -774,3 +774,4 @@ RECIPE_MAINTAINER_pn-xwininfo = "Armin Kuster <akuster808@gmail.com>"
RECIPE_MAINTAINER_pn-xz = "Denys Dmytriyenko <denys@ti.com>"
RECIPE_MAINTAINER_pn-zip = "Denys Dmytriyenko <denys@ti.com>"
RECIPE_MAINTAINER_pn-zlib = "Denys Dmytriyenko <denys@ti.com>"
+RECIPE_MAINTAINER_pn-zstd = "Alex Kiernan <alex.kiernan@gmail.com>"
diff --git a/meta/recipes-extended/zstd/zstd_1.4.4.bb b/meta/recipes-extended/zstd/zstd_1.4.4.bb
new file mode 100644
index 0000000000..eb201f4139
--- /dev/null
+++ b/meta/recipes-extended/zstd/zstd_1.4.4.bb
@@ -0,0 +1,35 @@
+SUMMARY = "Zstandard - Fast real-time compression algorithm"
+DESCRIPTION = "Zstandard is a fast lossless compression algorithm, targeting \
+real-time compression scenarios at zlib-level and better compression ratios. \
+It's backed by a very fast entropy stage, provided by Huff0 and FSE library."
+HOMEPAGE = "http://www.zstd.net/"
+SECTION = "console/utils"
+
+LICENSE = "BSD-3-Clause & GPLv2"
+LIC_FILES_CHKSUM = "file://LICENSE;md5=c7f0b161edbe52f5f345a3d1311d0b32 \
+ file://COPYING;md5=39bba7d2cf0ba1036f2a6e2be52fe3f0"
+
+SRC_URI = "git://github.com/facebook/zstd.git;nobranch=1"
+
+SRCREV = "10f0e6993f9d2f682da6d04aa2385b7d53cbb4ee"
+UPSTREAM_CHECK_GITTAGREGEX = "v(?P<pver>\d+(\.\d+)+)"
+
+S = "${WORKDIR}/git"
+
+PACKAGECONFIG ??= ""
+PACKAGECONFIG[lz4] = "HAVE_LZ4=1,HAVE_LZ4=0,lz4"
+PACKAGECONFIG[lzma] = "HAVE_LZMA=1,HAVE_LZMA=0,xz"
+PACKAGECONFIG[zlib] = "HAVE_ZLIB=1,HAVE_ZLIB=0,zlib"
+
+# See programs/README.md for how to use this
+ZSTD_LEGACY_SUPPORT ??= "4"
+
+do_compile () {
+ oe_runmake ${PACKAGECONFIG_CONFARGS} ZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_SUPPORT}
+}
+
+do_install () {
+ oe_runmake install 'DESTDIR=${D}'
+}
+
+BBCLASSEXTEND = "native nativesdk"
--
2.17.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PATCH v4 2/2] image_types: add Zstandard conversion support
2019-11-20 9:12 [PATCH v4 1/2] zstd: move recipe to oe-core from meta-oe Stefan Agner
@ 2019-11-20 9:12 ` Stefan Agner
2019-11-20 10:31 ` Adrian Bunk
2019-11-20 10:49 ` Alex Kiernan
0 siblings, 2 replies; 6+ messages in thread
From: Stefan Agner @ 2019-11-20 9:12 UTC (permalink / raw)
To: openembedded-core; +Cc: Stefan Agner
From: Stefan Agner <stefan.agner@toradex.com>
Add Zstandard (or just Zstd) compression support. This allows to
create Zstd compressed tarballs by using tar.zst as IMAGE_FSTYPES.
Signed-off-by: Stefan Agner <stefan.agner@toradex.com>
---
meta/classes/image_types.bbclass | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index 2eeffbb366..d29b9c5787 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32"
ZIP_COMPRESSION_LEVEL ?= "-9"
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+
JFFS2_SUM_EXTRA_ARGS ?= ""
IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}"
@@ -269,7 +271,7 @@ IMAGE_TYPES = " \
hddimg \
squashfs squashfs-xz squashfs-lzo squashfs-lz4 \
ubi ubifs multiubi \
- tar tar.gz tar.bz2 tar.xz tar.lz4 \
+ tar tar.gz tar.bz2 tar.xz tar.lz4 tar.zst \
cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \
wic wic.gz wic.bz2 wic.lzma \
container \
@@ -282,7 +284,7 @@ IMAGE_TYPES = " \
# CONVERSION_CMD/DEPENDS.
COMPRESSIONTYPES ?= ""
-CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
+CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip zst sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
CONVERSION_CMD_gz = "gzip -f -9 -n -c --rsyncable ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
@@ -290,6 +292,7 @@ CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=
CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4"
CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
+CONVERSION_CMD_zst = "zstd -f -k -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zst"
CONVERSION_CMD_sum = "sumtool -i ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
CONVERSION_CMD_md5sum = "md5sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.md5sum"
CONVERSION_CMD_sha1sum = "sha1sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha1sum"
@@ -310,6 +313,7 @@ CONVERSION_DEPENDS_xz = "xz-native"
CONVERSION_DEPENDS_lz4 = "lz4-native"
CONVERSION_DEPENDS_lzo = "lzop-native"
CONVERSION_DEPENDS_zip = "zip-native"
+CONVERSION_DEPENDS_zst = "zstd-native"
CONVERSION_DEPENDS_sum = "mtd-utils-native"
CONVERSION_DEPENDS_bmap = "bmap-tools-native"
CONVERSION_DEPENDS_u-boot = "u-boot-tools-native"
--
2.17.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
2019-11-20 9:12 ` [PATCH v4 2/2] image_types: add Zstandard conversion support Stefan Agner
@ 2019-11-20 10:31 ` Adrian Bunk
2019-11-20 10:48 ` Alex Kiernan
2019-11-20 10:49 ` Alex Kiernan
1 sibling, 1 reply; 6+ messages in thread
From: Adrian Bunk @ 2019-11-20 10:31 UTC (permalink / raw)
To: Stefan Agner; +Cc: openembedded-core
On Wed, Nov 20, 2019 at 09:12:58AM +0000, Stefan Agner wrote:
>...
> --- a/meta/classes/image_types.bbclass
> +++ b/meta/classes/image_types.bbclass
> @@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32"
>
> ZIP_COMPRESSION_LEVEL ?= "-9"
>
> +ZSTD_COMPRESSION_LEVEL ?= "-3"
> +
>...
Why -3 and not -19 ?
Compression speed is not a real issue, and AFAIR decompression
tends to be slightly faster with smaller compressed size.
Plus the benefit that smaller images are faster to load
from network/storage.
cu
Adrian
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
2019-11-20 10:31 ` Adrian Bunk
@ 2019-11-20 10:48 ` Alex Kiernan
0 siblings, 0 replies; 6+ messages in thread
From: Alex Kiernan @ 2019-11-20 10:48 UTC (permalink / raw)
To: Adrian Bunk; +Cc: Patches and discussions about the oe-core layer
On Wed, Nov 20, 2019 at 10:32 AM Adrian Bunk <bunk@stusta.de> wrote:
>
> On Wed, Nov 20, 2019 at 09:12:58AM +0000, Stefan Agner wrote:
> >...
> > --- a/meta/classes/image_types.bbclass
> > +++ b/meta/classes/image_types.bbclass
> > @@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32"
> >
> > ZIP_COMPRESSION_LEVEL ?= "-9"
> >
> > +ZSTD_COMPRESSION_LEVEL ?= "-3"
> > +
> >...
>
> Why -3 and not -19 ?
>
> Compression speed is not a real issue, and AFAIR decompression
> tends to be slightly faster with smaller compressed size.
> Plus the benefit that smaller images are faster to load
> from network/storage.
>
I've been carrying similar functionality locally and at some point
pushed to `--ultra -22` and with a large image (~2GiB) the compression
stage then took literally hours. Not sure I tested with `-19` though
--
Alex Kiernan
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
2019-11-20 9:12 ` [PATCH v4 2/2] image_types: add Zstandard conversion support Stefan Agner
2019-11-20 10:31 ` Adrian Bunk
@ 2019-11-20 10:49 ` Alex Kiernan
2019-11-20 12:27 ` Stefan Agner
1 sibling, 1 reply; 6+ messages in thread
From: Alex Kiernan @ 2019-11-20 10:49 UTC (permalink / raw)
To: Stefan Agner
Cc: Stefan Agner, Patches and discussions about the oe-core layer
On Wed, Nov 20, 2019 at 9:13 AM Stefan Agner <stefan@agner.ch> wrote:
>
> From: Stefan Agner <stefan.agner@toradex.com>
>
> Add Zstandard (or just Zstd) compression support. This allows to
> create Zstd compressed tarballs by using tar.zst as IMAGE_FSTYPES.
>
> Signed-off-by: Stefan Agner <stefan.agner@toradex.com>
> ---
> meta/classes/image_types.bbclass | 8 ++++++--
> 1 file changed, 6 insertions(+), 2 deletions(-)
>
> diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
> index 2eeffbb366..d29b9c5787 100644
> --- a/meta/classes/image_types.bbclass
> +++ b/meta/classes/image_types.bbclass
> @@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32"
>
> ZIP_COMPRESSION_LEVEL ?= "-9"
>
> +ZSTD_COMPRESSION_LEVEL ?= "-3"
> +
> JFFS2_SUM_EXTRA_ARGS ?= ""
> IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}"
>
> @@ -269,7 +271,7 @@ IMAGE_TYPES = " \
> hddimg \
> squashfs squashfs-xz squashfs-lzo squashfs-lz4 \
> ubi ubifs multiubi \
> - tar tar.gz tar.bz2 tar.xz tar.lz4 \
> + tar tar.gz tar.bz2 tar.xz tar.lz4 tar.zst \
> cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \
> wic wic.gz wic.bz2 wic.lzma \
> container \
> @@ -282,7 +284,7 @@ IMAGE_TYPES = " \
> # CONVERSION_CMD/DEPENDS.
> COMPRESSIONTYPES ?= ""
>
> -CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
> +CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip zst sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
> CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
> CONVERSION_CMD_gz = "gzip -f -9 -n -c --rsyncable ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
> CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
> @@ -290,6 +292,7 @@ CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=
> CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4"
> CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
> CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
> +CONVERSION_CMD_zst = "zstd -f -k -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zst"
Add -T0 so we default to parallel?
--
Alex Kiernan
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH v4 2/2] image_types: add Zstandard conversion support
2019-11-20 10:49 ` Alex Kiernan
@ 2019-11-20 12:27 ` Stefan Agner
0 siblings, 0 replies; 6+ messages in thread
From: Stefan Agner @ 2019-11-20 12:27 UTC (permalink / raw)
To: Alex Kiernan
Cc: Stefan Agner, Patches and discussions about the oe-core layer
On 2019-11-20 11:49, Alex Kiernan wrote:
> On Wed, Nov 20, 2019 at 9:13 AM Stefan Agner <stefan@agner.ch> wrote:
>>
>> From: Stefan Agner <stefan.agner@toradex.com>
>>
>> Add Zstandard (or just Zstd) compression support. This allows to
>> create Zstd compressed tarballs by using tar.zst as IMAGE_FSTYPES.
>>
>> Signed-off-by: Stefan Agner <stefan.agner@toradex.com>
>> ---
>> meta/classes/image_types.bbclass | 8 ++++++--
>> 1 file changed, 6 insertions(+), 2 deletions(-)
>>
>> diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
>> index 2eeffbb366..d29b9c5787 100644
>> --- a/meta/classes/image_types.bbclass
>> +++ b/meta/classes/image_types.bbclass
>> @@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32"
>>
>> ZIP_COMPRESSION_LEVEL ?= "-9"
>>
>> +ZSTD_COMPRESSION_LEVEL ?= "-3"
>> +
>> JFFS2_SUM_EXTRA_ARGS ?= ""
>> IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}"
>>
>> @@ -269,7 +271,7 @@ IMAGE_TYPES = " \
>> hddimg \
>> squashfs squashfs-xz squashfs-lzo squashfs-lz4 \
>> ubi ubifs multiubi \
>> - tar tar.gz tar.bz2 tar.xz tar.lz4 \
>> + tar tar.gz tar.bz2 tar.xz tar.lz4 tar.zst \
>> cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \
>> wic wic.gz wic.bz2 wic.lzma \
>> container \
>> @@ -282,7 +284,7 @@ IMAGE_TYPES = " \
>> # CONVERSION_CMD/DEPENDS.
>> COMPRESSIONTYPES ?= ""
>>
>> -CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
>> +CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip zst sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
>> CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
>> CONVERSION_CMD_gz = "gzip -f -9 -n -c --rsyncable ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
>> CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
>> @@ -290,6 +292,7 @@ CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=
>> CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4"
>> CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
>> CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
>> +CONVERSION_CMD_zst = "zstd -f -k -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zst"
>
> Add -T0 so we default to parallel?
Sensible, will send a v5.
--
Stefan
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2019-11-20 12:27 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-11-20 9:12 [PATCH v4 1/2] zstd: move recipe to oe-core from meta-oe Stefan Agner
2019-11-20 9:12 ` [PATCH v4 2/2] image_types: add Zstandard conversion support Stefan Agner
2019-11-20 10:31 ` Adrian Bunk
2019-11-20 10:48 ` Alex Kiernan
2019-11-20 10:49 ` Alex Kiernan
2019-11-20 12:27 ` Stefan Agner
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.