Hi Alex,
I cannot see abelloni/kirkstone-next here: http://git.yoctoproject.org/cgit/cgit.cgi/poky/refs/heads
Do I need to get them somewhere else?
Kind regards, Henry
From: poky@lists.yoctoproject.org <poky@lists.yoctoproject.org> on behalf of Alexander Kanavin <alex.kanavin@gmail.com>
Sent: 04 October 2021 12:10
To: Henry Kleynhans <hkleynhans@fb.com>
Cc: OE-core <openembedded-core@lists.openembedded.org>; poky@lists.yoctoproject.org <poky@lists.yoctoproject.org>; Michael van der Westhuizen <rmikey@fb.com>
Subject: Re: [poky] [PATCH] sstate: Switch to ZStandard compressor supportZSTD_THREADS is already defined in bitbake.conf, you do not need to redefine it.(please use abelloni/kirkstone-next for testing, it is there)
Alex
On Mon, 4 Oct 2021 at 13:00, Henry Kleynhans via lists.yoctoproject.org <hkleynhans=fb.com@lists.yoctoproject.org> wrote:
This patch switches the compressor from Gzip to ZStandard for ssate cache
files.
Zstandard compression provides a significant improvement in
decompression speed as well as improvement in compression speed and disk
usage over the 'tgz' format in use. Furthermore, its configurable
compression level offers a trade-off between time spent compressing
sstate cache files and disk space used by those files. The reduced disk
usage also contributes to saving network traffic for those sharing their
sstate cache with others.
Zstandard should therefore be a good choice when:
* disk space is at a premium
* network speed / resources are limited
* the CI server can sstate packages can be created at high compression
* less CPU on the build server should be used for sstate decompression
Signed-off-by: Henry Kleynhans <hkleynhans@fb.com>
---
meta/classes/sstate.bbclass | 30 ++++++++++++++--------
meta/conf/bitbake.conf | 2 +-
scripts/sstate-cache-management.sh | 40 +++++++++++++++---------------
3 files changed, 41 insertions(+), 31 deletions(-)
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 92a73114bb..9d40c8ebe4 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1,17 +1,20 @@
SSTATE_VERSION = "3"
+ZSTD_THREADS ?= "${BB_NUMBER_THREADS}"
+SSTATE_ZSTD_CLEVEL = "8"
+
SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
def generate_sstatefn(spec, hash, taskname, siginfo, d):
if taskname is None:
return ""
- extension = ".tgz"
+ extension = ".tar.zst"
# 8 chars reserved for siginfo
limit = 254 - 8
if siginfo:
limit = 254
- extension = ".tgz.siginfo"
+ extension = ".tar.zst.siginfo"
if not hash:
hash = "INVALID"
fn = spec + hash + "_" + taskname + extension
@@ -37,7 +40,7 @@ SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PK
SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
SSTATE_EXTRAPATH = ""
SSTATE_EXTRAPATHWILDCARD = ""
-SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
+SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*"
# explicitly make PV to depend on evaluated value of PV variable
PV[vardepvalue] = "${PV}"
@@ -825,23 +828,24 @@ sstate_create_package () {
mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
- # Use pigz if available
- OPT="-czS"
- if [ -x "$(command -v pigz)" ]; then
- OPT="-I pigz -cS"
+ OPT="-cS"
+ ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
+ # Use pzstd if available
+ if [ -x "$(command -v pzstd)" ]; then
+ ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}"
fi
# Need to handle empty directories
if [ "$(ls -A)" ]; then
set +e
- tar $OPT -f $TFILE *
+ tar -I "$ZSTD" $OPT -f $TFILE *
ret=$?
if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
exit 1
fi
set -e
else
- tar $OPT --file=$TFILE --files-from=/dev/null
+ tar -I "$ZSTD" $OPT --file=$TFILE --files-from=/dev/null
fi
chmod 0664 $TFILE
# Skip if it was already created by some other process
@@ -880,7 +884,13 @@ python sstate_report_unihash() {
# Will be run from within SSTATE_INSTDIR.
#
sstate_unpack_package () {
- tar -xvzf ${SSTATE_PKG}
+ ZSTD="zstd -T${ZSTD_THREADS}"
+ # Use pzstd if available
+ if [ -x "$(command -v pzstd)" ]; then
+ ZSTD="pzstd -p ${ZSTD_THREADS}"
+ fi
+
+ tar -I "$ZSTD" -xvf ${SSTATE_PKG}
# update .siginfo atime on local/NFS mirror
[ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
# Use "! -w ||" to return true for read only files
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index e25d632dc1..ed746bdf99 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -498,7 +498,7 @@ HOSTTOOLS += " \
HOSTTOOLS += "${@'ip ping ps scp ssh stty' if (bb.utils.contains_any('IMAGE_CLASSES', 'testimage testsdk', True, False, d) or any(x in (d.getVar("BBINCLUDED") or "") for x in ["testimage.bbclass", "testsdk.bbclass"])) else ''}"
# Link to these if present
-HOSTTOOLS_NONFATAL += "aws gcc-ar gpg ld.bfd ld.gold nc pigz sftp socat ssh sudo"
+HOSTTOOLS_NONFATAL += "aws gcc-ar gpg ld.bfd ld.gold nc pigz pzstd sftp socat ssh sudo"
# Temporary add few more detected in bitbake world
HOSTTOOLS_NONFATAL += "join nl size yes zcat"
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index f1706a2229..d39671f7c6 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -114,7 +114,7 @@ echo_error () {
# * Add .done/.siginfo to the remove list
# * Add destination of symlink to the remove list
#
-# $1: output file, others: sstate cache file (.tgz)
+# $1: output file, others: sstate cache file (.tar.zst)
gen_rmlist (){
local rmlist_file="$1"
shift
@@ -131,13 +131,13 @@ gen_rmlist (){
dest="`readlink -e $i`"
if [ -n "$dest" ]; then
echo $dest >> $rmlist_file
- # Remove the .siginfo when .tgz is removed
+ # Remove the .siginfo when .tar.zst is removed
if [ -f "$dest.siginfo" ]; then
echo $dest.siginfo >> $rmlist_file
fi
fi
fi
- # Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
+ # Add the ".tar.zst.done" and ".siginfo.done" (may exist in the future)
base_fn="${i##/*/}"
t_fn="$base_fn.done"
s_fn="$base_fn.siginfo.done"
@@ -188,10 +188,10 @@ remove_duplicated () {
total_files=`find $cache_dir -name 'sstate*' | wc -l`
# Save all the sstate files in a file
sstate_files_list=`mktemp` || exit 1
- find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list
+ find $cache_dir -iname 'sstate:*:*:*:*:*:*:*.tar.zst*' >$sstate_files_list
echo "Figuring out the suffixes in the sstate cache dir ... "
- sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`"
+ sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tar\.zst.*%\1%g' $sstate_files_list | sort -u`"
echo "Done"
echo "The following suffixes have been found in the cache dir:"
echo $sstate_suffixes
@@ -200,10 +200,10 @@ remove_duplicated () {
# Using this SSTATE_PKGSPEC definition it's 6th colon separated field
# SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
for arch in $all_archs; do
- grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
+ grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tar\.zst$" $sstate_files_list
[ $? -eq 0 ] && ava_archs="$ava_archs $arch"
# ${builder_arch}_$arch used by toolchain sstate
- grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
+ grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tar\.zst$" $sstate_files_list
[ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
done
echo "Done"
@@ -219,13 +219,13 @@ remove_duplicated () {
continue
fi
# Total number of files including .siginfo and .done files
- total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null`
- total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null`
+ total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst.*" $sstate_files_list | wc -l 2>/dev/null`
+ total_archive_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst$" $sstate_files_list | wc -l 2>/dev/null`
# Save the file list to a file, some suffix's file may not exist
- grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null
- local deleted_tgz=0
+ grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst.*" $sstate_files_list >$list_suffix 2>/dev/null
+ local deleted_archives=0
local deleted_files=0
- for ext in tgz tgz.siginfo tgz.done; do
+ for ext in tar.zst tar.zst.siginfo tar.zst.done; do
echo "Figuring out the sstate:xxx_$suffix.$ext ... "
# Uniq BPNs
file_names=`for arch in $ava_archs ""; do
@@ -268,19 +268,19 @@ remove_duplicated () {
done
done
done
- deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l`
+ deleted_archives=`cat $rm_list.* 2>/dev/null | grep "\.tar\.zst$" | wc -l`
deleted_files=`cat $rm_list.* 2>/dev/null | wc -l`
[ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.*
- echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
+ echo "($deleted_archives out of $total_archives_suffix .tar.zst files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
let total_deleted=$total_deleted+$deleted_files
done
- deleted_tgz=0
+ deleted_archives=0
rm_old_list=$remove_listdir/sstate-old-filenames
- find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
- [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l`
+ find $cache_dir -name 'sstate-*.tar.zst' >$rm_old_list
+ [ -s "$rm_old_list" ] && deleted_archives=`cat $rm_old_list | grep "\.tar\.zst$" | wc -l`
[ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l`
[ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
- echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
+ echo "($deleted_archives or .tar.zst files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
let total_deleted=$total_deleted+$deleted_files
rm -f $list_suffix
@@ -289,7 +289,7 @@ remove_duplicated () {
read_confirm
if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
for list in `ls $remove_listdir/`; do
- echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
+ echo "Removing $list.tar.zst archive (`cat $remove_listdir/$list | wc -w` files) ... "
# Remove them one by one to avoid the argument list too long error
for i in `cat $remove_listdir/$list`; do
rm -f $verbose $i
@@ -322,7 +322,7 @@ rm_by_stamps (){
find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list
echo "Figuring out the suffixes in the sstate cache dir ... "
- local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`"
+ local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tar\.zst.*%\1%g' $cache_list | sort -u`"
echo "Done"
echo "The following suffixes have been found in the cache dir:"
echo $sstate_suffixes
--
2.30.2