[OE-Core][PATCH v13 1/8] path.py: add support for ACLs and all additional attributes

[OE-Core][PATCH v13 1/8] path.py: add support for ACLs and all additional attributes

[Piotr Łobacz]

From: Piotr Łobacz via lists.openembedded.org <p.lobacz=welotec.com@lists.openembedded.org>

Extend `tar` command inside copytree() and copyhardlinktree() functions,
with additional parameters, in order to support ACLs and xattr. The posix
format doesn't need to be set in here, as GNU tar switches to it whenewer
is uses --acls and/or --xattrs parameters.

Additionaly change preservation of additional attributes for `cp` command
to all (meaning mode, ownership, timestamps etc.) not only xattrs, which
were previously.

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 meta/lib/oe/path.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
index 0dc8f172d5..56e8e214ac 100644
--- a/meta/lib/oe/path.py
+++ b/meta/lib/oe/path.py
@@ -92,7 +92,7 @@ def copytree(src, dst):
     # This way we also preserve hardlinks between files in the tree.
 
     bb.utils.mkdirhier(dst)
-    cmd = "tar --xattrs --xattrs-include='*' -cf - -S -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
+    cmd = "tar --acls --xattrs --xattrs-include='*' -cf - -S -C %s -p . | tar --acls --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
     subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 
 def copyhardlinktree(src, dst):
@@ -119,7 +119,7 @@ def copyhardlinktree(src, dst):
     if (canhard):
         # Need to copy directories only with tar first since cp will error if two 
         # writers try and create a directory at the same time
-        cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -S -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst)
+        cmd = "cd %s; find . -type d -print | tar --acls --xattrs --xattrs-include='*' -cf - -S -C %s -p --no-recursion --files-from - | tar --acls --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
         source = ''
         if os.path.isdir(src):
@@ -130,7 +130,7 @@ def copyhardlinktree(src, dst):
         else:
             source = src
             s_dir = os.getcwd()
-        cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst))
+        cmd = 'cp -afl --preserve=all %s %s' % (source, os.path.realpath(dst))
         subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT)
     else:
         copytree(src, dst)
-- 
2.34.1

[OE-Core][PATCH v13 2/8] package_ipk.bbclass: add support for ACLs and xattr

[Piotr Łobacz]

Extend OPKGBUILDCMD variable, with additional parameters, depending
on target distro features, in order to support ACLs and xattr.

With fix pushed to the opkg-devel:
https://groups.google.com/g/opkg-devel/c/dYNHrLjDwg8
opkg-build is able to create tar archives with ACLs and xattr.

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 meta/classes-global/package_ipk.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes-global/package_ipk.bbclass b/meta/classes-global/package_ipk.bbclass
index b4b7bc9ac2..a0f106e4ad 100644
--- a/meta/classes-global/package_ipk.bbclass
+++ b/meta/classes-global/package_ipk.bbclass
@@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
 PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
 
 # Program to be used to build opkg packages
-OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
+OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}" ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)} ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'
 
 OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
 OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
-- 
2.34.1

[OE-Core][PATCH v13 3/8] package.bbclass: add support for ACLs and xattr

[Piotr Łobacz]

Extend `tar` command, with additional parameters, depending
on choosen package class and target distro features, in order
to support ACLs and xattr.

Currently only `package_ipk` supports fully ACLs and xattr.

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 meta/classes-global/package.bbclass | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/meta/classes-global/package.bbclass b/meta/classes-global/package.bbclass
index e8055a9cdc..6b65641aea 100644
--- a/meta/classes-global/package.bbclass
+++ b/meta/classes-global/package.bbclass
@@ -342,8 +342,13 @@ python perform_packagecopy () {
 
     # Start by package population by taking a copy of the installed
     # files to operate on
-    # Preserve sparse files and hard links
-    cmd = 'tar --exclude=./sysroot-only -cf - -C %s -p -S . | tar -xf - -C %s' % (dest, dvar)
+    # Preserve sparse files, hard links, ACLs and extended attributes
+    # TODO: for the moment only ipk packages are supporting ACLs and extended attributes
+    # we need to add support for other package systems as well, but that doesn't bother
+    # tar from creating archives with acl and/or xattr support
+    acl = bb.utils.contains('DISTRO_FEATURES', 'acl', '--acls', '', d)
+    xattr = bb.utils.contains('DISTRO_FEATURES', 'xattr', '--xattrs', '', d)
+    cmd = f'tar --format=posix {acl} {xattr} --numeric-owner --exclude=./sysroot-only -cf - -C {dest} -p -S . | tar --format=posix {acl} {xattr} -xf - -C {dvar}'
     subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 
     # replace RPATHs for the nativesdk binaries, to make them relocatable
-- 
2.34.1

[OE-Core][PATCH v13 4/8] sstate.bbclass: add support for ACLs and xattr

[Piotr Łobacz]

Extend `tar` command, with additional parameters, depending
on choosen package class and target distro features, in order
to support ACLs and xattr.

Additionaly set archive posix format, in order to preserve
milliseconds in timestamps for reproducibility tests.

Currently only `package_ipk` supports fully ACLs and xattr.

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 meta/classes-global/sstate.bbclass | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/classes-global/sstate.bbclass b/meta/classes-global/sstate.bbclass
index 95373fd60a..ac890fc98e 100644
--- a/meta/classes-global/sstate.bbclass
+++ b/meta/classes-global/sstate.bbclass
@@ -848,7 +848,7 @@ sstate_create_package () {
 	mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
 	TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
 
-	OPT="-cS"
+	OPT="--format=posix ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '--acls', '', d)} ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '--xattrs', '', d)} --numeric-owner -cS"
 	ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
 	# Use pzstd if available
 	if [ -x "$(command -v pzstd)" ]; then
@@ -914,7 +914,7 @@ sstate_unpack_package () {
 		ZSTD="pzstd -p ${ZSTD_THREADS}"
 	fi
 
-	tar -I "$ZSTD" -xvpf ${SSTATE_PKG}
+	tar -I "$ZSTD" --format=posix ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '--acls', '', d)} ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '--xattrs', '', d)} -xvpf ${SSTATE_PKG}
 	# update .siginfo atime on local/NFS mirror if it is a symbolic link
 	[ ! -h ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch -a ${SSTATE_PKG}.siginfo 2>/dev/null || true
 	# update each symbolic link instead of any referenced file
-- 
2.34.1

[OE-Core][PATCH v13 5/8] sstatesig.py: fix hash calculation for timestamp

[Piotr Łobacz]

Passing timestamp as an integer was losing precision, which led
to problems with badly recreated sstate cache.

Passing it as floating point number fixes the issue.

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 meta/lib/oe/sstatesig.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 633a0fd450..f7ea3d366c 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -588,7 +588,7 @@ def OEOuthashBasic(path, sigfile, task, d):
                         raise Exception(msg).with_traceback(e.__traceback__)
 
                 if include_timestamps:
-                    update_hash(" %10d" % s.st_mtime)
+                    update_hash(" %f" % s.st_mtime)
 
                 update_hash(" ")
                 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
-- 
2.34.1

[OE-Core][PATCH v13 6/8] opkg-utils: add acl and xattr support

[Piotr Łobacz]

Add support for tar archives created with --acls and/or --xattrs options,
PAX header format.

GNU tar and libarchive already supports ACLs and extended attributes.
We can now add this support as well to opkg-build script in order to use
fsetattr or setcap inside do_install command and end up with a file in
an image with the relevant ACLs and xattrs.

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 ...kg-build-Add-acls-and-xattrs-support.patch | 174 ++++++++++++++++++
 .../opkg-utils/opkg-utils_0.6.2.bb            |   1 +
 2 files changed, 175 insertions(+)
 create mode 100644 meta/recipes-devtools/opkg-utils/opkg-utils/0002-opkg-build-Add-acls-and-xattrs-support.patch

diff --git a/meta/recipes-devtools/opkg-utils/opkg-utils/0002-opkg-build-Add-acls-and-xattrs-support.patch b/meta/recipes-devtools/opkg-utils/opkg-utils/0002-opkg-build-Add-acls-and-xattrs-support.patch
new file mode 100644
index 0000000000..32513d9b16
--- /dev/null
+++ b/meta/recipes-devtools/opkg-utils/opkg-utils/0002-opkg-build-Add-acls-and-xattrs-support.patch
@@ -0,0 +1,174 @@
+From 8d9953dd8d589e9b740307976cbe474e0ce292a0 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Piotr=20=C5=81obacz?= <p.lobacz@welotec.com>
+Date: Wed, 16 Aug 2023 14:59:35 +0200
+Subject: [PATCH 1/2] opkg-build: Add acls and xattrs support
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Add support for tar archives created with --acls and/or --xattrs options,
+PAX header format.
+
+GNU tar and libarchive already supports ACLs and extended attributes.
+We can now add this support as well to opkg-build script in order to use
+fsetattr or setcap inside do_install command and end up with a file in
+an image with the relevant ACLs and xattrs.
+
+Upstream-Status: Accepted [https://git.yoctoproject.org/opkg-utils/commit/?id=8d9953dd8d589e9b740307976cbe474e0ce292a0]
+
+[1] https://bugzilla.yoctoproject.org/show_bug.cgi?id=15097
+[2] https://groups.google.com/g/opkg-devel/c/aEGL7XRXfaA
+
+Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
+---
+ opkg-build | 81 +++++++++++++++++++++++++++++++++++-------------------
+ 1 file changed, 53 insertions(+), 28 deletions(-)
+
+diff --git a/opkg-build b/opkg-build
+index a9e45d4..fb0873d 100755
+--- a/opkg-build
++++ b/opkg-build
+@@ -145,6 +145,7 @@ You probably want to chown these to a system user: " >&2
+ ###
+ # opkg-build "main"
+ ###
++attributesargs=""
+ ogargs=""
+ outer=ar
+ noclean=0
+@@ -153,22 +154,6 @@ compressor=gzip
+ zipargs="-9n"
+ compressorargs=""
+ 
+-# Determine if tar supports the --format argument by checking the help output.
+-#
+-# This is needed because:
+-#    - Busybox tar doesn't support '--format'
+-#    - On some Linux distros, tar now defaults to posix format if '--format'
+-#      isn't explicitly specified
+-#    - Opkg doesn't currently support posix format archives
+-#
+-# It's easier to check for mention of the '--format' option than to detect the
+-# tar implementation and maintain a list of which support '--format'.
+-tarformat=""
+-if tar --help 2>&1 | grep -- "--format" > /dev/null;
+-then
+-    tarformat="--format=gnu"
+-fi
+-
+ compressor_ext() {
+     case $1 in
+ 	gzip|pigz)
+@@ -197,20 +182,24 @@ compressor_ext() {
+ : <<=cut
+ =head1 SYNOPSIS
+ 
+-B<opkg-build> [B<-c>] [B<-C>] [B<-Z> I<compressor>] [B<-a>] [B<-O>] [B<-o> I<owner>] [B<-g> I<group>] I<pkg_directory> [I<destination_directory>]
++B<opkg-build> [B<-A>] [B<-X>] [B<-c>] [B<-C>] [B<-Z> I<compressor>] [B<-a>] [B<-O>] [B<-o> I<owner>] [B<-g> I<group>] I<pkg_directory> [I<destination_directory>]
+ 
+ =cut
+ 
+-usage="Usage: $0 [-c] [-C] [-Z compressor] [-a compressor_args] [-O] [-o owner] [-g group] <pkg_directory> [<destination_directory>]"
+-while getopts "a:cCg:ho:vOZ:" opt; do
++usage="Usage: $0 [-A] [-X] [-c] [-C] [-Z compressor] [-a compressor_args] [-O] [-o owner] [-g group] <pkg_directory> [<destination_directory>]"
++while getopts "Aa:cCg:ho:vOXZ:" opt; do
+     case $opt in
++	A ) attributesargs="${attributesargs:+$attributesargs }--acls"
++	    ;;
++	X ) attributesargs="${attributesargs:+$attributesargs }--xattrs"
++	    ;;
+ 	o ) owner=$OPTARG
+-	    ogargs="--owner=$owner"
++	    ogargs="${ogargs:+$ogargs }--owner=$owner"
+ 	    ;;
+ 	O ) opkext=1
+ 	    ;;
+ 	g ) group=$OPTARG
+-	    ogargs="$ogargs --group=$group"
++	    ogargs="${ogargs:+$ogargs }--group=$group"
+ 	    ;;
+ 	c ) outer=tar
+ 	    ;;
+@@ -232,6 +221,32 @@ while getopts "a:cCg:ho:vOZ:" opt; do
+     esac
+ done
+ 
++# Determine if tar supports the --format argument by checking the help output.
++#
++# This is needed because:
++#    - Busybox tar doesn't support '--format'
++#    - On some Linux distros, tar now defaults to posix format if '--format'
++#      isn't explicitly specified
++#    - Opkg currently supports posix format archives, but gnu format is left
++#      here intentionally for backward compatibility
++#
++# It's easier to check for mention of the '--format' option than to detect the
++# tar implementation and maintain a list of which support '--format'.
++tarformat=""
++if tar --help 2>&1 | grep -- "--format" > /dev/null;
++then
++    # For ACLs or xattr support, gnu format will not work
++    # we need to set posix format instead
++    if [ ! -z "$attributesargs" ] ; then
++	    tarformat="--format=posix"
++    else
++	    tarformat="--format=gnu"
++    fi
++elif [ ! -z "$attributesargs" ] ; then
++	echo "*** Error: Attributes: $attributesargs, don't work, without posix format, which is not supported by host's tar command." >&2
++	exit 1
++fi
++
+ cext=$(compressor_ext $compressor)
+ 
+ # pgzip requires -T to avoid timestamps on the gzip archive
+@@ -301,21 +316,31 @@ fi
+ tmp_dir=$dest_dir/IPKG_BUILD.$$
+ mkdir $tmp_dir
+ 
+-build_date="${SOURCE_DATE_EPOCH:-$(date +%s)}"
+-
+-mtime_args=""
++mtime_args="--mtime=@${SOURCE_DATE_EPOCH:-$(date +%s)}"
+ # --clamp-mtime requires tar > 1.28. Only use it if SOURCE_DATE_EPOCH is set, to avoid having a generic case dependency on tar > 1.28.
+ # this setting will make sure files generated at build time have consistent mtimes, for reproducible builds.
+ if [ ! -z "$SOURCE_DATE_EPOCH"  ]; then
+-    mtime_args="--mtime=@$build_date --clamp-mtime"
++    mtime_args="$mtime_args --clamp-mtime"
++fi
++
++# Notice, that if you create an archive in POSIX format (see section GNU tar and POSIX tar) and the environment variable POSIXLY_CORRECT is set,
++# then the two archives created using the same options on the same set of files will not be byte-to-byte equivalent even with the above option.
++# This is because the posix default for extended header names includes the PID of the tar process, which is different at each run. To produce
++# byte-to-byte equivalent archives in this case, either unset POSIXLY_CORRECT, or use the following option:
++#
++# --pax-option=exthdr.name=%d/PaxHeaders/%f,atime:=0,ctime:=0
++#
++# https://www.gnu.org/software/tar/manual/html_node/PAX-keywords.html
++if [[ "$tarformat" == "--format=posix" ]]; then
++    mtime_args="$mtime_args --pax-option=exthdr.name=%d/PaxHeaders/%f,atime:=0,ctime:=0"
+ fi
+ 
+ export LANG=C
+ export LC_ALL=C
+ ( cd $pkg_dir/$CONTROL && find . -type f | sort > $tmp_dir/control_list )
+ ( cd $pkg_dir && find . -path ./$CONTROL -prune -o -path . -o -print  | sort > $tmp_dir/file_list )
+-( cd $pkg_dir && tar $ogargs $tsortargs --no-recursion $mtime_args -c $tarformat -T $tmp_dir/file_list | $compressor $compressorargs > $tmp_dir/data.tar.$cext )
+-( cd $pkg_dir/$CONTROL && tar $ogargs $tsortargs --no-recursion --mtime=@$build_date -c $tarformat -T $tmp_dir/control_list | gzip $zipargs > $tmp_dir/control.tar.gz )
++( cd $pkg_dir && tar $attributesargs $ogargs $tsortargs --numeric-owner --no-recursion $mtime_args -c $tarformat -T $tmp_dir/file_list | $compressor $compressorargs > $tmp_dir/data.tar.$cext )
++( cd $pkg_dir/$CONTROL && tar $ogargs $tsortargs --no-recursion $mtime_args -c $tarformat -T $tmp_dir/control_list | gzip $zipargs > $tmp_dir/control.tar.gz )
+ rm $tmp_dir/file_list
+ rm $tmp_dir/control_list
+ 
+@@ -331,7 +356,7 @@ rm -f $pkg_file
+ if [ "$outer" = "ar" ] ; then
+   ( cd $tmp_dir && ar -crfD $pkg_file ./debian-binary ./control.tar.gz ./data.tar.$cext )
+ else
+-  ( cd $tmp_dir && tar -c $tsortargs --mtime=@$build_date $tarformat ./debian-binary ./control.tar.gz ./data.tar.$cext | gzip $zipargs > $pkg_file )
++  ( cd $tmp_dir && tar -c $tsortargs $mtime_args $tarformat ./debian-binary ./control.tar.gz ./data.tar.$cext | gzip $zipargs > $pkg_file )
+ fi
+ 
+ rm $tmp_dir/debian-binary $tmp_dir/data.tar.$cext $tmp_dir/control.tar.gz
+-- 
+2.34.1
+
diff --git a/meta/recipes-devtools/opkg-utils/opkg-utils_0.6.2.bb b/meta/recipes-devtools/opkg-utils/opkg-utils_0.6.2.bb
index eb88b9b734..d5ce2cfbe2 100644
--- a/meta/recipes-devtools/opkg-utils/opkg-utils_0.6.2.bb
+++ b/meta/recipes-devtools/opkg-utils/opkg-utils_0.6.2.bb
@@ -9,6 +9,7 @@ PROVIDES += "${@bb.utils.contains('PACKAGECONFIG', 'update-alternatives', 'virtu
 
 SRC_URI = "git://git.yoctoproject.org/opkg-utils;protocol=https;branch=master \
            file://0001-update-alternatives-correctly-match-priority.patch \
+           file://0002-opkg-build-Add-acls-and-xattrs-support.patch \
            "
 SRCREV = "67994e62dc598282830385da75ba9b1abbbda941"
 
-- 
2.34.1

[OE-Core][PATCH v13 7/8] opkg: add options to enable support for acl and xattr

[Piotr Łobacz]

The libarchive library, which is being used by opkg, supports ACLs
and xattr already.

More informations can be read at this link:
https://github.com/libarchive/libarchive/pull/691

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 ...-to-enable-support-for-acl-and-xattr.patch | 70 +++++++++++++++++++
 meta/recipes-devtools/opkg/opkg_0.6.2.bb      |  5 +-
 2 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 meta/recipes-devtools/opkg/opkg/0002-Add-options-to-enable-support-for-acl-and-xattr.patch

diff --git a/meta/recipes-devtools/opkg/opkg/0002-Add-options-to-enable-support-for-acl-and-xattr.patch b/meta/recipes-devtools/opkg/opkg/0002-Add-options-to-enable-support-for-acl-and-xattr.patch
new file mode 100644
index 0000000000..d6cb1d79fb
--- /dev/null
+++ b/meta/recipes-devtools/opkg/opkg/0002-Add-options-to-enable-support-for-acl-and-xattr.patch
@@ -0,0 +1,70 @@
+From 1c935e994bd572d9fff436f660ac1a060a434df0 Mon Sep 17 00:00:00 2001
+From: Maciej Liszewski <m.liszewski@welotec.com>
+Date: Tue, 4 Jul 2023 22:01:58 +0200
+Subject: [PATCH] Add options to enable support for acl and xattr
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+The libarchive library, which is being used by opkg, supports ACLs
+and xattr already.
+
+More informations can be read at this link:
+https://github.com/libarchive/libarchive/pull/691
+
+Upstream-Status: Accepted [https://groups.google.com/g/opkg-devel/c/aEGL7XRXfaA]
+
+[1] https://bugzilla.yoctoproject.org/show_bug.cgi?id=15097
+
+Signed-off-by: Maciej Liszewski <m.liszewski@welotec.com>
+Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
+---
+ configure.ac           | 12 ++++++++++++
+ libopkg/opkg_archive.c |  8 ++++++++
+ 2 files changed, 20 insertions(+)
+
+diff --git a/configure.ac b/configure.ac
+index 389a818..46949cd 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -158,6 +158,18 @@ return OPENSSL_VERSION_NUMBER; ],
+   AC_SUBST(OPENSSL_LIBS)
+ fi
+ 
++# check for ACL support
++AC_ARG_WITH([acl], [AS_HELP_STRING([--with-acl], [Enable ACL support])])
++if test "x$with_acl" = "xyes"; then
++  AC_DEFINE([ENABLE_ACL], [1], [Enable ACL support])
++fi
++
++# check for xattr support
++AC_ARG_WITH([xattr], [AS_HELP_STRING([--with-xattr], [Enable xattr support])])
++if test "x$with_xattr" = "xyes"; then
++  AC_DEFINE([ENABLE_XATTR], [1], [Enable xattr support])
++fi
++
+ # check for libsolv solver
+ AC_ARG_WITH(libsolv, AC_HELP_STRING([--with-libsolv], [Use libsolv solver support.
+   ]), [], [with_libsolv="no"])
+diff --git a/libopkg/opkg_archive.c b/libopkg/opkg_archive.c
+index 03a4afb..8dd902d 100644
+--- a/libopkg/opkg_archive.c
++++ b/libopkg/opkg_archive.c
+@@ -912,6 +912,14 @@ struct opkg_ar *ar_open_pkg_data_archive(const char *filename)
+     ar->extract_flags = ARCHIVE_EXTRACT_OWNER | ARCHIVE_EXTRACT_PERM |
+         ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_UNLINK | ARCHIVE_EXTRACT_NO_OVERWRITE;
+ 
++#ifdef ENABLE_ACL
++    ar->extract_flags |= ARCHIVE_EXTRACT_ACL;
++#endif
++
++#ifdef ENABLE_XATTR
++    ar->extract_flags |= ARCHIVE_EXTRACT_FFLAGS | ARCHIVE_EXTRACT_XATTR;
++#endif
++
+     if (opkg_config->ignore_uid)
+         ar->extract_flags &= ~ARCHIVE_EXTRACT_OWNER;
+ 
+-- 
+2.34.1
+
diff --git a/meta/recipes-devtools/opkg/opkg_0.6.2.bb b/meta/recipes-devtools/opkg/opkg_0.6.2.bb
index 46be137354..d7dc6ab715 100644
--- a/meta/recipes-devtools/opkg/opkg_0.6.2.bb
+++ b/meta/recipes-devtools/opkg/opkg_0.6.2.bb
@@ -15,6 +15,7 @@ PE = "1"
 SRC_URI = "http://downloads.yoctoproject.org/releases/${BPN}/${BPN}-${PV}.tar.gz \
            file://opkg.conf \
            file://0001-opkg_conf-create-opkg.lock-in-run-instead-of-var-run.patch \
+           file://0002-Add-options-to-enable-support-for-acl-and-xattr.patch \
            file://run-ptest \
            "
 
@@ -30,8 +31,10 @@ inherit autotools pkgconfig ptest
 target_localstatedir := "${localstatedir}"
 OPKGLIBDIR ??= "${target_localstatedir}/lib"
 
-PACKAGECONFIG ??= "libsolv"
+PACKAGECONFIG ??= "libsolv ${@bb.utils.filter('DISTRO_FEATURES', 'acl xattr', d)}"
 
+PACKAGECONFIG[acl] = "--with-acl,--without-acl"
+PACKAGECONFIG[xattr] = "--with-xattr,--without-xattr"
 PACKAGECONFIG[gpg] = "--enable-gpg,--disable-gpg,\
     gnupg gpgme libgpg-error,\
     ${@ "gnupg" if ("native" in d.getVar("PN")) else "gnupg-gpg"}\
-- 
2.34.1

[OE-Core][PATCH v13 8/8] opkg: set locale from system environment variables

[Piotr Łobacz]

A C program inherits its locale environment variables when it starts up.
This happens automatically. However, these variables do not automatically
control the locale used by the library functions, because ISO C says that
all programs start by default in the standard ‘C’ locale.

Fixes warnings:
Warning when reading ar archive header: Pathname can't be converted from UTF-8 to current locale. (errno=84)

Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
---
 ...le-from-system-environment-variables.patch | 48 +++++++++++++++++++
 meta/recipes-devtools/opkg/opkg_0.6.2.bb      |  1 +
 2 files changed, 49 insertions(+)
 create mode 100644 meta/recipes-devtools/opkg/opkg/0003-opkg-set-locale-from-system-environment-variables.patch

diff --git a/meta/recipes-devtools/opkg/opkg/0003-opkg-set-locale-from-system-environment-variables.patch b/meta/recipes-devtools/opkg/opkg/0003-opkg-set-locale-from-system-environment-variables.patch
new file mode 100644
index 0000000000..32a375e4de
--- /dev/null
+++ b/meta/recipes-devtools/opkg/opkg/0003-opkg-set-locale-from-system-environment-variables.patch
@@ -0,0 +1,48 @@
+From 712895b1914bf63ee4d669863bfd106814329076 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Piotr=20=C5=81obacz?= <p.lobacz@welotec.com>
+Date: Wed, 19 Jul 2023 21:26:09 +0200
+Subject: [PATCH] opkg: set locale from system environment variables
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+A C program inherits its locale environment variables when it starts up.
+This happens automatically. However, these variables do not automatically
+control the locale used by the library functions, because ISO C says that
+all programs start by default in the standard ‘C’ locale.
+
+Fixes warnings:
+Warning when reading ar archive header: Pathname can't be converted from UTF-8 to current locale. (errno=84)
+
+Upstream-Status: Accepted [https://git.yoctoproject.org/opkg/commit/?id=9e62a38a4a52974007e9ea174504c42069da1a02]
+
+[1] https://www.gnu.org/software/libc/manual/html_node/Setting-the-Locale.html
+
+Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
+---
+ src/opkg.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/opkg.c b/src/opkg.c
+index 544c58a..0c729ff 100644
+--- a/src/opkg.c
++++ b/src/opkg.c
+@@ -27,6 +27,7 @@
+ #include <stdio.h>
+ #include <getopt.h>
+ #include <stdlib.h>
++#include <locale.h>
+ 
+ #include "opkg_conf.h"
+ #include "opkg_cmd.h"
+@@ -408,6 +409,7 @@ int main(int argc, char *argv[])
+     if (opkg_conf_init())
+         goto err0;
+ 
++    setlocale(LC_ALL, "");
+     opkg_config->verbosity = NOTICE;
+ 
+     opts = args_parse(argc, argv);
+-- 
+2.34.1
+
diff --git a/meta/recipes-devtools/opkg/opkg_0.6.2.bb b/meta/recipes-devtools/opkg/opkg_0.6.2.bb
index d7dc6ab715..3b5d51d74a 100644
--- a/meta/recipes-devtools/opkg/opkg_0.6.2.bb
+++ b/meta/recipes-devtools/opkg/opkg_0.6.2.bb
@@ -16,6 +16,7 @@ SRC_URI = "http://downloads.yoctoproject.org/releases/${BPN}/${BPN}-${PV}.tar.gz
            file://opkg.conf \
            file://0001-opkg_conf-create-opkg.lock-in-run-instead-of-var-run.patch \
            file://0002-Add-options-to-enable-support-for-acl-and-xattr.patch \
+           file://0003-opkg-set-locale-from-system-environment-variables.patch \
            file://run-ptest \
            "
 
-- 
2.34.1

Re: [OE-Core][PATCH v13 5/8] sstatesig.py: fix hash calculation for timestamp

[Richard Purdie]

On Thu, 2023-08-17 at 14:46 +0200, Piotr Łobacz wrote:
> Passing timestamp as an integer was losing precision, which led
> to problems with badly recreated sstate cache.
> 
> Passing it as floating point number fixes the issue.
> 
> Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
> ---
>  meta/lib/oe/sstatesig.py | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
> index 633a0fd450..f7ea3d366c 100644
> --- a/meta/lib/oe/sstatesig.py
> +++ b/meta/lib/oe/sstatesig.py
> @@ -588,7 +588,7 @@ def OEOuthashBasic(path, sigfile, task, d):
>                          raise Exception(msg).with_traceback(e.__traceback__)
>  
>                  if include_timestamps:
> -                    update_hash(" %10d" % s.st_mtime)
> +                    update_hash(" %f" % s.st_mtime)
>  
>                  update_hash(" ")
>                  if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):

As mentioned on irc, we likely need to record ACL/XATTR data in this
file as well, else well just have a different set of bugs in future.
Now we've realised the issue we do need to fix it.

Cheers,

Richard

Re: [OE-Core][PATCH v13 2/8] package_ipk.bbclass: add support for ACLs and xattr

[Khem Raj]

On Thu, Aug 17, 2023 at 5:47 AM Piotr Łobacz <p.lobacz@welotec.com> wrote:
>
> Extend OPKGBUILDCMD variable, with additional parameters, depending
> on target distro features, in order to support ACLs and xattr.
>
> With fix pushed to the opkg-devel:
> https://groups.google.com/g/opkg-devel/c/dYNHrLjDwg8
> opkg-build is able to create tar archives with ACLs and xattr.
>
> Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
> ---
>  meta/classes-global/package_ipk.bbclass | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/meta/classes-global/package_ipk.bbclass b/meta/classes-global/package_ipk.bbclass
> index b4b7bc9ac2..a0f106e4ad 100644
> --- a/meta/classes-global/package_ipk.bbclass
> +++ b/meta/classes-global/package_ipk.bbclass
> @@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
>  PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
>
>  # Program to be used to build opkg packages
> -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
> +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}" ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)} ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'

I wonder if it should be an append instead, because these are
dependent on DISTRO_FEATURES anyway and it does not work if distro
features are
enabled and these options are removed from opkg-build cmdline. So
these are required if distro features are enabled. It will also help
the distros overriding OPKGBUILDCMD

>
>  OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
>  OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
> --
> 2.34.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#186307): https://lists.openembedded.org/g/openembedded-core/message/186307
> Mute This Topic: https://lists.openembedded.org/mt/100799493/1997914
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [raj.khem@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

Re: [OE-Core][PATCH v13 2/8] package_ipk.bbclass: add support for ACLs and xattr

[Khem Raj]

On Tue, Aug 22, 2023 at 8:58 AM Khem Raj <raj.khem@gmail.com> wrote:
>
> On Thu, Aug 17, 2023 at 5:47 AM Piotr Łobacz <p.lobacz@welotec.com> wrote:
> >
> > Extend OPKGBUILDCMD variable, with additional parameters, depending
> > on target distro features, in order to support ACLs and xattr.
> >
> > With fix pushed to the opkg-devel:
> > https://groups.google.com/g/opkg-devel/c/dYNHrLjDwg8
> > opkg-build is able to create tar archives with ACLs and xattr.
> >
> > Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
> > ---
> >  meta/classes-global/package_ipk.bbclass | 2 +-
> >  1 file changed, 1 insertion(+), 1 deletion(-)
> >
> > diff --git a/meta/classes-global/package_ipk.bbclass b/meta/classes-global/package_ipk.bbclass
> > index b4b7bc9ac2..a0f106e4ad 100644
> > --- a/meta/classes-global/package_ipk.bbclass
> > +++ b/meta/classes-global/package_ipk.bbclass
> > @@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
> >  PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
> >
> >  # Program to be used to build opkg packages
> > -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
> > +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}" ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)} ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'
>
> I wonder if it should be an append instead, because these are
> dependent on DISTRO_FEATURES anyway and it does not work if distro
> features are
> enabled and these options are removed from opkg-build cmdline. So
> these are required if distro features are enabled. It will also help
> the distros overriding OPKGBUILDCMD
>

btw. I am encountering packaging failures in several packages using
zstd compression instead of xz e.g.
https://snips.sh/f/R42MbTZryH

 Here is my OPKGBUILDCMD

OPKGBUILDCMD = 'opkg-build -Z zstd -a "--threads=${ZSTD_THREADS}"
${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)}
${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'


> >
> >  OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
> >  OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
> > --
> > 2.34.1
> >
> >
> > -=-=-=-=-=-=-=-=-=-=-=-
> > Links: You receive all messages sent to this group.
> > View/Reply Online (#186307): https://lists.openembedded.org/g/openembedded-core/message/186307
> > Mute This Topic: https://lists.openembedded.org/mt/100799493/1997914
> > Group Owner: openembedded-core+owner@lists.openembedded.org
> > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [raj.khem@gmail.com]
> > -=-=-=-=-=-=-=-=-=-=-=-
> >

Re: [OE-Core][PATCH v13 2/8] package_ipk.bbclass: add support for ACLs and xattr

[Khem Raj]

OK I have narrowed down the problem a bit more

It seems to trigger with poky master + this patch series on archlinux 
build host ( it has glibc 2.38 natively ) . It works ok with nodistro ( 
I guess its because nodistro does not use uninative but its just a guess )

Add following at the end of conf/local.conf

PACKAGE_CLASSES = "package_ipk"

then run

bitbake python3


On 8/22/23 10:03, Khem Raj wrote:
> On Tue, Aug 22, 2023 at 8:58 AM Khem Raj <raj.khem@gmail.com> wrote:
>>
>> On Thu, Aug 17, 2023 at 5:47 AM Piotr Łobacz <p.lobacz@welotec.com> wrote:
>>>
>>> Extend OPKGBUILDCMD variable, with additional parameters, depending
>>> on target distro features, in order to support ACLs and xattr.
>>>
>>> With fix pushed to the opkg-devel:
>>> https://groups.google.com/g/opkg-devel/c/dYNHrLjDwg8
>>> opkg-build is able to create tar archives with ACLs and xattr.
>>>
>>> Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
>>> ---
>>>   meta/classes-global/package_ipk.bbclass | 2 +-
>>>   1 file changed, 1 insertion(+), 1 deletion(-)
>>>
>>> diff --git a/meta/classes-global/package_ipk.bbclass b/meta/classes-global/package_ipk.bbclass
>>> index b4b7bc9ac2..a0f106e4ad 100644
>>> --- a/meta/classes-global/package_ipk.bbclass
>>> +++ b/meta/classes-global/package_ipk.bbclass
>>> @@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
>>>   PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
>>>
>>>   # Program to be used to build opkg packages
>>> -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
>>> +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}" ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)} ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'
>>
>> I wonder if it should be an append instead, because these are
>> dependent on DISTRO_FEATURES anyway and it does not work if distro
>> features are
>> enabled and these options are removed from opkg-build cmdline. So
>> these are required if distro features are enabled. It will also help
>> the distros overriding OPKGBUILDCMD
>>
> 
> btw. I am encountering packaging failures in several packages using
> zstd compression instead of xz e.g.
> https://snips.sh/f/R42MbTZryH
> 
>   Here is my OPKGBUILDCMD
> 
> OPKGBUILDCMD = 'opkg-build -Z zstd -a "--threads=${ZSTD_THREADS}"
> ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)}
> ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'
> 
> 
>>>
>>>   OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
>>>   OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
>>> --
>>> 2.34.1
>>>
>>>
>>> -=-=-=-=-=-=-=-=-=-=-=-
>>> Links: You receive all messages sent to this group.
>>> View/Reply Online (#186307): https://lists.openembedded.org/g/openembedded-core/message/186307
>>> Mute This Topic: https://lists.openembedded.org/mt/100799493/1997914
>>> Group Owner: openembedded-core+owner@lists.openembedded.org
>>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [raj.khem@gmail.com]
>>> -=-=-=-=-=-=-=-=-=-=-=-
>>>

Re: [OE-Core][PATCH v13 2/8] package_ipk.bbclass: add support for ACLs and xattr

[Khem Raj]

On Tue, Aug 22, 2023 at 5:25 PM Khem Raj <raj.khem@gmail.com> wrote:
>
> OK I have narrowed down the problem a bit more
>
> It seems to trigger with poky master + this patch series on archlinux
> build host ( it has glibc 2.38 natively ) . It works ok with nodistro (
> I guess its because nodistro does not use uninative but its just a guess )
>
> Add following at the end of conf/local.conf
>
> PACKAGE_CLASSES = "package_ipk"
>
> then run
>
> bitbake python3

If I just revert the following patch then do_package_write_ipk starts
to work normally.
I think this should be investigated and root caused, I am afraid that
if we accept this
series in its own form then once fedora/ubuntu/debian starts to get
glibc 2.38 this problem
will resurface.Lets hold on to this patch until this issue is resolved

commit 0ecd39717533ae11dc00c0a2e049c657ba41411f
Author: Piotr Łobacz <p.lobacz@welotec.com>
Date:   Thu Aug 17 14:46:10 2023 +0200

   opkg-utils: add acl and xattr support

   Add support for tar archives created with --acls and/or --xattrs options,
   PAX header format.

   GNU tar and libarchive already supports ACLs and extended attributes.
   We can now add this support as well to opkg-build script in order to use
   fsetattr or setcap inside do_install command and end up with a file in
   an image with the relevant ACLs and xattrs.

   (From OE-Core rev: f6228716c3ce25cda54c7a2d62df5f7be72765ec)

   Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
   Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>


>
>
> On 8/22/23 10:03, Khem Raj wrote:
> > On Tue, Aug 22, 2023 at 8:58 AM Khem Raj <raj.khem@gmail.com> wrote:
> >>
> >> On Thu, Aug 17, 2023 at 5:47 AM Piotr Łobacz <p.lobacz@welotec.com> wrote:
> >>>
> >>> Extend OPKGBUILDCMD variable, with additional parameters, depending
> >>> on target distro features, in order to support ACLs and xattr.
> >>>
> >>> With fix pushed to the opkg-devel:
> >>> https://groups.google.com/g/opkg-devel/c/dYNHrLjDwg8
> >>> opkg-build is able to create tar archives with ACLs and xattr.
> >>>
> >>> Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
> >>> ---
> >>>   meta/classes-global/package_ipk.bbclass | 2 +-
> >>>   1 file changed, 1 insertion(+), 1 deletion(-)
> >>>
> >>> diff --git a/meta/classes-global/package_ipk.bbclass b/meta/classes-global/package_ipk.bbclass
> >>> index b4b7bc9ac2..a0f106e4ad 100644
> >>> --- a/meta/classes-global/package_ipk.bbclass
> >>> +++ b/meta/classes-global/package_ipk.bbclass
> >>> @@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET = "${WORKDIR}/opkg-sdk-target.conf"
> >>>   PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
> >>>
> >>>   # Program to be used to build opkg packages
> >>> -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
> >>> +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}" ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)} ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'
> >>
> >> I wonder if it should be an append instead, because these are
> >> dependent on DISTRO_FEATURES anyway and it does not work if distro
> >> features are
> >> enabled and these options are removed from opkg-build cmdline. So
> >> these are required if distro features are enabled. It will also help
> >> the distros overriding OPKGBUILDCMD
> >>
> >
> > btw. I am encountering packaging failures in several packages using
> > zstd compression instead of xz e.g.
> > https://snips.sh/f/R42MbTZryH
> >
> >   Here is my OPKGBUILDCMD
> >
> > OPKGBUILDCMD = 'opkg-build -Z zstd -a "--threads=${ZSTD_THREADS}"
> > ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)}
> > ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'
> >
> >
> >>>
> >>>   OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
> >>>   OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
> >>> --
> >>> 2.34.1
> >>>
> >>>
> >>> -=-=-=-=-=-=-=-=-=-=-=-
> >>> Links: You receive all messages sent to this group.
> >>> View/Reply Online (#186307): https://lists.openembedded.org/g/openembedded-core/message/186307
> >>> Mute This Topic: https://lists.openembedded.org/mt/100799493/1997914
> >>> Group Owner: openembedded-core+owner@lists.openembedded.org
> >>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [raj.khem@gmail.com]
> >>> -=-=-=-=-=-=-=-=-=-=-=-
> >>>

Re: [OE-Core][PATCH v13 2/8] package_ipk.bbclass: add support for ACLs and xattr

[Piotr Łobacz]

Dnia środa, 23 sierpnia 2023 03:00:59 CEST Khem Raj pisze:
> On Tue, Aug 22, 2023 at 5:25 PM Khem Raj <raj.khem@gmail.com> wrote:
> > OK I have narrowed down the problem a bit more
> > 
> > It seems to trigger with poky master + this patch series on archlinux
> > build host ( it has glibc 2.38 natively ) . It works ok with nodistro (
> > I guess its because nodistro does not use uninative but its just a guess )
> > 
> > Add following at the end of conf/local.conf
> > 
> > PACKAGE_CLASSES = "package_ipk"
> > 
> > then run
> > 
> > bitbake python3
> 
> If I just revert the following patch then do_package_write_ipk starts
> to work normally.
> I think this should be investigated and root caused, I am afraid that
> if we accept this
> series in its own form then once fedora/ubuntu/debian starts to get
> glibc 2.38 this problem
> will resurface.Lets hold on to this patch until this issue is resolved
> 
> commit 0ecd39717533ae11dc00c0a2e049c657ba41411f
> Author: Piotr Łobacz <p.lobacz@welotec.com>
> Date:   Thu Aug 17 14:46:10 2023 +0200
> 
>    opkg-utils: add acl and xattr support
> 
>    Add support for tar archives created with --acls and/or --xattrs options,
> PAX header format.
> 
>    GNU tar and libarchive already supports ACLs and extended attributes.
>    We can now add this support as well to opkg-build script in order to use
>    fsetattr or setcap inside do_install command and end up with a file in
>    an image with the relevant ACLs and xattrs.
> 
>    (From OE-Core rev: f6228716c3ce25cda54c7a2d62df5f7be72765ec)
> 
>    Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
>    Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> 
> > On 8/22/23 10:03, Khem Raj wrote:
> > > On Tue, Aug 22, 2023 at 8:58 AM Khem Raj <raj.khem@gmail.com> wrote:
> > >> On Thu, Aug 17, 2023 at 5:47 AM Piotr Łobacz <p.lobacz@welotec.com> 
wrote:
> > >>> Extend OPKGBUILDCMD variable, with additional parameters, depending
> > >>> on target distro features, in order to support ACLs and xattr.
> > >>> 
> > >>> With fix pushed to the opkg-devel:
> > >>> https://eur04.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgrou
> > >>> ps.google.com%2Fg%2Fopkg-devel%2Fc%2FdYNHrLjDwg8&data=05%7C01%7Cp.loba
> > >>> cz%40welotec.com%7C34b048f9a1ac434ef11d08dba3747a90%7C25111a7f1d5a4c51
> > >>> a4ca7f8e44011b39%7C0%7C0%7C638283492895128587%7CUnknown%7CTWFpbGZsb3d8
> > >>> eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3
> > >>> 000%7C%7C%7C&sdata=EW1InBDnANNoMNZi2FEvYejU6qJW70thWgDajc4FKIw%3D&rese
> > >>> rved=0 opkg-build is able to create tar archives with ACLs and xattr.
> > >>> 
> > >>> Signed-off-by: Piotr Łobacz <p.lobacz@welotec.com>
> > >>> ---
> > >>> 
> > >>>   meta/classes-global/package_ipk.bbclass | 2 +-
> > >>>   1 file changed, 1 insertion(+), 1 deletion(-)
> > >>> 
> > >>> diff --git a/meta/classes-global/package_ipk.bbclass
> > >>> b/meta/classes-global/package_ipk.bbclass index
> > >>> b4b7bc9ac2..a0f106e4ad 100644
> > >>> --- a/meta/classes-global/package_ipk.bbclass
> > >>> +++ b/meta/classes-global/package_ipk.bbclass
> > >>> @@ -15,7 +15,7 @@ IPKGCONF_SDK_TARGET =
> > >>> "${WORKDIR}/opkg-sdk-target.conf"
> > >>> 
> > >>>   PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
> > >>>   
> > >>>   # Program to be used to build opkg packages
> > >>> 
> > >>> -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
> > >>> +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"
> > >>> ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)}
> > >>> ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'> >> 
> > >> I wonder if it should be an append instead, because these are
> > >> dependent on DISTRO_FEATURES anyway and it does not work if distro
> > >> features are
> > >> enabled and these options are removed from opkg-build cmdline. So
> > >> these are required if distro features are enabled. It will also help
> > >> the distros overriding OPKGBUILDCMD
> > > 
> > > btw. I am encountering packaging failures in several packages using
> > > zstd compression instead of xz e.g.
> > > https://eur04.safelinks.protection.outlook.com/?url=https%3A%2F%2Fsnips.
> > > sh%2Ff%2FR42MbTZryH&data=05%7C01%7Cp.lobacz%40welotec.com%7C34b048f9a1ac
> > > 434ef11d08dba3747a90%7C25111a7f1d5a4c51a4ca7f8e44011b39%7C0%7C0%7C638283
> > > 492895128587%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzI
> > > iLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=%2F5Txit2xk7Y9FSwI
> > > Pu2T%2BkqUKKMHAGZUoiEgK%2F12kLg%3D&reserved=0> > 
> > >   Here is my OPKGBUILDCMD
> > > 
> > > OPKGBUILDCMD = 'opkg-build -Z zstd -a "--threads=${ZSTD_THREADS}"
> > > ${@bb.utils.contains('DISTRO_FEATURES', 'acl', '-A', '', d)}
> > > ${@bb.utils.contains('DISTRO_FEATURES', 'xattr', '-X', '', d)}'
> > > 
> > >>>   OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
> > >>>   OPKG_ARGS += "${@['',
> > >>>   '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"> 
>>> 
> > >>> --
> > >>> 2.34.1
> > >>> 
> > >>> 
> > >>> -=-=-=-=-=-=-=-=-=-=-=-
> > >>> Links: You receive all messages sent to this group.
> > >>> View/Reply Online (#186307):
> > >>> https://eur04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
> > >>> ts.openembedded.org%2Fg%2Fopenembedded-core%2Fmessage%2F186307&data=05
> > >>> %7C01%7Cp.lobacz%40welotec.com%7C34b048f9a1ac434ef11d08dba3747a90%7C25
> > >>> 111a7f1d5a4c51a4ca7f8e44011b39%7C0%7C0%7C638283492895128587%7CUnknown%
> > >>> 7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > >>> VCI6Mn0%3D%7C3000%7C%7C%7C&sdata=hMQ3FL5OPsrbtWmI6Rnzomc27Tqs%2B6eKNpB
> > >>> zyZPFWDw%3D&reserved=0 Mute This Topic:
> > >>> https://eur04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
> > >>> ts.openembedded.org%2Fmt%2F100799493%2F1997914&data=05%7C01%7Cp.lobacz
> > >>> %40welotec.com%7C34b048f9a1ac434ef11d08dba3747a90%7C25111a7f1d5a4c51a4
> > >>> ca7f8e44011b39%7C0%7C0%7C638283492895128587%7CUnknown%7CTWFpbGZsb3d8ey
> > >>> JWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C300
> > >>> 0%7C%7C%7C&sdata=zQb36kiaKV6zlb4anGj15%2FvxjIdNh6zAKC3RyaWunhY%3D&rese
> > >>> rved=0 Group Owner: openembedded-core+owner@lists.openembedded.org
> > >>> Unsubscribe:
> > >>> https://eur04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
> > >>> ts.openembedded.org%2Fg%2Fopenembedded-core%2Funsub&data=05%7C01%7Cp.l
> > >>> obacz%40welotec.com%7C34b048f9a1ac434ef11d08dba3747a90%7C25111a7f1d5a4
> > >>> c51a4ca7f8e44011b39%7C0%7C0%7C638283492895128587%7CUnknown%7CTWFpbGZsb
> > >>> 3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%
> > >>> 7C3000%7C%7C%7C&sdata=yPC7QLDkwvqMuB25A3NzINdE6Ny%2BJkL1U82MD7UuBoI%3D
> > >>> &reserved=0 [raj.khem@gmail.com] -=-=-=-=-=-=-=-=-=-=-=-

As already spoken on IRC this does not happen with gentoo and newest glibc. 
I'm sending you my specs:

Portage 3.0.51 (python 3.11.5-final-0, default/linux/amd64/17.1/desktop/plasma/
systemd, gcc-12, glibc-2.38-r1, 6.1.46-gentoo-x86_64 x86_64)
=================================================================
System uname: Linux-6.1.46-gentoo-x86_64-x86_64-11th_Gen_Intel-R-_Core-TM-
_i7-11850H_@_2.50GHz-with-glibc2.38
KiB Mem:    32590392 total,   6513972 free
KiB Swap:   67108860 total,  62742056 free
Timestamp of repository gentoo: Thu, 31 Aug 2023 17:30:01 +0000
Head commit of repository gentoo: 02073da0e62df87543fbf41c59f3f9a9c948701d
Timestamp of repository stuff: Tue, 29 Aug 2023 15:50:45 +0000
Head commit of repository stuff: 948af023ae655b15674a680d72f965167d1435d9

sh bash 5.2_p15-r6
ld GNU ld (Gentoo 2.40 p5) 2.40.0
app-misc/pax-utils:        1.3.7::gentoo
app-shells/bash:           5.2_p15-r6::gentoo
dev-lang/perl:             5.38.0-r1::gentoo
dev-lang/python:           3.11.5::gentoo, 3.12.0_rc1_p6::gentoo
dev-lang/rust-bin:         1.72.0::gentoo
dev-util/cmake:            3.27.4::gentoo
dev-util/meson:            1.2.1-r1::gentoo
sys-apps/baselayout:       2.14::gentoo
sys-apps/sandbox:          2.38::gentoo
sys-apps/systemd:          254.1-r1::gentoo
sys-devel/autoconf:        2.13-r8::gentoo, 2.71-r7::gentoo
sys-devel/automake:        1.16.5-r1::gentoo
sys-devel/binutils:        2.40-r5::gentoo, 2.41-r1::gentoo
sys-devel/binutils-config: 5.5::gentoo
sys-devel/clang:           15.0.7-r3::gentoo, 16.0.6::gentoo
sys-devel/gcc:             12.3.1_p20230526::gentoo, 13.2.1_p20230826::gentoo
sys-devel/gcc-config:      2.11::gentoo
sys-devel/libtool:         2.4.7-r1::gentoo
sys-devel/lld:             15.0.7::gentoo, 16.0.6::gentoo
sys-devel/llvm:            15.0.7-r3::gentoo, 16.0.6::gentoo
sys-devel/make:            4.4.1-r1::gentoo
sys-kernel/linux-headers:  6.5::gentoo (virtual/os-headers)
sys-libs/glibc:            2.38-r1::gentoo
Repositories:

gentoo
    location: /var/db/repos/gentoo
    sync-type: rsync
    sync-uri: rsync://rsync.gentoo.org/gentoo-portage
    priority: -1000
    volatile: False
    sync-rsync-extra-opts: 
    sync-rsync-verify-max-age: 24
    sync-rsync-verify-metamanifest: yes
    sync-rsync-verify-jobs: 1

stuff
    location: /var/db/repos/stuff
    sync-type: git
    sync-uri: https://github.com/gentoo-mirror/stuff.git
    masters: gentoo
    volatile: False

Binary Repositories:

gentoobinhost
    priority: 1
    sync-uri: https://gentoo.osuosl.org/releases/amd64/binpackages/17.1/x86-64

ACCEPT_KEYWORDS="amd64 ~amd64"
ACCEPT_LICENSE="@FREE"
CBUILD="x86_64-pc-linux-gnu"
CFLAGS="-O2 -pipe"
CHOST="x86_64-pc-linux-gnu"
CONFIG_PROTECT="/etc /usr/lib64/libreoffice/program/sofficerc /usr/share/config /
usr/share/gnupg/qualified.txt"
CONFIG_PROTECT_MASK="/etc/ca-certificates.conf /etc/dconf /etc/env.d /etc/
fonts/fonts.conf /etc/gconf /etc/gentoo-release /etc/sandbox.d /etc/terminfo"
CXXFLAGS="-O2 -pipe"
DISTDIR="/var/cache/distfiles"
ENV_UNSET="CARGO_HOME DBUS_SESSION_BUS_ADDRESS DISPLAY GDK_PIXBUF_MODULE_FILE 
GOBIN GOPATH PERL5LIB PERL5OPT PERLPREFIX PERL_CORE PERL_MB_OPT PERL_MM_OPT 
XAUTHORITY XDG_CACHE_HOME XDG_CONFIG_HOME XDG_DATA_HOME XDG_RUNTIME_DIR 
XDG_STATE_HOME"
FCFLAGS="-O2 -pipe"
FEATURES="assume-digests binpkg-docompress binpkg-dostrip binpkg-logs binpkg-
multi-instance buildpkg-live config-protect-if-modified distlocks ebuild-locks 
fixlafiles ipc-sandbox merge-sync multilib-strict network-sandbox news parallel-
fetch pid-sandbox preserve-libs protect-owned qa-unresolved-soname-deps 
sandbox sfperms strict unknown-features-warn unmerge-logs unmerge-orphans 
userfetch userpriv usersandbox usersync xattr"
FFLAGS="-O2 -pipe"
GENTOO_MIRRORS="http://distfiles.gentoo.org"
LANG="pl_PL.utf8"
LDFLAGS="-Wl,-O1 -Wl,--as-needed"
LEX="flex"
MAKEOPTS="-j8"
PKGDIR="/var/cache/binpkgs"
PORTAGE_CONFIGROOT="/"
PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --omit-
dir-times --compress --force --whole-file --delete --stats --human-readable --
timeout=180 --exclude=/distfiles --exclude=/local --exclude=/packages --
exclude=/.git"
PORTAGE_TMPDIR="/var/tmp"
SHELL="/bin/bash"
USE="X a52 aac acl acpi activities alsa amd64 bluetooth branding bzip2 cairo 
cdda cdr cli crypt cups dbus declarative dri dts dvd dvdr encode exif flac 
fortran gdbm gif gpm gtk gui iconv icu ipv6 jpeg kde kwallet lcms libnotify 
libtirpc mad mng mp3 mp4 mpeg multilib ncurses nls nptl ogg opengl openmp pam 
pango pcre pdf pipewire plasma png policykit ppds pulseaudio qml qt5 readline 
screencast sdl seccomp semantic-desktop sound spell split-usr ssl startup-
notification svg systemd test-rust tiff truetype udev udisks unicode upower usb 
vorbis vulkan wayland widgets wxwidgets x264 xattr xcb xft xml xv xvid zlib" 
ABI_X86="64" ADA_TARGET="gnat_2021" APACHE2_MODULES="authn_core authz_core 
socache_shmcb unixd actions alias auth_basic authn_alias authn_anon authn_dbm 
authn_default authn_file authz_dbm authz_default authz_groupfile authz_host 
authz_owner authz_user autoindex cache cgi cgid dav dav_fs dav_lock deflate dir 
disk_cache env expires ext_filter file_cache filter headers include info 
log_config logio mem_cache mime mime_magic negotiation rewrite setenvif speling 
status unique_id userdir usertrack vhost_alias" CALLIGRA_FEATURES="karbon 
sheets words" COLLECTD_PLUGINS="df interface irq load memory rrdtool swap 
syslog" CPU_FLAGS_X86="mmx mmxext sse sse2 aes avx avx2 avx512bw avx512cd 
avx512dq avx512f avx512vbmi avx512vl f16c fma3 pclmul popcnt rdrand sha sse3 
sse4_1 sse4_2 ssse3" ELIBC="glibc" GPSD_PROTOCOLS="ashtech aivdm earthmate 
evermore fv18 garmin garmintxt gpsclock greis isync itrax mtk3301 nmea ntrip 
navcom oceanserver oldstyle oncore rtcm104v2 rtcm104v3 sirf skytraq superstar2 
timing tsip tripmate tnt ublox ubx" INPUT_DEVICES="libinput" KERNEL="linux" 
L10N="pl" LCD_DEVICES="bayrad cfontz cfontz633 glk hd44780 lb216 lcdm001 
mtxorb ncurses text" LIBREOFFICE_EXTENSIONS="presenter-console presenter-
minimizer" LUA_SINGLE_TARGET="lua5-1" LUA_TARGETS="lua5-1" 
OFFICE_IMPLEMENTATION="libreoffice" PHP_TARGETS="php8-1" 
POSTGRES_TARGETS="postgres15" PYTHON_SINGLE_TARGET="python3_11" 
PYTHON_TARGETS="python3_11" RUBY_TARGETS="ruby31" VIDEO_CARDS="amdgpu fbdev 
intel nouveau radeon radeonsi vesa dummy v4l" XTABLES_ADDONS="quota2 psd 
pknock lscan length2 ipv4options ipset ipp2p iface geoip fuzzy condition tee 
tarpit sysrq proto steal rawnat logmark ipmark dhcpmac delude chaos account"
Unset:  ADDR2LINE, AR, ARFLAGS, AS, ASFLAGS, CC, CCLD, CONFIG_SHELL, CPP, 
CPPFLAGS, CTARGET, CXX, CXXFILT, ELFEDIT, EMERGE_DEFAULT_OPTS, EXTRA_ECONF, 
F77FLAGS, FC, GCOV, GPROF, INSTALL_MASK, LC_ALL, LD, LFLAGS, LIBTOOL, LINGUAS, 
MAKE, MAKEFLAGS, NM, OBJCOPY, OBJDUMP, PORTAGE_BINHOST, 
PORTAGE_BUNZIP2_COMMAND, PORTAGE_COMPRESS, PORTAGE_COMPRESS_FLAGS, 
PORTAGE_RSYNC_EXTRA_OPTS, RANLIB, READELF, RUSTFLAGS, SIZE, STRINGS, STRIP, 
YACC, YFLAGS