[PATCH 1/6] ldconfig-native: don't write auxiliary cache

[PATCH 1/6] ldconfig-native: don't write auxiliary cache

[Ross Burton]

The auxiliary cache written by ldconfig is only useful for speeding up
future runs of ldconfig, and as it contains inode numbers as the keys
in a dictionary it is entirely pointless to generate on the build host.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 .../ldconfig-native-2.12.1/no-aux-cache.patch | 19 +++++++++++++++++++
 .../glibc/ldconfig-native_2.12.1.bb           |  1 +
 2 files changed, 20 insertions(+)
 create mode 100644 meta/recipes-core/glibc/ldconfig-native-2.12.1/no-aux-cache.patch

diff --git a/meta/recipes-core/glibc/ldconfig-native-2.12.1/no-aux-cache.patch b/meta/recipes-core/glibc/ldconfig-native-2.12.1/no-aux-cache.patch
new file mode 100644
index 00000000000..c6765ba00d2
--- /dev/null
+++ b/meta/recipes-core/glibc/ldconfig-native-2.12.1/no-aux-cache.patch
@@ -0,0 +1,19 @@
+The ldconfig auxiliary cache is a dictionary where the keys include inode, so
+there is no point in writing these files on the build host. 
+
+Upstream-Status: Inappropriate
+Signed-off-by: Ross Burton <ross.burton@arm.com>
+
+diff --git a/ldconfig.c b/ldconfig.c
+index 2c4eb57..2d6dc92 100644
+--- a/ldconfig.c
++++ b/ldconfig.c
+@@ -1399,8 +1399,6 @@ main (int argc, char **argv)
+   if (opt_build_cache)
+     {
+       save_cache (cache_file);
+-      if (aux_cache_file)
+-        save_aux_cache (aux_cache_file);
+     }
+ 
+   return 0;
diff --git a/meta/recipes-core/glibc/ldconfig-native_2.12.1.bb b/meta/recipes-core/glibc/ldconfig-native_2.12.1.bb
index 93c0b186714..919d11417d3 100644
--- a/meta/recipes-core/glibc/ldconfig-native_2.12.1.bb
+++ b/meta/recipes-core/glibc/ldconfig-native_2.12.1.bb
@@ -14,6 +14,7 @@ SRC_URI = "file://ldconfig-native-2.12.1.tar.bz2 \
            file://ldconfig-default-to-all-multilib-dirs.patch \
            file://endian-ness_handling_fix.patch \
            file://add-64-bit-flag-for-ELF64-entries.patch \
+           file://no-aux-cache.patch \
 "
 
 PR = "r2"
-- 
2.25.1

[PATCH 2/6] image_types: remove obsolete tar comment

[Ross Burton]

We now depend on tar 1.28, so talking about older tar versions is just
confusing.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/image_types.bbclass | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index 66884af8e09..3ec321765d9 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -108,16 +108,6 @@ IMAGE_CMD_squashfs-xz = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME
 IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lzo ${EXTRA_IMAGECMD} -noappend -comp lzo"
 IMAGE_CMD_squashfs-lz4 = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lz4 ${EXTRA_IMAGECMD} -noappend -comp lz4"
 
-# By default, tar from the host is used, which can be quite old. If
-# you need special parameters (like --xattrs) which are only supported
-# by GNU tar upstream >= 1.27, then override that default:
-# IMAGE_CMD_TAR = "tar --xattrs --xattrs-include=*"
-# do_image_tar[depends] += "tar-replacement-native:do_populate_sysroot"
-# EXTRANATIVEPATH += "tar-native"
-#
-# The GNU documentation does not specify whether --xattrs-include is necessary.
-# In practice, it turned out to be not needed when creating archives and
-# required when extracting, but it seems prudent to use it in both cases.
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
 IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
-- 
2.25.1

[PATCH 3/6] image_types: sort tarball file listings

[Ross Burton]

Help rootfs tarballs be reproducible by sorting the file listing.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/image_types.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index 3ec321765d9..286009057e3 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -110,7 +110,7 @@ IMAGE_CMD_squashfs-lz4 = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAM
 
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
-IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --sort=name --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD_cpio () {
-- 
2.25.1

[PATCH 4/6] reproducible_build: clamp the rootfs tarball mtime to the epoch

[Ross Burton]

To handle files being written to the rootfs outside of the package manager,
clamp the mtimes to the epoch.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/reproducible_build.bbclass | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass
index 2f3bd90b073..28a25fb492d 100644
--- a/meta/classes/reproducible_build.bbclass
+++ b/meta/classes/reproducible_build.bbclass
@@ -116,3 +116,6 @@ python () {
     if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
         d.appendVarFlag("do_unpack", "postfuncs", " create_source_date_epoch_stamp")
 }
+
+# Clamp the mtimes of the tarball image filesystem type
+IMAGE_CMD_tar_append = " --clamp-mtime --mtime=@${SOURCE_DATE_EPOCH}"
-- 
2.25.1

Re: [OE-core] [PATCH 4/6] reproducible_build: clamp the rootfs tarball mtime to the epoch

[Richard Purdie]

On Tue, 2020-12-01 at 15:23 +0000, Ross Burton wrote:
> To handle files being written to the rootfs outside of the package
> manager,
> clamp the mtimes to the epoch.
> 
> Signed-off-by: Ross Burton <ross.burton@arm.com>
> ---
>  meta/classes/reproducible_build.bbclass | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git a/meta/classes/reproducible_build.bbclass
> b/meta/classes/reproducible_build.bbclass
> index 2f3bd90b073..28a25fb492d 100644
> --- a/meta/classes/reproducible_build.bbclass
> +++ b/meta/classes/reproducible_build.bbclass
> @@ -116,3 +116,6 @@ python () {
>      if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
>          d.appendVarFlag("do_unpack", "postfuncs", "
> create_source_date_epoch_stamp")
>  }
> +
> +# Clamp the mtimes of the tarball image filesystem type
> +IMAGE_CMD_tar_append = " --clamp-mtime --
> mtime=@${SOURCE_DATE_EPOCH}"
> 

I'm not convinced this is correct as it could mess up the timestamps of
stamps set in other recipes? 

It's also tar specific and should really be across all image types?

Cheers,

Richard

Re: [OE-core] [PATCH 4/6] reproducible_build: clamp the rootfs tarball mtime to the epoch

[Ross Burton]

On Tue, 1 Dec 2020 at 15:45, Richard Purdie
<richard.purdie@linuxfoundation.org> wrote:
> I'm not convinced this is correct as it could mess up the timestamps of
> stamps set in other recipes?
>
> It's also tar specific and should really be across all image types?

Yes, it most likely should be done at rootfs time instead.

Ross

Re: [OE-core] [PATCH 4/6] reproducible_build: clamp the rootfs tarball mtime to the epoch

[Jate Sujjavanich]

[-- Attachment #1: Type: text/plain, Size: 1761 bytes --]

I have started using the following code to set the mtimes of files written
after the build timestamp. It assumes that SOURCE_DATE_EPOCH of packages
are never in the future.

# Set to empty to allow custom function below to run
REPRODUCIBLE_TIMESTAMP_ROOTFS ?= ""

# Use DATETIME of build as SOURCE_DATE_EPOCH for image
export SOURCE_DATE_EPOCH = "${@int( time.mktime( time.strptime('${DATETIME}',
'%Y%m%d%H%M%S') ) ) }"

reproducible_final_image_task () {
if [ "${BUILD_REPRODUCIBLE_BINARIES}" = "1" ]; then
if [ "$REPRODUCIBLE_TIMESTAMP_ROOTFS" = "" ]; then
# Use commit time of tuxzilla root in usual Syntux build
REPRODUCIBLE_TIMESTAMP_ROOTFS=`git -C "${COREBASE}/../" log -1 --pretty=%ct
2>/dev/null`
if [ "$REPRODUCIBLE_TIMESTAMP_ROOTFS" = "" ]; then
REPRODUCIBLE_TIMESTAMP_ROOTFS=`stat -c%Y ${@bb.utils.which(d.getVar("BBPATH"),
"conf/bitbake.conf")}`
fi
fi
# Set mtime of all files to a reproducible value
bbnote "reproducible_final_image_task: mtime set to
$REPRODUCIBLE_TIMESTAMP_ROOTFS"
# Files with mtime after build stamp should be part of rootfs creation
DAYS_SINCE_BUILD=`python3 -c "import time; print((time.time() -
${SOURCE_DATE_EPOCH}) / 86400)"`
find ${IMAGE_ROOTFS} -mtime -$DAYS_SINCE_BUILD -exec touch -h --date=@
$REPRODUCIBLE_TIMESTAMP_ROOTFS {} \;
fi
}


On Tue, Dec 1, 2020 at 12:16 PM Ross Burton <ross@burtonini.com> wrote:

> On Tue, 1 Dec 2020 at 15:45, Richard Purdie
> <richard.purdie@linuxfoundation.org> wrote:
> > I'm not convinced this is correct as it could mess up the timestamps of
> > stamps set in other recipes?
> >
> > It's also tar specific and should really be across all image types?
>
> Yes, it most likely should be done at rootfs time instead.
>
> Ross
>
> 
>
>

[-- Attachment #2: Type: text/html, Size: 6717 bytes --]

[PATCH 5/6] package_manager/ipk: neaten OPKGLIBDIR logic

[Ross Burton]

oe.path.join handles path components starting with / for us.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/lib/oe/package_manager/ipk/__init__.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
index 416ed23d474..2753b139ed1 100644
--- a/meta/lib/oe/package_manager/ipk/__init__.py
+++ b/meta/lib/oe/package_manager/ipk/__init__.py
@@ -172,12 +172,7 @@ class OpkgPM(OpkgDpkgPM):
         if prepare_index:
             create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
 
-        opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
-        if opkg_lib_dir[0] == "/":
-            opkg_lib_dir = opkg_lib_dir[1:]
-
-        self.opkg_dir = os.path.join(target_rootfs, opkg_lib_dir, "opkg")
-
+        self.opkg_dir = oe.path.join(target_rootfs, self.d.getVar('OPKGLIBDIR'), "opkg")
         bb.utils.mkdirhier(self.opkg_dir)
 
         self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
-- 
2.25.1

[PATCH 6/6] package_manager/ipk: improve remove_packaging_data

[Ross Burton]

/var/cache/opkg wasn't being deleted, and /var/lib/opkg doesn't need
to exist as there are no lockfiles that write into it after this step.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/lib/oe/package_manager/ipk/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
index 2753b139ed1..da488c1c7f0 100644
--- a/meta/lib/oe/package_manager/ipk/__init__.py
+++ b/meta/lib/oe/package_manager/ipk/__init__.py
@@ -403,9 +403,9 @@ class OpkgPM(OpkgDpkgPM):
             bb.fatal(result)
 
     def remove_packaging_data(self):
+        cachedir = oe.path.join(self.target_rootfs, self.d.getVar("localstatedir"), "cache", "opkg")
         bb.utils.remove(self.opkg_dir, True)
-        # create the directory back, it's needed by PM lock
-        bb.utils.mkdirhier(self.opkg_dir)
+        bb.utils.remove(cachedir, True)
 
     def remove_lists(self):
         if not self.from_feeds:
-- 
2.25.1