[Buildroot] [PATCH 01/15 v3] fs: run filesystem hooks under fakeroot

[Buildroot] [PATCH 01/15 v3] fs: run filesystem hooks under fakeroot

[Yann E. MORIN]

Currently, the pre-gen hooks of the various filesystems are run before
we enter the fakeroot.

However, this precludes those hooks from doing actions that require
root, like creating a pseudo-device or the likes.

So, move those pre-gen hooks under fakeroot.

This has currently no side-effect, as they are still called before
everything else in the fakeroot script, even the system-wide chown call.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fs/common.mk b/fs/common.mk
index 6d579db810..274a3e9c7b 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -76,9 +76,10 @@ $$(BINARIES_DIR)/rootfs.$(1): target-finalize $$(ROOTFS_$(2)_DEPENDENCIES)
 	@$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
 	rm -rf $(FS_DIR)
 	mkdir -p $(FS_DIR)
-	$$(foreach hook,$$(ROOTFS_$(2)_PRE_GEN_HOOKS),$$(call $$(hook))$$(sep))
 	echo '#!/bin/sh' > $$(FAKEROOT_SCRIPT)
 	echo "set -e" >> $$(FAKEROOT_SCRIPT)
+	$$(foreach hook,$$(ROOTFS_$(2)_PRE_GEN_HOOKS),\
+		$$(call PRINTF,$$($$(hook))) >> $$(FAKEROOT_SCRIPT)$$(sep))
 	echo "chown -h -R 0:0 $$(TARGET_DIR)" >> $$(FAKEROOT_SCRIPT)
 ifneq ($$(ROOTFS_USERS_TABLES),)
 	cat $$(ROOTFS_USERS_TABLES) >> $$(USERS_TABLE)
-- 
2.14.1

[Buildroot] [PATCH 02/15 v3] fs/cpio: don't extend packages' permissions table

[Yann E. MORIN]

Currently, when we build a cpio filesystem without static devices, we
shoehorn the /dev/console node as if it were specified by a package.

This means that this device is added for all filesystems as well, not
just the cpio. But if we disable cpio, that device is not created for
other filesystems.

This is not very clean, and may break expectations.

Instead, use an explicit mknod as part of the _CMD, as we know it's
going to run under fakeroot.

This is still visible to all filesystems built after cpio, and not to
those built before it, though.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/cpio/cpio.mk | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fs/cpio/cpio.mk b/fs/cpio/cpio.mk
index aa4d947fa5..505976512b 100644
--- a/fs/cpio/cpio.mk
+++ b/fs/cpio/cpio.mk
@@ -15,14 +15,15 @@ endef
 else
 # devtmpfs does not get automounted when initramfs is used.
 # Add a pre-init script to mount it before running init
+# We must have /dev/console very early, even before /init runs, so
+# that it can at least open its stdin/stdout/stderr filedescriptors.
 define ROOTFS_CPIO_ADD_INIT
 	if [ ! -e $(TARGET_DIR)/init ]; then \
 		$(INSTALL) -m 0755 fs/cpio/init $(TARGET_DIR)/init; \
 	fi
+	mknod -m 0622 $(TARGET_DIR)/dev/console c 5 1
 endef
 
-PACKAGES_PERMISSIONS_TABLE += /dev/console c 622 0 0 5 1 - - -$(sep)
-
 endif # BR2_ROOTFS_DEVICE_CREATION_STATIC
 
 ROOTFS_CPIO_PRE_GEN_HOOKS += ROOTFS_CPIO_ADD_INIT
-- 
2.14.1

[Buildroot] [PATCH 03/15 v3] fs/iso9660: rename internal variable

[Yann E. MORIN]

We're soon to introduce a generic variable with that name, i.e.
$(FS_NAME)_TARGET_DIR, so our internal, temporary variable would
clash with it.

Just rename it.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/iso9660/iso9660.mk | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/fs/iso9660/iso9660.mk b/fs/iso9660/iso9660.mk
index 207c6aa39f..b33cf4cd8a 100644
--- a/fs/iso9660/iso9660.mk
+++ b/fs/iso9660/iso9660.mk
@@ -35,47 +35,47 @@ ROOTFS_ISO9660_USE_INITRD = YES
 endif
 
 ifeq ($(ROOTFS_ISO9660_USE_INITRD),YES)
-ROOTFS_ISO9660_TARGET_DIR = $(FS_DIR)/rootfs.iso9660.tmp
+ROOTFS_ISO9660_TMP_TARGET_DIR = $(FS_DIR)/rootfs.iso9660.tmp
 define ROOTFS_ISO9660_CREATE_TEMPDIR
-	$(RM) -rf $(ROOTFS_ISO9660_TARGET_DIR)
-	mkdir -p $(ROOTFS_ISO9660_TARGET_DIR)
+	$(RM) -rf $(ROOTFS_ISO9660_TMP_TARGET_DIR)
+	mkdir -p $(ROOTFS_ISO9660_TMP_TARGET_DIR)
 endef
 ROOTFS_ISO9660_PRE_GEN_HOOKS += ROOTFS_ISO9660_CREATE_TEMPDIR
 else ifeq ($(BR2_TARGET_ROOTFS_ISO9660_TRANSPARENT_COMPRESSION),y)
 ROOTFS_ISO9660_DEPENDENCIES += host-zisofs-tools
-ROOTFS_ISO9660_TARGET_DIR = $(FS_DIR)/rootfs.iso9660.tmp
+ROOTFS_ISO9660_TMP_TARGET_DIR = $(FS_DIR)/rootfs.iso9660.tmp
 # This must be early, before we copy the bootloader files.
 define ROOTFS_ISO9660_MKZFTREE
-	$(RM) -rf $(ROOTFS_ISO9660_TARGET_DIR)
+	$(RM) -rf $(ROOTFS_ISO9660_TMP_TARGET_DIR)
 	$(HOST_DIR)/bin/mkzftree -X -z 9 -p $(PARALLEL_JOBS) \
 		$(TARGET_DIR) \
-		$(ROOTFS_ISO9660_TARGET_DIR)
+		$(ROOTFS_ISO9660_TMP_TARGET_DIR)
 endef
 ROOTFS_ISO9660_PRE_GEN_HOOKS += ROOTFS_ISO9660_MKZFTREE
 ROOTFS_ISO9660_GENISOIMAGE_OPTS += -z
 else
-ROOTFS_ISO9660_TARGET_DIR = $(TARGET_DIR)
+ROOTFS_ISO9660_TMP_TARGET_DIR = $(TARGET_DIR)
 endif
 
 ifeq ($(BR2_TARGET_ROOTFS_ISO9660_GRUB2),y)
 ROOTFS_ISO9660_DEPENDENCIES += grub2
 ROOTFS_ISO9660_BOOTLOADER_CONFIG_PATH = \
-	$(ROOTFS_ISO9660_TARGET_DIR)/boot/grub/grub.cfg
+	$(ROOTFS_ISO9660_TMP_TARGET_DIR)/boot/grub/grub.cfg
 ROOTFS_ISO9660_BOOT_IMAGE = boot/grub/grub-eltorito.img
 define ROOTFS_ISO9660_INSTALL_BOOTLOADER
 	$(INSTALL) -D -m 0644 $(BINARIES_DIR)/grub-eltorito.img \
-		$(ROOTFS_ISO9660_TARGET_DIR)/boot/grub/grub-eltorito.img
+		$(ROOTFS_ISO9660_TMP_TARGET_DIR)/boot/grub/grub-eltorito.img
 endef
 else ifeq ($(BR2_TARGET_ROOTFS_ISO9660_ISOLINUX),y)
 ROOTFS_ISO9660_DEPENDENCIES += syslinux
 ROOTFS_ISO9660_BOOTLOADER_CONFIG_PATH = \
-	$(ROOTFS_ISO9660_TARGET_DIR)/isolinux/isolinux.cfg
+	$(ROOTFS_ISO9660_TMP_TARGET_DIR)/isolinux/isolinux.cfg
 ROOTFS_ISO9660_BOOT_IMAGE = isolinux/isolinux.bin
 define ROOTFS_ISO9660_INSTALL_BOOTLOADER
 	$(INSTALL) -D -m 0644 $(BINARIES_DIR)/syslinux/* \
-		$(ROOTFS_ISO9660_TARGET_DIR)/isolinux/
+		$(ROOTFS_ISO9660_TMP_TARGET_DIR)/isolinux/
 	$(INSTALL) -D -m 0644 $(HOST_DIR)/share/syslinux/ldlinux.c32 \
-		$(ROOTFS_ISO9660_TARGET_DIR)/isolinux/ldlinux.c32
+		$(ROOTFS_ISO9660_TMP_TARGET_DIR)/isolinux/ldlinux.c32
 endef
 endif
 
@@ -96,7 +96,7 @@ endef
 # Copy the kernel to temporary filesystem
 define ROOTFS_ISO9660_COPY_KERNEL
 	$(INSTALL) -D -m 0644 $(LINUX_IMAGE_PATH) \
-		$(ROOTFS_ISO9660_TARGET_DIR)/boot/$(LINUX_IMAGE_NAME)
+		$(ROOTFS_ISO9660_TMP_TARGET_DIR)/boot/$(LINUX_IMAGE_NAME)
 endef
 
 ifeq ($(ROOTFS_ISO9660_USE_INITRD),YES)
@@ -111,7 +111,7 @@ else
 ROOTFS_ISO9660_DEPENDENCIES += rootfs-cpio
 define ROOTFS_ISO9660_COPY_INITRD
 	$(INSTALL) -D -m 0644 $(BINARIES_DIR)/rootfs.cpio$(ROOTFS_CPIO_COMPRESS_EXT) \
-		$(ROOTFS_ISO9660_TARGET_DIR)/boot/initrd
+		$(ROOTFS_ISO9660_TMP_TARGET_DIR)/boot/initrd
 	$(SED) "s%__INITRD_PATH__%/boot/initrd%" \
 		$(ROOTFS_ISO9660_BOOTLOADER_CONFIG_PATH)
 endef
@@ -133,7 +133,7 @@ define ROOTFS_ISO9660_CMD
 	$(HOST_DIR)/bin/genisoimage -J -R -b $(ROOTFS_ISO9660_BOOT_IMAGE) \
 		-no-emul-boot -boot-load-size 4 -boot-info-table \
 		$(ROOTFS_ISO9660_GENISOIMAGE_OPTS) \
-		-o $@ $(ROOTFS_ISO9660_TARGET_DIR)
+		-o $@ $(ROOTFS_ISO9660_TMP_TARGET_DIR)
 endef
 
 ifeq ($(BR2_TARGET_ROOTFS_ISO9660_HYBRID),y)
-- 
2.14.1

[Buildroot] [PATCH 04/15 v3] core: introduce intermediate BASE_TARGET_DIR variable

[Yann E. MORIN]

This new BASE_TARGET_DIR variable is set in stone to point to the real
location where packages will be installed. Its name is modelled after
its definition: it is located in $(BASE_DIR), and it is named 'target/',
hence BASE_TARGET_DIR.

The already-existing TARGET_DIR variable now simply points to the same
location, except that it is recursively expanded, so that we can later
change it depending on the context.

All locations that really need to reference the existing target/
directory, are changed to use BASE_TARGET_DIR; surprinsigly enough, they
all seem to be located in the main Makefile. :-) The rest is left with
using good-old TARGET_DIR.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 Makefile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 1b34818fc8..1c451eca45 100644
--- a/Makefile
+++ b/Makefile
@@ -215,7 +215,8 @@ BR_GRAPH_OUT := $(or $(BR2_GRAPH_OUT),pdf)
 
 BUILD_DIR := $(BASE_DIR)/build
 BINARIES_DIR := $(BASE_DIR)/images
-TARGET_DIR := $(BASE_DIR)/target
+BASE_TARGET_DIR := $(BASE_DIR)/target
+TARGET_DIR = $(BASE_TARGET_DIR)
 # initial definition so that 'make clean' works for most users, even without
 # .config. HOST_DIR will be overwritten later when .config is included.
 HOST_DIR := $(BASE_DIR)/host
@@ -237,7 +238,7 @@ LEGAL_REPORT = $(LEGAL_INFO_DIR)/README
 # dependencies anywhere else
 #
 ################################################################################
-$(BUILD_DIR) $(TARGET_DIR) $(HOST_DIR) $(BINARIES_DIR) $(LEGAL_INFO_DIR) $(REDIST_SOURCES_DIR_TARGET) $(REDIST_SOURCES_DIR_HOST):
+$(BUILD_DIR) $(BASE_TARGET_DIR) $(HOST_DIR) $(BINARIES_DIR) $(LEGAL_INFO_DIR) $(REDIST_SOURCES_DIR_TARGET) $(REDIST_SOURCES_DIR_HOST):
 	@mkdir -p $@
 
 BR2_CONFIG = $(CONFIG_DIR)/.config
@@ -460,7 +461,7 @@ BR_PATH = "$(HOST_DIR)/bin:$(HOST_DIR)/sbin:$(PATH)"
 
 # Location of a file giving a big fat warning that output/target
 # should not be used as the root filesystem.
-TARGET_DIR_WARNING_FILE = $(TARGET_DIR)/THIS_IS_NOT_YOUR_ROOT_FILESYSTEM
+TARGET_DIR_WARNING_FILE = $(BASE_TARGET_DIR)/THIS_IS_NOT_YOUR_ROOT_FILESYSTEM
 
 ifeq ($(BR2_CCACHE),y)
 CCACHE := $(HOST_DIR)/bin/ccache
@@ -559,7 +560,7 @@ $(foreach pkg,$(call UPPERCASE,$(PACKAGES)),\
 endif
 
 .PHONY: dirs
-dirs: $(BUILD_DIR) $(STAGING_DIR) $(TARGET_DIR) \
+dirs: $(BUILD_DIR) $(STAGING_DIR) $(BASE_TARGET_DIR) \
 	$(HOST_DIR) $(HOST_DIR)/usr $(HOST_DIR)/lib $(BINARIES_DIR)
 
 $(BUILD_DIR)/buildroot-config/auto.conf: $(BR2_CONFIG)
@@ -985,7 +986,7 @@ printvars:
 
 .PHONY: clean
 clean:
-	rm -rf $(TARGET_DIR) $(BINARIES_DIR) $(HOST_DIR) \
+	rm -rf $(BASE_TARGET_DIR) $(BINARIES_DIR) $(HOST_DIR) \
 		$(BUILD_DIR) $(BASE_DIR)/staging \
 		$(LEGAL_INFO_DIR) $(GRAPHS_DIR)
 
-- 
2.14.1

[Buildroot] [PATCH 05/15 v3] fs: set per-rootfs variable name

[Yann E. MORIN]

Like we do for packages with the PKG variable, set ROOTFS to contain the
upper-case name of the rootfs currently being generated.

This will be usefull in later patches, when we need more per-rootfs
variables, like a per-rootfs TARGET_DIR for example.

In Makefiles, per-rule variables trickle down the dependency chain, to
all dependencies of that rule, so we have to stop ROOTFS as soone as
we're not in a rootfs. This means we have to stop it at target-finalize
(which is a dependency of all filesystems), and for each package
individually, since some packages (host or target) can be direct
dependencies of filesystems as well.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 Makefile               | 3 +++
 fs/common.mk           | 3 +++
 package/pkg-generic.mk | 4 ++++
 3 files changed, 10 insertions(+)

diff --git a/Makefile b/Makefile
index 1c451eca45..439aaa9cb0 100644
--- a/Makefile
+++ b/Makefile
@@ -693,6 +693,9 @@ endif
 
 $(TARGETS_ROOTFS): target-finalize
 
+# Avoid the rootfs name leaking down the dependency chain
+target-finalize: ROOTFS=
+
 .PHONY: target-finalize
 target-finalize: $(PACKAGES)
 	@$(call MESSAGE,"Finalizing target directory")
diff --git a/fs/common.mk b/fs/common.mk
index 274a3e9c7b..af6799bb83 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -39,6 +39,8 @@ ROOTFS_USERS_TABLES = $(call qstrip,$(BR2_ROOTFS_USERS_TABLES))
 # all variable references except the arguments must be $$-quoted.
 define inner-rootfs
 
+ROOTFS_$(2)_NAME = $(1)
+
 # extra deps
 ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs \
 	$$(if $$(PACKAGES_USERS)$$(ROOTFS_USERS_TABLES),host-mkpasswd)
@@ -72,6 +74,7 @@ ROOTFS_$(2)_COMPRESS_EXT = .xz
 ROOTFS_$(2)_COMPRESS_CMD = xz -9 -C crc32 -c
 endif
 
+$$(BINARIES_DIR)/rootfs.$(1): ROOTFS=$(2)
 $$(BINARIES_DIR)/rootfs.$(1): target-finalize $$(ROOTFS_$(2)_DEPENDENCIES)
 	@$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
 	rm -rf $(FS_DIR)
diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk
index 9eddaeee57..1a637ea288 100644
--- a/package/pkg-generic.mk
+++ b/package/pkg-generic.mk
@@ -411,6 +411,10 @@ endef
 
 define inner-generic-package
 
+# When doing a package, we're definitely not doing a rootfs, but we
+# may inherit it via the dependency chain, so we reset it.
+$(1): ROOTFS=
+
 # Ensure the package is only declared once, i.e. do not accept that a
 # package be re-defined by a br2-external tree
 ifneq ($(call strip,$(filter $(1),$(PACKAGES_ALL))),)
-- 
2.14.1

[Buildroot] [PATCH 06/15 v3] fs: define per-rootfs directory

[Yann E. MORIN]

This will serve in future commits to store pre-rootfs files, like
fakeroot script...

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fs/common.mk b/fs/common.mk
index af6799bb83..eebe83d6e5 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -40,6 +40,7 @@ ROOTFS_USERS_TABLES = $(call qstrip,$(BR2_ROOTFS_USERS_TABLES))
 define inner-rootfs
 
 ROOTFS_$(2)_NAME = $(1)
+ROOTFS_$(2)_DIR = $$(FS_DIR)/$(1)
 
 # extra deps
 ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs \
-- 
2.14.1

[Buildroot] [PATCH 07/15 v3] fs: use a per-rootfs fakeroot script

[Yann E. MORIN]

... and locate that script in a per-rootfs directory.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fs/common.mk b/fs/common.mk
index eebe83d6e5..db4f8c23ad 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -28,7 +28,6 @@
 # macro will automatically generate a compressed filesystem image.
 
 FS_DIR = $(BUILD_DIR)/buildroot-fs
-FAKEROOT_SCRIPT = $(FS_DIR)/fakeroot.fs
 FULL_DEVICE_TABLE = $(FS_DIR)/device_table.txt
 ROOTFS_DEVICE_TABLES = $(call qstrip,$(BR2_ROOTFS_DEVICE_TABLE) \
 	$(BR2_ROOTFS_STATIC_DEVICE_TABLE))
@@ -76,10 +75,11 @@ ROOTFS_$(2)_COMPRESS_CMD = xz -9 -C crc32 -c
 endif
 
 $$(BINARIES_DIR)/rootfs.$(1): ROOTFS=$(2)
+$$(BINARIES_DIR)/rootfs.$(1): FAKEROOT_SCRIPT=$$(ROOTFS_$(2)_DIR)/fakeroot
 $$(BINARIES_DIR)/rootfs.$(1): target-finalize $$(ROOTFS_$(2)_DEPENDENCIES)
 	@$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
-	rm -rf $(FS_DIR)
-	mkdir -p $(FS_DIR)
+	rm -rf $(FS_DIR) $$(ROOTFS_$(2)_DIR)
+	mkdir -p $(FS_DIR) $$(ROOTFS_$(2)_DIR)
 	echo '#!/bin/sh' > $$(FAKEROOT_SCRIPT)
 	echo "set -e" >> $$(FAKEROOT_SCRIPT)
 	$$(foreach hook,$$(ROOTFS_$(2)_PRE_GEN_HOOKS),\
-- 
2.14.1

[Buildroot] [PATCH 08/15 v3] fs: split per-rootfs dependency line

[Yann E. MORIN]

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>

---
Note: in its own patch for easier review of a later, bigger patch.
---
 fs/common.mk | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fs/common.mk b/fs/common.mk
index db4f8c23ad..cd132cdbc7 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -76,7 +76,8 @@ endif
 
 $$(BINARIES_DIR)/rootfs.$(1): ROOTFS=$(2)
 $$(BINARIES_DIR)/rootfs.$(1): FAKEROOT_SCRIPT=$$(ROOTFS_$(2)_DIR)/fakeroot
-$$(BINARIES_DIR)/rootfs.$(1): target-finalize $$(ROOTFS_$(2)_DEPENDENCIES)
+$$(BINARIES_DIR)/rootfs.$(1): target-finalize
+$$(BINARIES_DIR)/rootfs.$(1): $$(ROOTFS_$(2)_DEPENDENCIES)
 	@$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
 	rm -rf $(FS_DIR) $$(ROOTFS_$(2)_DIR)
 	mkdir -p $(FS_DIR) $$(ROOTFS_$(2)_DIR)
-- 
2.14.1

[Buildroot] [PATCH 09/15 v3] fs: introduce per-rootfs TARGET_DIR variable

[Yann E. MORIN]

... which for now still points to the base target directory, but this is
a step forward.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 Makefile     | 4 +++-
 fs/common.mk | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 439aaa9cb0..0c872568d5 100644
--- a/Makefile
+++ b/Makefile
@@ -215,8 +215,10 @@ BR_GRAPH_OUT := $(or $(BR2_GRAPH_OUT),pdf)
 
 BUILD_DIR := $(BASE_DIR)/build
 BINARIES_DIR := $(BASE_DIR)/images
+# The target directory is common to all packages,
+# but there is one that is specific to each filesystem.
 BASE_TARGET_DIR := $(BASE_DIR)/target
-TARGET_DIR = $(BASE_TARGET_DIR)
+TARGET_DIR = $(if $(ROOTFS),$(ROOTFS_$(ROOTFS)_TARGET_DIR),$(BASE_TARGET_DIR))
 # initial definition so that 'make clean' works for most users, even without
 # .config. HOST_DIR will be overwritten later when .config is included.
 HOST_DIR := $(BASE_DIR)/host
diff --git a/fs/common.mk b/fs/common.mk
index cd132cdbc7..363798bb20 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -40,6 +40,7 @@ define inner-rootfs
 
 ROOTFS_$(2)_NAME = $(1)
 ROOTFS_$(2)_DIR = $$(FS_DIR)/$(1)
+ROOTFS_$(2)_TARGET_DIR = $$(BASE_TARGET_DIR)
 
 # extra deps
 ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs \
-- 
2.14.1

[Buildroot] [PATCH 10/15 v3] fs: new intermediate rootfs-common to gather common dependencies

[Yann E. MORIN]

Before we can create an intermediate tarball for all filesystems, we
nedd to move the common dependencies needed to generate that
intermediate tarball, rather than levae those dependencies to each
filesystem.

So, we introduce rootfs-common, which gathers all those common
dependencies.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/fs/common.mk b/fs/common.mk
index 363798bb20..3505a32e5d 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -34,6 +34,24 @@ ROOTFS_DEVICE_TABLES = $(call qstrip,$(BR2_ROOTFS_DEVICE_TABLE) \
 USERS_TABLE = $(FS_DIR)/users_table.txt
 ROOTFS_USERS_TABLES = $(call qstrip,$(BR2_ROOTFS_USERS_TABLES))
 
+ROOTFS_COMMON_TAR = $(FS_DIR)/rootfs.common.tar
+
+.PHONY: rootfs-common
+rootfs-common: $(ROOTFS_COMMON_TAR)
+
+ROOTFS_COMMON_DEPENDENCIES = \
+	host-fakeroot host-makedevs \
+	$(if $(PACKAGES_USERS)$(ROOTFS_USERS_TABLES),host-mkpasswd)
+
+.PHONY: $(ROOTFS_COMMON_TAR)
+# When doing the common tarball, we're not really doing a rootfs.
+$(ROOTFS_COMMON_TAR): ROOTFS=
+$(ROOTFS_COMMON_TAR): $(ROOTFS_COMMON_DEPENDENCIES) target-finalize
+	@:
+
+rootfs-common-show-depends:
+	@echo $(ROOTFS_COMMON_DEPENDENCIES)
+
 # Since this function will be called from within an $(eval ...)
 # all variable references except the arguments must be $$-quoted.
 define inner-rootfs
@@ -42,9 +60,7 @@ ROOTFS_$(2)_NAME = $(1)
 ROOTFS_$(2)_DIR = $$(FS_DIR)/$(1)
 ROOTFS_$(2)_TARGET_DIR = $$(BASE_TARGET_DIR)
 
-# extra deps
-ROOTFS_$(2)_DEPENDENCIES += host-fakeroot host-makedevs \
-	$$(if $$(PACKAGES_USERS)$$(ROOTFS_USERS_TABLES),host-mkpasswd)
+ROOTFS_$(2)_DEPENDENCIES += rootfs-common
 
 ifeq ($$(BR2_TARGET_ROOTFS_$(2)_GZIP),y)
 ROOTFS_$(2)_COMPRESS_EXT = .gz
@@ -77,7 +93,6 @@ endif
 
 $$(BINARIES_DIR)/rootfs.$(1): ROOTFS=$(2)
 $$(BINARIES_DIR)/rootfs.$(1): FAKEROOT_SCRIPT=$$(ROOTFS_$(2)_DIR)/fakeroot
-$$(BINARIES_DIR)/rootfs.$(1): target-finalize
 $$(BINARIES_DIR)/rootfs.$(1): $$(ROOTFS_$(2)_DEPENDENCIES)
 	@$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
 	rm -rf $(FS_DIR) $$(ROOTFS_$(2)_DIR)
-- 
2.14.1

[Buildroot] [PATCH 00/15 v3] fs: support parallel filesystems build

[Yann E. MORIN]

Hello All!

TL;DR: filsystems are not parallel-safe, so we generate an intermediate
tarball from target/, which then serves as input for the various
filesystems; we also run pre-gen hooks under fakeroot, and get rid of
post-fs package-provided hooks.

Full explanations follows:

Currently, the filesystems are all built sequentially. This is
unfortunate, because each filesystem commands may modify the content of
target/, e.g. when systemd is enabled on a read-only filesystem, or cpio
to add its /init stub. cpio even changes the packages-supplied list of
devices, to shoe-horn /dev/console.

This has two issues:

  - the content of the generated rootfs depends on whether other
    filesystems are enabled or not;

  - it is not posible to build filesystems in parallel, becasue of
    concurency on target/.

A further (less critical) issue is that the fakeroot stuff is repeated
again and again for each filesystem: generating the user, groups and
device tables...

The only sane solution is to have each filesystem operate on its own
copy of target/, which is assembled once uner fakeroot and then re-used
by each filesystem, so that they can each do their own last-ditch tweaks
to the content of the filesystem before generating the image.

So we introduce an intermediate tarball that is assembled under
fakeroot, to run all the common fakeroot-needing setup. Each filesytem
then run a very simple fakeroot script that extracts that intermediate
tarball in a transient per-filesystem location and use that as input to
their image generation commands. This also allows to run the fakerooted,
common actions only once.

Today, the filesystems pre-/post-gen hooks are called very early,
outside of the fakeroot script, which means they can not do modifications
that require root access, like creating device nodes.

And as a last straw, the recently added (in August 2017) support for
package-provided pre-/post-fs hooks, and used solely by systemd on a
read-only filesystem, means that the target directory may be left in an
inconsistent state should the filesystem image generator fails, because
those pre-/post-fs hooks really need an atomic seciont which we can not
provide.

For this reason, the only solution is to act on a copy of target/.

So, now that all the issues have been exposed and their solutions
explained, the series is split as such:

  - first we move the filesystemns pre-gen hooks under fakeroot, and we
    fix the cpio filesystem to correctly create its device node; the
    post-gen hooks need not run under fakeroot, because they are only
    supposed to act on the generated image, which does not require root
    rights (and modifying target/ after the iamge has been generated is
    pointless anyway;

  - then we introduce per-rootfs variables (per rootfs name and directory,
    and per rootfs target directory) and fakeroot scripts, and off-load
    the common dependencies to a common rule;

  - then we eventually split the filesystem generation in two, introducing
    the intermediate tarball, which filesystems re-use as input to
    generate their images. We also introduce a copy of target/ from which
    to generate the intermediate tarball, which allows us to get rid of
    the post-fs hooks provided by packages.

  - finally, the last patchs adds the removal of the transient copies of
    the target/ directory, as well as of the intermediate tarball, since
    they all can be rather large and are not meant to be user-visible.

As an aside, the pre-gen and post-gen hooks are not totally symetric
anymore: the pre-gen hooks run under fakeroot while the post-gen hooks
do not. A brief discussion on IRC suggested a change of name, since they
are no longer symetric, but in the end, I left this renaming as a
further exercise.

The series still runs the test suite with a few failures, that are
already present on master it based on:

  - master: https://gitlab.com/buildroot.org/buildroot/pipelines/17079175
  - series: https://gitlab.com/ymorin/buildroot-ci/pipelines/17080587

Note: the entire series was tested by Matthew, but I carry his tested-by
tag only on the last commit.

Changes v2 -> v3:
  - none; rebased on master

Changes v1 -> v2:
  - drop applied patches
  - write a complete commit log for now-first patch  (Thomas)


Regards,
Yann E. MORIN.


The following changes since commit 7b3bb51809517ea045dedc8544db51fc0515d5b4

  tpm2-tss: new package (2018-03-08 09:58:02 +0100)


are available in the git repository at:

  git://git.buildroot.org/~ymorin/git/buildroot.git

for you to fetch changes up to fa877c5eefa966867f045f6b37a9e568eb019dbd

  fs: remove intermediate artefacts (2018-03-08 21:02:14 +0100)


----------------------------------------------------------------
Yann E. MORIN (15):
      fs: run filesystem hooks under fakeroot
      fs/cpio: don't extend packages' permissions table
      fs/iso9660: rename internal variable
      core: introduce intermediate BASE_TARGET_DIR variable
      fs: set per-rootfs variable name
      fs: define per-rootfs directory
      fs: use a per-rootfs fakeroot script
      fs: split per-rootfs dependency line
      fs: introduce per-rootfs TARGET_DIR variable
      fs: new intermediate rootfs-common to gather common dependencies
      fs: introduce a macro for reproducible command
      fs: use a common tarball as base for the other filesystems
      fs: run packages' filesystem hooks in a copy of target/
      fs: get rid of package-provided post-fs hooks
      fs: remove intermediate artefacts

 Makefile                                           |  17 ++-
 fs/common.mk                                       | 114 +++++++++++++++------
 fs/cpio/cpio.mk                                    |   5 +-
 fs/iso9660/iso9660.mk                              |  30 +++---
 package/pkg-generic.mk                             |   6 +-
 .../skeleton-init-systemd/skeleton-init-systemd.mk |   6 --
 support/testing/tests/core/test_post_scripts.py    |  23 +++--
 7 files changed, 128 insertions(+), 73 deletions(-)

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 223 225 172 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'

[Buildroot] [PATCH 11/15 v3] fs: introduce a macro for reproducible command

[Yann E. MORIN]

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/fs/common.mk b/fs/common.mk
index 3505a32e5d..05ae107192 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -34,6 +34,12 @@ ROOTFS_DEVICE_TABLES = $(call qstrip,$(BR2_ROOTFS_DEVICE_TABLE) \
 USERS_TABLE = $(FS_DIR)/users_table.txt
 ROOTFS_USERS_TABLES = $(call qstrip,$(BR2_ROOTFS_USERS_TABLES))
 
+ifeq ($(BR2_REPRODUCIBLE),y)
+define ROOTFS_REPRODUCIBLE
+	find $(TARGET_DIR) -print0 | xargs -0 -r touch -hd @$(SOURCE_DATE_EPOCH)
+endef
+endif
+
 ROOTFS_COMMON_TAR = $(FS_DIR)/rootfs.common.tar
 
 .PHONY: rootfs-common
@@ -120,9 +126,7 @@ endif
 		echo $$(EXTRA_ENV) $$(s) $$(TARGET_DIR) $$(BR2_ROOTFS_POST_SCRIPT_ARGS) >> $$(FAKEROOT_SCRIPT)$$(sep))
 	$$(foreach hook,$$(ROOTFS_PRE_CMD_HOOKS),\
 		$$(call PRINTF,$$($$(hook))) >> $$(FAKEROOT_SCRIPT)$$(sep))
-ifeq ($$(BR2_REPRODUCIBLE),y)
-	echo "find $$(TARGET_DIR) -print0 | xargs -0 -r touch -hd @$$(SOURCE_DATE_EPOCH)" >> $$(FAKEROOT_SCRIPT)
-endif
+	$$(call PRINTF,$$(ROOTFS_REPRODUCIBLE)) >> $$(FAKEROOT_SCRIPT)
 	$$(call PRINTF,$$(ROOTFS_$(2)_CMD)) >> $$(FAKEROOT_SCRIPT)
 	$$(foreach hook,$$(ROOTFS_POST_CMD_HOOKS),\
 		$$(call PRINTF,$$($$(hook))) >> $$(FAKEROOT_SCRIPT)$$(sep))
-- 
2.14.1

[Buildroot] [PATCH 12/15 v3] fs: use a common tarball as base for the other filesystems

[Yann E. MORIN]

Currently, some filesystems may want to tweak the content of the target
directory, create special device nodes etc... This all means that:

  - the content of the target directory for a specific filesystems may
    depend on whether another filesystem is enabled or not; for example,
    cpio will create a /init script or symlink and a /dev/console node;

  - the filesysems can not be built in parallel, because they may change
    the content of the target directory wil another is being assembled.

Furthermore, the same fakeroot script is executed over-and-over-again
for each filesystem, to create the device nodes, the users and their
homes and files, and setting permissions...

We introduce an intermediate tarball, for which we do the full fakeroot
shebang.

That tarball then serves as the base for the other filesystems, with a
very simple fakeroot script that untars the common tarball, and calls
the actual filesystem image generator on that.

Note that we use a very simple tar command to generate the intermediate
tarball, because we are not concerned with reproducibility of the
archive itself (only of the archived files).

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk | 75 +++++++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 47 insertions(+), 28 deletions(-)

diff --git a/fs/common.mk b/fs/common.mk
index 05ae107192..f3d42519f6 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -42,6 +42,19 @@ endif
 
 ROOTFS_COMMON_TAR = $(FS_DIR)/rootfs.common.tar
 
+# Command to create the common tarball from the base target directory.
+define ROOTFS_COMMON_TAR_CMD
+	tar cf $(ROOTFS_COMMON_TAR) --numeric-owner \
+		--exclude=$(notdir $(TARGET_DIR_WARNING_FILE)) \
+		-C $(TARGET_DIR) .
+endef
+
+# Command to extract the common tarball into the per-rootfs target directory
+define ROOTFS_COMMON_UNTAR_CMD
+	mkdir -p $(TARGET_DIR)
+	tar xf $(ROOTFS_COMMON_TAR) -C $(TARGET_DIR)
+endef
+
 .PHONY: rootfs-common
 rootfs-common: $(ROOTFS_COMMON_TAR)
 
@@ -49,11 +62,39 @@ ROOTFS_COMMON_DEPENDENCIES = \
 	host-fakeroot host-makedevs \
 	$(if $(PACKAGES_USERS)$(ROOTFS_USERS_TABLES),host-mkpasswd)
 
-.PHONY: $(ROOTFS_COMMON_TAR)
 # When doing the common tarball, we're not really doing a rootfs.
 $(ROOTFS_COMMON_TAR): ROOTFS=
+$(ROOTFS_COMMON_TAR): FAKEROOT_SCRIPT=$(FS_DIR)/fakeroot.fs
 $(ROOTFS_COMMON_TAR): $(ROOTFS_COMMON_DEPENDENCIES) target-finalize
-	@:
+	@$(call MESSAGE,"Generating common rootfs tarball")
+	rm -rf $(FS_DIR)
+	mkdir -p $(FS_DIR)
+	echo '#!/bin/sh' > $(FAKEROOT_SCRIPT)
+	echo "set -e" >> $(FAKEROOT_SCRIPT)
+	echo "chown -h -R 0:0 $(TARGET_DIR)" >> $(FAKEROOT_SCRIPT)
+ifneq ($(ROOTFS_USERS_TABLES),)
+	cat $(ROOTFS_USERS_TABLES) >> $(USERS_TABLE)
+endif
+	$(call PRINTF,$(PACKAGES_USERS)) >> $(USERS_TABLE)
+	PATH=$(BR_PATH) $(TOPDIR)/support/scripts/mkusers $(USERS_TABLE) $(TARGET_DIR) >> $(FAKEROOT_SCRIPT)
+ifneq ($(ROOTFS_DEVICE_TABLES),)
+	cat $(ROOTFS_DEVICE_TABLES) > $(FULL_DEVICE_TABLE)
+ifeq ($(BR2_ROOTFS_DEVICE_CREATION_STATIC),y)
+	$(call PRINTF,$(PACKAGES_DEVICES_TABLE)) >> $(FULL_DEVICE_TABLE)
+endif
+endif
+	$(call PRINTF,$(PACKAGES_PERMISSIONS_TABLE)) >> $(FULL_DEVICE_TABLE)
+	echo "$(HOST_DIR)/bin/makedevs -d $(FULL_DEVICE_TABLE) $(TARGET_DIR)" >> $(FAKEROOT_SCRIPT)
+	$(foreach s,$(call qstrip,$(BR2_ROOTFS_POST_FAKEROOT_SCRIPT)),\
+		echo "echo '$(TERM_BOLD)>>>   Executing fakeroot script $(s)$(TERM_RESET)'" >> $(FAKEROOT_SCRIPT); \
+		echo $(EXTRA_ENV) $(s) $(TARGET_DIR) $(BR2_ROOTFS_POST_SCRIPT_ARGS) >> $(FAKEROOT_SCRIPT)$(sep))
+	$(foreach hook,$(ROOTFS_PRE_CMD_HOOKS),\
+		$(call PRINTF,$($(hook))) >> $(FAKEROOT_SCRIPT)$(sep))
+	$(call PRINTF,$(ROOTFS_COMMON_TAR_CMD)) >> $(FAKEROOT_SCRIPT)
+	$(foreach hook,$(ROOTFS_POST_CMD_HOOKS),\
+		$(call PRINTF,$($(hook))) >> $(FAKEROOT_SCRIPT)$(sep))
+	chmod a+x $(FAKEROOT_SCRIPT)
+	PATH=$(BR_PATH) $(HOST_DIR)/bin/fakeroot -- $(FAKEROOT_SCRIPT)
 
 rootfs-common-show-depends:
 	@echo $(ROOTFS_COMMON_DEPENDENCIES)
@@ -64,7 +105,7 @@ define inner-rootfs
 
 ROOTFS_$(2)_NAME = $(1)
 ROOTFS_$(2)_DIR = $$(FS_DIR)/$(1)
-ROOTFS_$(2)_TARGET_DIR = $$(BASE_TARGET_DIR)
+ROOTFS_$(2)_TARGET_DIR = $$(ROOTFS_$(2)_DIR)/target
 
 ROOTFS_$(2)_DEPENDENCIES += rootfs-common
 
@@ -101,39 +142,17 @@ $$(BINARIES_DIR)/rootfs.$(1): ROOTFS=$(2)
 $$(BINARIES_DIR)/rootfs.$(1): FAKEROOT_SCRIPT=$$(ROOTFS_$(2)_DIR)/fakeroot
 $$(BINARIES_DIR)/rootfs.$(1): $$(ROOTFS_$(2)_DEPENDENCIES)
 	@$$(call MESSAGE,"Generating root filesystem image rootfs.$(1)")
-	rm -rf $(FS_DIR) $$(ROOTFS_$(2)_DIR)
-	mkdir -p $(FS_DIR) $$(ROOTFS_$(2)_DIR)
+	rm -rf $$(ROOTFS_$(2)_DIR)
+	mkdir -p $$(ROOTFS_$(2)_DIR)
 	echo '#!/bin/sh' > $$(FAKEROOT_SCRIPT)
 	echo "set -e" >> $$(FAKEROOT_SCRIPT)
+	$$(call PRINTF,$$(ROOTFS_COMMON_UNTAR_CMD)) >> $$(FAKEROOT_SCRIPT)
 	$$(foreach hook,$$(ROOTFS_$(2)_PRE_GEN_HOOKS),\
 		$$(call PRINTF,$$($$(hook))) >> $$(FAKEROOT_SCRIPT)$$(sep))
-	echo "chown -h -R 0:0 $$(TARGET_DIR)" >> $$(FAKEROOT_SCRIPT)
-ifneq ($$(ROOTFS_USERS_TABLES),)
-	cat $$(ROOTFS_USERS_TABLES) >> $$(USERS_TABLE)
-endif
-	$$(call PRINTF,$$(PACKAGES_USERS)) >> $$(USERS_TABLE)
-	PATH=$$(BR_PATH) $$(TOPDIR)/support/scripts/mkusers $$(USERS_TABLE) $$(TARGET_DIR) >> $$(FAKEROOT_SCRIPT)
-ifneq ($$(ROOTFS_DEVICE_TABLES),)
-	cat $$(ROOTFS_DEVICE_TABLES) > $$(FULL_DEVICE_TABLE)
-ifeq ($$(BR2_ROOTFS_DEVICE_CREATION_STATIC),y)
-	$$(call PRINTF,$$(PACKAGES_DEVICES_TABLE)) >> $$(FULL_DEVICE_TABLE)
-endif
-endif
-	$$(call PRINTF,$$(PACKAGES_PERMISSIONS_TABLE)) >> $$(FULL_DEVICE_TABLE)
-	echo "$$(HOST_DIR)/bin/makedevs -d $$(FULL_DEVICE_TABLE) $$(TARGET_DIR)" >> $$(FAKEROOT_SCRIPT)
-	$$(foreach s,$$(call qstrip,$$(BR2_ROOTFS_POST_FAKEROOT_SCRIPT)),\
-		echo "echo '$$(TERM_BOLD)>>>   Executing fakeroot script $$(s)$$(TERM_RESET)'" >> $$(FAKEROOT_SCRIPT); \
-		echo $$(EXTRA_ENV) $$(s) $$(TARGET_DIR) $$(BR2_ROOTFS_POST_SCRIPT_ARGS) >> $$(FAKEROOT_SCRIPT)$$(sep))
-	$$(foreach hook,$$(ROOTFS_PRE_CMD_HOOKS),\
-		$$(call PRINTF,$$($$(hook))) >> $$(FAKEROOT_SCRIPT)$$(sep))
 	$$(call PRINTF,$$(ROOTFS_REPRODUCIBLE)) >> $$(FAKEROOT_SCRIPT)
 	$$(call PRINTF,$$(ROOTFS_$(2)_CMD)) >> $$(FAKEROOT_SCRIPT)
-	$$(foreach hook,$$(ROOTFS_POST_CMD_HOOKS),\
-		$$(call PRINTF,$$($$(hook))) >> $$(FAKEROOT_SCRIPT)$$(sep))
 	chmod a+x $$(FAKEROOT_SCRIPT)
-	rm -f $$(TARGET_DIR_WARNING_FILE)
 	PATH=$$(BR_PATH) $$(HOST_DIR)/bin/fakeroot -- $$(FAKEROOT_SCRIPT)
-	$$(INSTALL) -m 0644 support/misc/target-dir-warning.txt $$(TARGET_DIR_WARNING_FILE)
 ifneq ($$(ROOTFS_$(2)_COMPRESS_CMD),)
 	PATH=$$(BR_PATH) $$(ROOTFS_$(2)_COMPRESS_CMD) $$@ > $$@$$(ROOTFS_$(2)_COMPRESS_EXT)
 endif
-- 
2.14.1

[Buildroot] [PATCH 13/15 v3] fs: run packages' filesystem hooks in a copy of target/

[Yann E. MORIN]

Currently, some packages may regigter hooks to be run just before and
judt after the generic tarball image is generated, because they need to
prepare the filesystem for read-only or read-write operation.

However, this means that, if any of the hooks or the image generation
fails, the target directory is left in a dangling, inconsistent state.

We fix that by doing a copy of target/, run the hooks on that copy,
generate the generic tarball image out of that, and get rid of the copy.

This way, we can guarantee consistency of the target directory, and we
can even ditch support for post-fs hooks (those that restore target/).

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk                                    |  7 +++++--
 support/testing/tests/core/test_post_scripts.py | 23 +++++++++++++----------
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/fs/common.mk b/fs/common.mk
index f3d42519f6..8bb26f1146 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -58,17 +58,20 @@ endef
 .PHONY: rootfs-common
 rootfs-common: $(ROOTFS_COMMON_TAR)
 
+# Emulate being in a filesystem, so that we can have our own TARGET_DIR.
+ROOTFS_COMMON_TARGET_DIR = $(FS_DIR)/target
+
 ROOTFS_COMMON_DEPENDENCIES = \
 	host-fakeroot host-makedevs \
 	$(if $(PACKAGES_USERS)$(ROOTFS_USERS_TABLES),host-mkpasswd)
 
-# When doing the common tarball, we're not really doing a rootfs.
-$(ROOTFS_COMMON_TAR): ROOTFS=
+$(ROOTFS_COMMON_TAR): ROOTFS=COMMON
 $(ROOTFS_COMMON_TAR): FAKEROOT_SCRIPT=$(FS_DIR)/fakeroot.fs
 $(ROOTFS_COMMON_TAR): $(ROOTFS_COMMON_DEPENDENCIES) target-finalize
 	@$(call MESSAGE,"Generating common rootfs tarball")
 	rm -rf $(FS_DIR)
 	mkdir -p $(FS_DIR)
+	rsync -au $(BASE_TARGET_DIR)/ $(TARGET_DIR)
 	echo '#!/bin/sh' > $(FAKEROOT_SCRIPT)
 	echo "set -e" >> $(FAKEROOT_SCRIPT)
 	echo "chown -h -R 0:0 $(TARGET_DIR)" >> $(FAKEROOT_SCRIPT)
diff --git a/support/testing/tests/core/test_post_scripts.py b/support/testing/tests/core/test_post_scripts.py
index edb339d8c4..a0e5b6b454 100644
--- a/support/testing/tests/core/test_post_scripts.py
+++ b/support/testing/tests/core/test_post_scripts.py
@@ -18,17 +18,17 @@ class TestPostScripts(infra.basetest.BRTest):
                    infra.filepath("tests/core/post-fakeroot.sh"),
                    infra.filepath("tests/core/post-image.sh"))
 
-    def check_post_log_file(self, path, what):
+    def check_post_log_file(self, f, what, target_dir):
         lines = {}
-        with open(path, 'rb') as csvfile:
+        with open(os.path.join(self.builddir, "build", f), 'rb') as csvfile:
             r = csv.reader(csvfile, delimiter=',')
             for row in r:
                 lines[row[0]] = row[1]
 
-        self.assertEqual(lines["arg1"], os.path.join(self.builddir, what))
+        self.assertEqual(lines["arg1"], what)
         self.assertEqual(lines["arg2"], "foobar")
         self.assertEqual(lines["arg3"], "baz")
-        self.assertEqual(lines["TARGET_DIR"], os.path.join(self.builddir, "target"))
+        self.assertEqual(lines["TARGET_DIR"], target_dir)
         self.assertEqual(lines["BUILD_DIR"], os.path.join(self.builddir, "build"))
         self.assertEqual(lines["HOST_DIR"], os.path.join(self.builddir, "host"))
         staging = os.readlink(os.path.join(self.builddir, "staging"))
@@ -37,9 +37,12 @@ class TestPostScripts(infra.basetest.BRTest):
         self.assertEqual(lines["BR2_CONFIG"], os.path.join(self.builddir, ".config"))
 
     def test_run(self):
-        f = os.path.join(self.builddir, "build", "post-build.log")
-        self.check_post_log_file(f, "target")
-        f = os.path.join(self.builddir, "build", "post-fakeroot.log")
-        self.check_post_log_file(f, "target")
-        f = os.path.join(self.builddir, "build", "post-image.log")
-        self.check_post_log_file(f, "images")
+        self.check_post_log_file("post-build.log",
+                                 os.path.join(self.builddir, "target"),
+                                 os.path.join(self.builddir, "target"))
+        self.check_post_log_file("post-fakeroot.log",
+                                 os.path.join(self.builddir, "build/buildroot-fs/target"),
+                                 os.path.join(self.builddir, "build/buildroot-fs/target"))
+        self.check_post_log_file("post-image.log",
+                                 os.path.join(self.builddir, "images"),
+                                 os.path.join(self.builddir, "target"))
-- 
2.14.1

[Buildroot] [PATCH 14/15 v3] fs: get rid of package-provided post-fs hooks

[Yann E. MORIN]

Now that the pre-fs ones are run on a transient copy of target/, the
post-fs hooks are no longer needed because we no longer need to restore
the target/ directory as it is only a internal copy.

Remove support for the post-fs hooks, and update the sole package using
them.

Unlike for the pre-/post-gen hooks, we do not add a legacy check because
this was mostly a purely-internal detail that was never really exposed
nor documented.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
---
 fs/common.mk                                           | 2 --
 package/pkg-generic.mk                                 | 2 --
 package/skeleton-init-systemd/skeleton-init-systemd.mk | 6 ------
 3 files changed, 10 deletions(-)

diff --git a/fs/common.mk b/fs/common.mk
index 8bb26f1146..bacc98ec68 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -94,8 +94,6 @@ endif
 	$(foreach hook,$(ROOTFS_PRE_CMD_HOOKS),\
 		$(call PRINTF,$($(hook))) >> $(FAKEROOT_SCRIPT)$(sep))
 	$(call PRINTF,$(ROOTFS_COMMON_TAR_CMD)) >> $(FAKEROOT_SCRIPT)
-	$(foreach hook,$(ROOTFS_POST_CMD_HOOKS),\
-		$(call PRINTF,$($(hook))) >> $(FAKEROOT_SCRIPT)$(sep))
 	chmod a+x $(FAKEROOT_SCRIPT)
 	PATH=$(BR_PATH) $(HOST_DIR)/bin/fakeroot -- $(FAKEROOT_SCRIPT)
 
diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk
index 1a637ea288..c0f100354e 100644
--- a/package/pkg-generic.mk
+++ b/package/pkg-generic.mk
@@ -638,7 +638,6 @@ $(2)_PRE_LEGAL_INFO_HOOKS       ?=
 $(2)_POST_LEGAL_INFO_HOOKS      ?=
 $(2)_TARGET_FINALIZE_HOOKS      ?=
 $(2)_ROOTFS_PRE_CMD_HOOKS       ?=
-$(2)_ROOTFS_POST_CMD_HOOKS      ?=
 
 # human-friendly targets and target sequencing
 $(1):			$(1)-install
@@ -949,7 +948,6 @@ PACKAGES_USERS += $$($(2)_USERS)$$(sep)
 endif
 TARGET_FINALIZE_HOOKS += $$($(2)_TARGET_FINALIZE_HOOKS)
 ROOTFS_PRE_CMD_HOOKS += $$($(2)_ROOTFS_PRE_CMD_HOOKS)
-ROOTFS_POST_CMD_HOOKS += $$($(2)_ROOTFS_POST_CMD_HOOKS)
 
 ifeq ($$($(2)_SITE_METHOD),svn)
 DL_TOOLS_DEPENDENCIES += svn
diff --git a/package/skeleton-init-systemd/skeleton-init-systemd.mk b/package/skeleton-init-systemd/skeleton-init-systemd.mk
index ff64205cbe..5d6b716f7a 100644
--- a/package/skeleton-init-systemd/skeleton-init-systemd.mk
+++ b/package/skeleton-init-systemd/skeleton-init-systemd.mk
@@ -55,12 +55,6 @@ define SKELETON_INIT_SYSTEMD_PRE_ROOTFS_VAR
 endef
 SKELETON_INIT_SYSTEMD_ROOTFS_PRE_CMD_HOOKS += SKELETON_INIT_SYSTEMD_PRE_ROOTFS_VAR
 
-define SKELETON_INIT_SYSTEMD_POST_ROOTFS_VAR
-	rm -rf $(TARGET_DIR)/var
-	mv $(TARGET_DIR)/usr/share/factory/var $(TARGET_DIR)/var
-endef
-SKELETON_INIT_SYSTEMD_ROOTFS_POST_CMD_HOOKS += SKELETON_INIT_SYSTEMD_POST_ROOTFS_VAR
-
 endif
 
 define SKELETON_INIT_SYSTEMD_INSTALL_TARGET_CMDS
-- 
2.14.1

[Buildroot] [PATCH 15/15 v3] fs: remove intermediate artefacts

[Yann E. MORIN]

Each of the intermediate, per-rootfs target directories, as well as the
intermediate tarball, can take quite some place, and is mostly a
duplication of what's already in target/. The only delta, if any, would
be the tweaks made by the filesystem image generations, but those tweaks
are most probably only meaningfull when seen as root.

We normally do not remove intermediate files, but those can be quite
large, and are not directly usable by, nor accessible to the user.
So, get rid of them once the filesystem has been generated.

This does not need to be done in fakeroot.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
Tested-by: Matt Weber <matthew.weber@rockwellcollins.com>
---
 Makefile     | 1 +
 fs/common.mk | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/Makefile b/Makefile
index 0c872568d5..cc3f983655 100644
--- a/Makefile
+++ b/Makefile
@@ -769,6 +769,7 @@ endif
 
 .PHONY: target-post-image
 target-post-image: $(TARGETS_ROOTFS) target-finalize
+	@rm -f $(ROOTFS_COMMON_TAR)
 	@$(foreach s, $(call qstrip,$(BR2_ROOTFS_POST_IMAGE_SCRIPT)), \
 		$(call MESSAGE,"Executing post-image script $(s)"); \
 		$(EXTRA_ENV) $(s) $(BINARIES_DIR) $(call qstrip,$(BR2_ROOTFS_POST_SCRIPT_ARGS))$(sep))
diff --git a/fs/common.mk b/fs/common.mk
index bacc98ec68..2ee130ee08 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -96,6 +96,7 @@ endif
 	$(call PRINTF,$(ROOTFS_COMMON_TAR_CMD)) >> $(FAKEROOT_SCRIPT)
 	chmod a+x $(FAKEROOT_SCRIPT)
 	PATH=$(BR_PATH) $(HOST_DIR)/bin/fakeroot -- $(FAKEROOT_SCRIPT)
+	$(Q)rm -rf $(TARGET_DIR)
 
 rootfs-common-show-depends:
 	@echo $(ROOTFS_COMMON_DEPENDENCIES)
@@ -154,6 +155,7 @@ $$(BINARIES_DIR)/rootfs.$(1): $$(ROOTFS_$(2)_DEPENDENCIES)
 	$$(call PRINTF,$$(ROOTFS_$(2)_CMD)) >> $$(FAKEROOT_SCRIPT)
 	chmod a+x $$(FAKEROOT_SCRIPT)
 	PATH=$$(BR_PATH) $$(HOST_DIR)/bin/fakeroot -- $$(FAKEROOT_SCRIPT)
+	$(Q)rm -rf $$(TARGET_DIR)
 ifneq ($$(ROOTFS_$(2)_COMPRESS_CMD),)
 	PATH=$$(BR_PATH) $$(ROOTFS_$(2)_COMPRESS_CMD) $$@ > $$@$$(ROOTFS_$(2)_COMPRESS_EXT)
 endif
-- 
2.14.1

[Buildroot] [PATCH 00/15 v3] fs: support parallel filesystems build

[Jan Kundrát]

On ?tvrtek 8. b?ezna 2018 22:15:35 CET, Yann E. MORIN wrote:
> TL;DR: filsystems are not parallel-safe, so we generate an intermediate
> tarball from target/, which then serves as input for the various
> filesystems; we also run pre-gen hooks under fakeroot, and get rid of
> post-fs package-provided hooks.

I like this -- it unbreaks a top-level parallel build for me. I no longer 
have to run `make -j32 target-finalize && make` :).

It also looks like something which will eventually enable genimage.sh to be 
run as a proper make-level target. That's important for me because I run 
genimage twice in a row; my second invocation takes the just-built image, 
puts that into a squashfs archive because of its compression, and uses the 
result to build another image for re-flashing the system via a USB stick.

Just build-tested for now, but I'll be happy to send a proper Tested-By 
once I verify that my board still boots with the result.

With kind regards,
Jan