[PATCH] siteinfo/autotools: Ensure task checksums reflect site files

* [PATCH] siteinfo/autotools: Ensure task checksums reflect site files
@ 2021-09-21 12:15 Richard Purdie
  2022-08-17  9:10 ` [OE-core] " Frieder Schrempf
  0 siblings, 1 reply; 3+ messages in thread
From: Richard Purdie @ 2021-09-21 12:15 UTC (permalink / raw)
  To: openembedded-core

Currently, if you change the site files, nothing rebuilds since they are
not accounted for in task checksums. They could/should be through the
file-checksums task flag. We need to cache all the files looked for,
whether the exist or not so that if they do exist and didn't,
the checksum also changes.

This gets complicated by the need to clean out hardcoded build
paths from the variable and that other layers can have site files.

This patch adds this functionality. A new variable, SITEINFO_PATHVARS
is added which controls which substitutions to make on the file-checksum
values to remove the hardcoded paths. Layers adding site files will need
to set this to a variable that has the layer path in it and is excluded
from task hashes (COREBASE is the one the core layer uses).

This patch will cause yocto-check-layer to fail for some layers
where site files are added yet the layer isn't a machine specific layer.
This is arguable correct since these additional site files apply to
all recipes and things from a layer like core could be changed by such
changes so it is right they should rebuild. There is a determinism issue
potentially there if not. meta-openembedded does have some such references
but looking at them they should move to core or likely just be removed as
most look obsolete anyway.

[YOCTO #13729]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/autotools.bbclass             |  8 ++++++-
 meta/classes/siteinfo.bbclass              | 28 +++++++++++++++++-----
 meta/classes/toolchain-scripts.bbclass     |  5 ++--
 meta/recipes-core/meta/meta-environment.bb |  5 ++++
 meta/recipes-core/meta/meta-ide-support.bb |  6 +++++
 5 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/meta/classes/autotools.bbclass b/meta/classes/autotools.bbclass
index 2c7968e6597..bc0c2ea83e5 100644
--- a/meta/classes/autotools.bbclass
+++ b/meta/classes/autotools.bbclass
@@ -145,7 +145,13 @@ ACLOCALEXTRAPATH:class-target = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 ACLOCALEXTRAPATH:class-nativesdk = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 
 python autotools_aclocals () {
-    d.setVar("CONFIG_SITE", siteinfo_get_files(d, sysrootcache=True))
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=True)
+    d.setVar("CONFIG_SITE", " ".join(sitefiles))
+}
+
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.appendVarFlag("do_configure", "file-checksums", " " + " ".join(searched))
 }
 
 CONFIGURE_FILES = "${S}/configure.in ${S}/configure.ac ${S}/config.h.in ${S}/acinclude.m4 Makefile.am"
diff --git a/meta/classes/siteinfo.bbclass b/meta/classes/siteinfo.bbclass
index 0bd1f36805d..596a40a431d 100644
--- a/meta/classes/siteinfo.bbclass
+++ b/meta/classes/siteinfo.bbclass
@@ -176,17 +176,33 @@ python () {
         bb.fatal("Please add your architecture to siteinfo.bbclass")
 }
 
-def siteinfo_get_files(d, sysrootcache = False):
+# Layers with siteconfig need to add a replacement path to this variable so the
+# sstate isn't path specific
+SITEINFO_PATHVARS = "COREBASE"
+
+def siteinfo_get_files(d, sysrootcache=False):
     sitedata = siteinfo_data(d)
-    sitefiles = ""
+    sitefiles = []
+    searched = []
     for path in d.getVar("BBPATH").split(":"):
         for element in sitedata:
             filename = os.path.join(path, "site", element)
             if os.path.exists(filename):
-                sitefiles += filename + " "
+                searched.append(filename + ":True")
+                sitefiles.append(filename)
+            else:
+                searched.append(filename + ":False")
+
+    # Have to parameterise out hardcoded paths such as COREBASE for the main site files
+    for var in d.getVar("SITEINFO_PATHVARS").split():
+        searched2 = []
+        replace = os.path.normpath(d.getVar(var))
+        for s in searched:
+            searched2.append(s.replace(replace, "${" + var + "}"))
+        searched = searched2
 
     if not sysrootcache:
-        return sitefiles
+        return sitefiles, searched
 
     # Now check for siteconfig cache files in sysroots
     path_siteconfig = d.getVar('SITECONFIG_SYSROOTCACHE')
@@ -195,8 +211,8 @@ def siteinfo_get_files(d, sysrootcache = False):
             if not i.endswith("_config"):
                 continue
             filename = os.path.join(path_siteconfig, i)
-            sitefiles += filename + " "
-    return sitefiles
+            sitefiles.append(filename)
+    return sitefiles, searched
 
 #
 # Make some information available via variables
diff --git a/meta/classes/toolchain-scripts.bbclass b/meta/classes/toolchain-scripts.bbclass
index 479f3b706ea..fb6261c91d0 100644
--- a/meta/classes/toolchain-scripts.bbclass
+++ b/meta/classes/toolchain-scripts.bbclass
@@ -65,6 +65,7 @@ toolchain_create_sdk_env_script () {
 
 # This function creates an environment-setup-script in the TMPDIR which enables
 # a OE-core IDE to integrate with the build tree
+# Caller must ensure CONFIG_SITE is setup
 toolchain_create_tree_env_script () {
 	script=${TMPDIR}/environment-setup-${REAL_MULTIMACH_TARGET_SYS}
 	rm -f $script
@@ -73,7 +74,7 @@ toolchain_create_tree_env_script () {
 	echo 'export PATH=${STAGING_DIR_NATIVE}/usr/bin:${STAGING_BINDIR_TOOLCHAIN}:$PATH' >> $script
 	echo 'export PKG_CONFIG_SYSROOT_DIR=${PKG_CONFIG_SYSROOT_DIR}' >> $script
 	echo 'export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}' >> $script
-	echo 'export CONFIG_SITE="${@siteinfo_get_files(d)}"' >> $script
+	echo 'export CONFIG_SITE="${CONFIG_SITE}"' >> $script
 	echo 'export SDKTARGETSYSROOT=${STAGING_DIR_TARGET}' >> $script
 	echo 'export OECORE_NATIVE_SYSROOT="${STAGING_DIR_NATIVE}"' >> $script
 	echo 'export OECORE_TARGET_SYSROOT="${STAGING_DIR_TARGET}"' >> $script
@@ -161,7 +162,7 @@ EOF
 }
 
 #we get the cached site config in the runtime
-TOOLCHAIN_CONFIGSITE_NOCACHE = "${@siteinfo_get_files(d)}"
+TOOLCHAIN_CONFIGSITE_NOCACHE = "${@' '.join(siteinfo_get_files(d)[0])}"
 TOOLCHAIN_CONFIGSITE_SYSROOTCACHE = "${STAGING_DIR}/${MLPREFIX}${MACHINE}/${target_datadir}/${TARGET_SYS}_config_site.d"
 TOOLCHAIN_NEED_CONFIGSITE_CACHE ??= "virtual/${MLPREFIX}libc ncurses"
 DEPENDS += "${TOOLCHAIN_NEED_CONFIGSITE_CACHE}"
diff --git a/meta/recipes-core/meta/meta-environment.bb b/meta/recipes-core/meta/meta-environment.bb
index 27f01036657..7118fb2aefc 100644
--- a/meta/recipes-core/meta/meta-environment.bb
+++ b/meta/recipes-core/meta/meta-environment.bb
@@ -47,6 +47,11 @@ python do_generate_content() {
 }
 addtask generate_content before do_install after do_compile
 
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.appendVarFlag("do_generate_content", "file-checksums", " " + " ".join(searched))
+}
+
 create_sdk_files() {
 	# Setup site file for external use
 	toolchain_create_sdk_siteconfig ${SDK_OUTPUT}/${SDKPATH}/site-config-${REAL_MULTIMACH_TARGET_SYS}
diff --git a/meta/recipes-core/meta/meta-ide-support.bb b/meta/recipes-core/meta/meta-ide-support.bb
index 37e65a49cac..39317d50e0c 100644
--- a/meta/recipes-core/meta/meta-ide-support.bb
+++ b/meta/recipes-core/meta/meta-ide-support.bb
@@ -12,4 +12,10 @@ do_populate_ide_support () {
   toolchain_create_tree_env_script
 }
 
+python () {
+    sitefiles, searched = siteinfo_get_files(d, sysrootcache=False)
+    d.setVar("CONFIG_SITE", " ".join(sitefiles))
+    d.appendVarFlag("do_populate_ide_support", "file-checksums", " " + " ".join(searched))
+}
+
 addtask populate_ide_support before do_build after do_install
-- 
2.32.0


^ permalink raw reply related	[flat|nested] 3+ messages in thread