[PATCH 01/11] classes/create-spdx-2.2: Use hashfn from BB_TASKDEPDATA instead of MACHINE

[PATCH 01/11] classes/create-spdx-2.2: Use hashfn from BB_TASKDEPDATA instead of MACHINE

[Richard Purdie]

From: Joshua Watt <JPEWhacker@gmail.com>

Enabling the SPDX class and running two builds, one with SDKMACHINE=i686
and then again with SDKMACHINE=x86_64 would lead to errors since the output
was stored per MACHINE and the overlapping files would cause manifest errors.

The hashfn data from bitbake has SSTATE_PKGARCH encoded into it which is how
sstate separates out it's targets and SDPX should be using the same structure.
Therefore switch to using this.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/create-spdx-2.2.bbclass | 61 +++++++++++++++++-----------
 meta/lib/oe/sbom.py                  | 22 ++++++++--
 2 files changed, 57 insertions(+), 26 deletions(-)

diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index e0f62a43a2d..6ec0c1465e7 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}"
+DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx"
 
 # The product name that the CVE database uses.  Defaults to BPN, but may need to
 # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
@@ -337,6 +337,20 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag
 
             package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc)
 
+def collect_deps(d):
+    current_task = "do_" + d.getVar("BB_CURRENTTASK")
+
+    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
+    deps = sorted(set(
+        (dep[0], dep[7]) for dep in taskdepdata.values() if
+            dep[1] == current_task and dep[0] != d.getVar("PN")
+    ))
+
+    return deps
+
+collect_deps[vardepsexclude] += "BB_TASKDEPDATA"
+collect_deps[vardeps] += "DEPENDS"
+
 def collect_dep_recipes(d, doc, spdx_recipe):
     from pathlib import Path
     import oe.sbom
@@ -345,13 +359,9 @@ def collect_dep_recipes(d, doc, spdx_recipe):
     deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
 
     dep_recipes = []
-    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-    deps = sorted(set(
-        dep[0] for dep in taskdepdata.values() if
-            dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN")
-    ))
-    for dep_pn in deps:
-        dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn)
+
+    for dep_pn, dep_hashfn in collect_deps(d):
+        dep_recipe_path = oe.sbom.doc_path_by_hashfn(deploy_dir_spdx, "recipe-" + dep_pn, dep_hashfn)
 
         spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path)
 
@@ -380,8 +390,6 @@ def collect_dep_recipes(d, doc, spdx_recipe):
 
     return dep_recipes
 
-collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA"
-collect_dep_recipes[vardeps] += "DEPENDS"
 
 def collect_dep_sources(d, dep_recipes):
     import oe.sbom
@@ -571,7 +579,7 @@ python do_create_spdx() {
 
     dep_recipes = collect_dep_recipes(d, doc, recipe)
 
-    doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d))
+    doc_sha1 = oe.sbom.write_doc(d, doc, d.getVar("SSTATE_PKGARCH"), "recipes", indent=get_json_indent(d))
     dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
 
     recipe_ref = oe.spdx.SPDXExternalDocumentRef()
@@ -636,7 +644,7 @@ python do_create_spdx() {
 
             add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources)
 
-            oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d))
+            oe.sbom.write_doc(d, package_doc, d.getVar("SSTATE_PKGARCH"), "packages", indent=get_json_indent(d))
 }
 # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
 addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
@@ -667,11 +675,11 @@ def collect_package_providers(d):
 
     taskdepdata = d.getVar("BB_TASKDEPDATA", False)
     deps = sorted(set(
-        dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN")
+        (dep[0], dep[7]) for dep in taskdepdata.values() if dep[0] != d.getVar("PN")
     ))
-    deps.append(d.getVar("PN"))
+    deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME")))
 
-    for dep_pn in deps:
+    for dep_pn, dep_hashfn in deps:
         recipe_data = oe.packagedata.read_pkgdata(dep_pn, d)
 
         for pkg in recipe_data.get("PACKAGES", "").split():
@@ -681,7 +689,7 @@ def collect_package_providers(d):
             rprovides.add(pkg)
 
             for r in rprovides:
-                providers[r] = pkg
+                providers[r] = (pkg, dep_hashfn)
 
     return providers
 
@@ -717,7 +725,7 @@ python do_create_runtime_spdx() {
             if not oe.packagedata.packaged(package, localdata):
                 continue
 
-            pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json")
+            pkg_spdx_path = oe.sbom.doc_path(deploy_dir_spdx, pkg_name, d.getVar("SSTATE_PKGARCH"), "packages")
 
             package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
 
@@ -761,7 +769,7 @@ python do_create_runtime_spdx() {
                 if dep not in providers:
                     continue
 
-                dep = providers[dep]
+                (dep, dep_hashfn) = providers[dep]
 
                 if not oe.packagedata.packaged(dep, localdata):
                     continue
@@ -772,7 +780,7 @@ python do_create_runtime_spdx() {
                 if dep in dep_package_cache:
                     (dep_spdx_package, dep_package_ref) = dep_package_cache[dep]
                 else:
-                    dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg)
+                    dep_path = oe.sbom.doc_path_by_hashfn(deploy_dir_spdx, dep_pkg, dep_hashfn)
 
                     spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path)
 
@@ -800,7 +808,7 @@ python do_create_runtime_spdx() {
                 )
                 seen_deps.add(dep)
 
-            oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d))
+            oe.sbom.write_doc(d, runtime_doc, d.getVar("SSTATE_PKGARCH"), "runtime", spdx_deploy, indent=get_json_indent(d))
 }
 
 addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work
@@ -933,6 +941,8 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
     import tarfile
     import bb.compress.zstd
 
+    providers = collect_package_providers(d)
+
     creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
     deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
     source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
@@ -956,7 +966,12 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
     doc.packages.append(image)
 
     for name in sorted(packages.keys()):
-        pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json")
+        if name not in providers:
+            bb.fatal("Unable to find provider for '%s'" % name)
+
+        pkg_name, pkg_hashfn = providers[name]
+
+        pkg_spdx_path = oe.sbom.doc_path_by_hashfn(deploy_dir_spdx, pkg_name, pkg_hashfn)
         pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
 
         for p in pkg_doc.packages:
@@ -973,7 +988,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
         else:
             bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path))
 
-        runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json")
+        runtime_spdx_path = oe.sbom.doc_path_by_hashfn(deploy_dir_spdx, "runtime-" + name, pkg_hashfn)
         runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path)
 
         runtime_ref = oe.spdx.SPDXExternalDocumentRef()
@@ -1045,7 +1060,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
                     })
 
                 for ref in doc.externalDocumentRefs:
-                    ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_")
+                    ref_path = oe.sbom.doc_path_by_namespace(deploy_dir_spdx, ref.spdxDocument)
                     collect_spdx_document(ref_path)
 
             collect_spdx_document(image_spdx_path)
diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py
index 22ed5070ea0..1130fa668bd 100644
--- a/meta/lib/oe/sbom.py
+++ b/meta/lib/oe/sbom.py
@@ -38,18 +38,34 @@ def get_sdk_spdxid(sdk):
     return "SPDXRef-SDK-%s" % sdk
 
 
-def write_doc(d, spdx_doc, subdir, spdx_deploy=None, indent=None):
+def doc_path_by_namespace(spdx_deploy, doc_namespace):
+    return spdx_deploy / "by-namespace" / doc_namespace.replace("/", "_")
+
+
+def doc_path_by_hashfn(spdx_deploy, doc_name, hashfn):
+    return spdx_deploy / "by-hash" / hashfn.split()[1] / (doc_name + ".spdx.json")
+
+
+def doc_path(spdx_deploy, doc_name, arch, subdir):
+    return spdx_deploy / arch/ subdir / (doc_name + ".spdx.json")
+
+
+def write_doc(d, spdx_doc, arch, subdir, spdx_deploy=None, indent=None):
     from pathlib import Path
 
     if spdx_deploy is None:
         spdx_deploy = Path(d.getVar("SPDXDEPLOY"))
 
-    dest = spdx_deploy / subdir / (spdx_doc.name + ".spdx.json")
+    dest = doc_path(spdx_deploy, spdx_doc.name, arch, subdir)
     dest.parent.mkdir(exist_ok=True, parents=True)
     with dest.open("wb") as f:
         doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=indent)
 
-    l = spdx_deploy / "by-namespace" / spdx_doc.documentNamespace.replace("/", "_")
+    l = doc_path_by_namespace(spdx_deploy, spdx_doc.documentNamespace)
+    l.parent.mkdir(exist_ok=True, parents=True)
+    l.symlink_to(os.path.relpath(dest, l.parent))
+
+    l = doc_path_by_hashfn(spdx_deploy, spdx_doc.name, d.getVar("BB_HASHFILENAME"))
     l.parent.mkdir(exist_ok=True, parents=True)
     l.symlink_to(os.path.relpath(dest, l.parent))
 
-- 
2.39.2

[PATCH 02/11] classes/create-spdx-2.2: Respect PKG for providers

[Richard Purdie]

From: Joshua Watt <JPEWhacker@gmail.com>

If a package renames itself using PKG, the new name should be respected
as a name that the recipe provides.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/create-spdx-2.2.bbclass | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index 6ec0c1465e7..f12e155f6ba 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -688,6 +688,10 @@ def collect_package_providers(d):
             rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
             rprovides.add(pkg)
 
+            if "PKG" in pkg_data:
+                pkg = pkg_data["PKG"]
+                rprovides.add(pkg)
+
             for r in rprovides:
                 providers[r] = (pkg, dep_hashfn)
 
-- 
2.39.2

[PATCH 03/11] classes/create-spdx-2.2: Fix build time dependency calculations

[Richard Purdie]

From: Joshua Watt <JPEWhacker@gmail.com>

Build time dependencies were not being correctly calculated for SPDX
documents because while a task can `deptask` itself (as do_create_spdx
did), those dependencies do not appear in BB_TASKDEPDATA (to avoid
circular dependencies).

To fix this, an intermediate task called do_collect_sdpx_deps is created
that does the 'deptask' on do_create_spdx and records the recipe
dependencies. do_create_spdx then runs after this new task. This breaks
the circular dependency and thus all of the do_create_spdx tasks
correctly show up as dependencies of do_collect_spdx_deps.

In addition, the dependency collection logic was improved to handle the
case of transitive dependencies (that is, a dependency of a dependency)
SPDX documents missing and causing an error. These transitive
dependencies don't actually need to be included anyway since one can
follow the relationship of the direct dependency to find them. As such,
the code is reworked to find the current task in BB_TASKDEPDATA, and
then only collect the immediate dependencies of the current task.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/create-spdx-2.2.bbclass | 71 +++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 17 deletions(-)

diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index f12e155f6ba..c461c77744a 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -16,6 +16,7 @@ SPDXDEPLOY = "${SPDXDIR}/deploy"
 SPDXWORK = "${SPDXDIR}/work"
 SPDXIMAGEWORK = "${SPDXDIR}/image-work"
 SPDXSDKWORK = "${SPDXDIR}/sdk-work"
+SPDXDEPS = "${SPDXDIR}/deps.json"
 
 SPDX_TOOL_NAME ??= "oe-spdx-creator"
 SPDX_TOOL_VERSION ??= "1.0"
@@ -337,30 +338,21 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag
 
             package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc)
 
-def collect_deps(d):
-    current_task = "do_" + d.getVar("BB_CURRENTTASK")
-
-    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-    deps = sorted(set(
-        (dep[0], dep[7]) for dep in taskdepdata.values() if
-            dep[1] == current_task and dep[0] != d.getVar("PN")
-    ))
-
-    return deps
-
-collect_deps[vardepsexclude] += "BB_TASKDEPDATA"
-collect_deps[vardeps] += "DEPENDS"
-
 def collect_dep_recipes(d, doc, spdx_recipe):
+    import json
     from pathlib import Path
     import oe.sbom
     import oe.spdx
 
     deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
 
     dep_recipes = []
 
-    for dep_pn, dep_hashfn in collect_deps(d):
+    with spdx_deps_file.open("r") as f:
+        deps = json.load(f)
+
+    for dep_pn, dep_hashfn in deps:
         dep_recipe_path = oe.sbom.doc_path_by_hashfn(deploy_dir_spdx, "recipe-" + dep_pn, dep_hashfn)
 
         spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path)
@@ -462,6 +454,52 @@ def add_download_packages(d, doc, recipe):
             # but this should be sufficient for now
             doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
 
+def collect_deps(d, dep_task):
+    current_task = "do_" + d.getVar("BB_CURRENTTASK")
+    pn = d.getVar("PN")
+
+    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
+
+    for this_dep in taskdepdata.values():
+        if this_dep[0] == pn and this_dep[1] == current_task:
+            break
+    else:
+        bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata")
+
+    deps = set()
+    for dep_name in this_dep[3]:
+        dep_data = taskdepdata[dep_name]
+        if dep_data[1] == dep_task and dep_data[0] != pn:
+            deps.add((dep_data[0], dep_data[7]))
+
+    return sorted(deps)
+
+collect_deps[vardepsexclude] += "BB_TASKDEPDATA"
+collect_deps[vardeps] += "DEPENDS"
+
+python do_collect_spdx_deps() {
+    # This task calculates the build time dependencies of the recipe, and is
+    # required because while a task can deptask on itself, those dependencies
+    # do not show up in BB_TASKDEPDATA. To work around that, this task does the
+    # deptask on do_create_spdx and writes out the dependencies it finds, then
+    # do_create_spdx reads in the found dependencies when writing the actual
+    # SPDX document
+    import json
+    from pathlib import Path
+
+    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
+
+    deps = collect_deps(d, "do_create_spdx")
+
+    with spdx_deps_file.open("w") as f:
+        json.dump(deps, f)
+}
+# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
+addtask do_collect_spdx_deps after do_unpack
+do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}"
+do_collect_spdx_deps[deptask] = "do_create_spdx"
+do_collect_spdx_deps[dirs] = "${SPDXDIR}"
+
 python do_create_spdx() {
     from datetime import datetime, timezone
     import oe.sbom
@@ -647,7 +685,7 @@ python do_create_spdx() {
             oe.sbom.write_doc(d, package_doc, d.getVar("SSTATE_PKGARCH"), "packages", indent=get_json_indent(d))
 }
 # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
-addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
+addtask do_create_spdx after do_package do_packagedata do_unpack do_collect_spdx_deps before do_populate_sdk do_build do_rm_work
 
 SSTATETASKS += "do_create_spdx"
 do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
@@ -661,7 +699,6 @@ addtask do_create_spdx_setscene
 do_create_spdx[dirs] = "${SPDXWORK}"
 do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
 do_create_spdx[depends] += "${PATCHDEPENDENCY}"
-do_create_spdx[deptask] = "do_create_spdx"
 
 def collect_package_providers(d):
     from pathlib import Path
-- 
2.39.2

[PATCH 04/11] classes/create-spdx-2.2: Fix runtime dependency calculations

[Richard Purdie]

From: Joshua Watt <JPEWhacker@gmail.com>

Fixes the runtime dependency calculations to also only follow direct
dependencies of the current task instead of all transient dependencies
in a similar way as the build time dependencies.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/create-spdx-2.2.bbclass | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index c461c77744a..f9cb3add387 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -454,7 +454,7 @@ def add_download_packages(d, doc, recipe):
             # but this should be sufficient for now
             doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
 
-def collect_deps(d, dep_task):
+def collect_direct_deps(d, dep_task):
     current_task = "do_" + d.getVar("BB_CURRENTTASK")
     pn = d.getVar("PN")
 
@@ -474,8 +474,8 @@ def collect_deps(d, dep_task):
 
     return sorted(deps)
 
-collect_deps[vardepsexclude] += "BB_TASKDEPDATA"
-collect_deps[vardeps] += "DEPENDS"
+collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA"
+collect_direct_deps[vardeps] += "DEPENDS"
 
 python do_collect_spdx_deps() {
     # This task calculates the build time dependencies of the recipe, and is
@@ -489,7 +489,7 @@ python do_collect_spdx_deps() {
 
     spdx_deps_file = Path(d.getVar("SPDXDEPS"))
 
-    deps = collect_deps(d, "do_create_spdx")
+    deps = collect_direct_deps(d, "do_create_spdx")
 
     with spdx_deps_file.open("w") as f:
         json.dump(deps, f)
@@ -710,10 +710,7 @@ def collect_package_providers(d):
 
     providers = {}
 
-    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-    deps = sorted(set(
-        (dep[0], dep[7]) for dep in taskdepdata.values() if dep[0] != d.getVar("PN")
-    ))
+    deps = collect_direct_deps(d, "do_create_spdx")
     deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME")))
 
     for dep_pn, dep_hashfn in deps:
-- 
2.39.2

[PATCH 05/11] create-spdx-2-2: Fix packagedata usage to work with SDK packages

[Richard Purdie]

There are two seperate PKGDATA_DIR directories, one for the target and one for
the SDK. Rather than fail when a package can't be found, try the SDK first.

We use a datastore copy to keep the code simple, rather than havng to parameterise
all the packagedata functions.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/create-spdx-2.2.bbclass | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index f9cb3add387..b8728c5705b 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -714,11 +714,16 @@ def collect_package_providers(d):
     deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME")))
 
     for dep_pn, dep_hashfn in deps:
-        recipe_data = oe.packagedata.read_pkgdata(dep_pn, d)
+        localdata = d
+        recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
+        if not recipe_data:
+            localdata = bb.data.createCopy(d)
+            localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
+            recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
 
         for pkg in recipe_data.get("PACKAGES", "").split():
 
-            pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d)
+            pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata)
             rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
             rprovides.add(pkg)
 
-- 
2.39.2

[PATCH 06/11] create-spdx-2.2: Add missing variable exclusions

[Richard Purdie]

Some variables cause issues with taskhashes and are accounted for in the
output in other ways, or don't affect it. Exclude these to improve the
taskhashes with create-spdx and allow the sstate selftests to work better.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/create-spdx-2.2.bbclass | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index b8728c5705b..7569f8792b2 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -338,6 +338,8 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag
 
             package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc)
 
+add_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR"
+
 def collect_dep_recipes(d, doc, spdx_recipe):
     import json
     from pathlib import Path
@@ -684,6 +686,7 @@ python do_create_spdx() {
 
             oe.sbom.write_doc(d, package_doc, d.getVar("SSTATE_PKGARCH"), "packages", indent=get_json_indent(d))
 }
+do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS"
 # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
 addtask do_create_spdx after do_package do_packagedata do_unpack do_collect_spdx_deps before do_populate_sdk do_build do_rm_work
 
@@ -854,6 +857,8 @@ python do_create_runtime_spdx() {
             oe.sbom.write_doc(d, runtime_doc, d.getVar("SSTATE_PKGARCH"), "runtime", spdx_deploy, indent=get_json_indent(d))
 }
 
+do_create_runtime_spdx[vardepsexclude] += "OVERRIDES"
+
 addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work
 SSTATETASKS += "do_create_runtime_spdx"
 do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}"
@@ -920,6 +925,8 @@ def spdx_get_src(d):
     finally:
         d.setVar("WORKDIR", workdir)
 
+spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
+
 do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
 do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"
 
@@ -1125,3 +1132,5 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
             info.gname = "root"
 
             tar.addfile(info, fileobj=index_str)
+
+combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS"
-- 
2.39.2

[PATCH 07/11] layer.conf: Add missing dependency exclusion

[Richard Purdie]

Add a dependency which should have been in this list but wasn't, found
when debugging create-spdx hash issues.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/conf/layer.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
index 948ded667e8..2cc7ed84155 100644
--- a/meta/conf/layer.conf
+++ b/meta/conf/layer.conf
@@ -69,6 +69,7 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
   initramfs-module-install->grub \
   initramfs-module-install->parted \
   initramfs-module-install->util-linux \
+  initramfs-module-setup-live->udev-extraconf \
   grub-efi->grub-bootconf \
   liberation-fonts->fontconfig \
   cantarell-fonts->fontconfig \
-- 
2.39.2

[PATCH 08/11] sstatesig: Drop SPDX special casing

[Richard Purdie]

Other changes in create-spdx code mean we shouldn't need to do this now. We
need the various exclusions to allow the task hashes to behave correctly
for the SPDX tasks too.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oe/sstatesig.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index ae7ef14453f..f943df181e6 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -26,8 +26,6 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches):
         return "/allarch.bbclass" in inherits
     def isImage(mc, fn):
         return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn])
-    def isSPDXTask(task):
-        return task in ("do_create_spdx", "do_create_runtime_spdx")
 
     depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep)
     mc, _ = bb.runqueue.split_mc(fn)
@@ -38,13 +36,6 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches):
     if task == "do_rm_work":
         return False
 
-    # Keep all dependencies between SPDX tasks in the signature. SPDX documents
-    # are linked together by hashes, which means if a dependent document changes,
-    # all downstream documents must be re-written (even if they are "safe"
-    # dependencies).
-    if isSPDXTask(task) and isSPDXTask(deptaskname):
-        return True
-
     # (Almost) always include our own inter-task dependencies (unless it comes
     # from a mcdepends). The exception is the special
     # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass.
-- 
2.39.2

[PATCH 09/11] selftest/incompatible_lic: Ensure create_sdpx isn't used with the tests

[Richard Purdie]

The SPDX class doens't get on well with non-standard licenses. Disable it for
the purposes of this test to avoid errors.

Add a new helper function to the core test code to allow this to be done easily.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/selftest/case.py                   | 7 +++++++
 meta/lib/oeqa/selftest/cases/incompatible_lic.py | 2 ++
 2 files changed, 9 insertions(+)

diff --git a/meta/lib/oeqa/selftest/case.py b/meta/lib/oeqa/selftest/case.py
index dcad4f76ecd..54d90c78ac5 100644
--- a/meta/lib/oeqa/selftest/case.py
+++ b/meta/lib/oeqa/selftest/case.py
@@ -249,6 +249,13 @@ class OESelftestTestCase(OETestCase):
         self.logger.debug("Writing to: %s\n%s\n" % (self.machineinc_path, data))
         ftools.write_file(self.machineinc_path, data)
 
+    def disable_class(self, classname):
+        destfile = "%s/classes/%s.bbclass" % (self.builddir, classname)
+        os.makedirs(os.path.dirname(destfile), exist_ok=True)
+        self.track_for_cleanup(destfile)
+        self.logger.debug("Creating empty class: %s\n" % (destfile))
+        ftools.write_file(destfile, "")
+
     # check does path exist
     def assertExists(self, expr, msg=None):
         if not os.path.exists(expr):
diff --git a/meta/lib/oeqa/selftest/cases/incompatible_lic.py b/meta/lib/oeqa/selftest/cases/incompatible_lic.py
index 4edf60fc55f..1597d30ab03 100644
--- a/meta/lib/oeqa/selftest/cases/incompatible_lic.py
+++ b/meta/lib/oeqa/selftest/cases/incompatible_lic.py
@@ -113,6 +113,7 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0* LGPL-3.0*"
             raise AssertionError(result.output)
 
     def test_bash_and_license(self):
+        self.disable_class("create-spdx")
         self.write_config(self.default_config() + '\nLICENSE:append:pn-bash = " & SomeLicense"')
         error_msg = "ERROR: core-image-minimal-1.0-r0 do_rootfs: Package bash cannot be installed into the image because it has incompatible license(s): GPL-3.0-or-later"
 
@@ -121,6 +122,7 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0* LGPL-3.0*"
             raise AssertionError(result.output)
 
     def test_bash_or_license(self):
+        self.disable_class("create-spdx")
         self.write_config(self.default_config() + '\nLICENSE:append:pn-bash = " | SomeLicense"')
 
         bitbake('core-image-minimal')
-- 
2.39.2

[PATCH 10/11] oeqa/selftest/sstatetests: Add easier debug option

[Richard Purdie]

In order to debug these selftest failures you end up having to comment out
the file cleanup. Make this an option at the top of the file to make it
a bit easier, I've had to do this too many times now.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/selftest/cases/sstatetests.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/sstatetests.py b/meta/lib/oeqa/selftest/cases/sstatetests.py
index e978313c61f..febafdb2f7a 100644
--- a/meta/lib/oeqa/selftest/cases/sstatetests.py
+++ b/meta/lib/oeqa/selftest/cases/sstatetests.py
@@ -18,6 +18,9 @@ from oeqa.selftest.case import OESelftestTestCase
 import oe
 import bb.siggen
 
+# Set to True to preserve stamp files after test execution for debugging failures
+keep_temp_files = False
+
 class SStateBase(OESelftestTestCase):
 
     def setUpLocal(self):
@@ -35,6 +38,10 @@ class SStateBase(OESelftestTestCase):
         self.target_os = bb_vars['TARGET_OS']
         self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro)
 
+    def track_for_cleanup(self, path):
+        if not keep_temp_files:
+            super().track_for_cleanup(path)
+
     # Creates a special sstate configuration with the option to add sstate mirrors
     def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]):
         self.temp_sstate_location = temp_sstate_location
-- 
2.39.2

[PATCH 11/11] classes/create-spdx-2.2: Make license errors fatal

[Richard Purdie]

From: Joshua Watt <JPEWhacker@gmail.com>

Otherwise the task is still put into sstate, which means the error can
effectively be hidden by ignoring it.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
 meta/classes/create-spdx-2.2.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index 7569f8792b2..844b674f6f1 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -138,7 +138,7 @@ def convert_license_to_spdx(lic, document, d, existing={}):
                     with open(filename, errors="replace") as f:
                         extracted_info.extractedText = f.read()
                 else:
-                    bb.error("Cannot find any text for license %s" % name)
+                    bb.fatal("Cannot find any text for license %s" % name)
 
         extracted[name] = extracted_info
         document.hasExtractedLicensingInfos.append(extracted_info)
-- 
2.39.2