[PATCH] create-spdx: add support for SDKs

[PATCH] create-spdx: add support for SDKs

[Andres Beltran]

Currently, SPDX SBOMs are only created for images. Add support for
SDKs. Fix json indent when outputting SBOMs for better readability.

Signed-off-by: Andres Beltran <abeltran@linux.microsoft.com>
---
 meta/classes/create-spdx.bbclass | 95 +++++++++++++++++++++-----------
 meta/lib/oe/sbom.py              |  6 +-
 2 files changed, 68 insertions(+), 33 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index e44a204a8fc..d0f987315ee 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -556,7 +556,7 @@ python do_create_spdx() {
             oe.sbom.write_doc(d, package_doc, "packages")
 }
 # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
-addtask do_create_spdx after do_package do_packagedata do_unpack before do_build do_rm_work
+addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
 
 SSTATETASKS += "do_create_spdx"
 do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
@@ -788,28 +788,77 @@ def spdx_get_src(d):
 do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
 
 ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
+
+do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
+POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " sdk_host_combine_spdx; "
+POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk = " sdk_target_combine_spdx; "
+
 python image_combine_spdx() {
+    import os
+    import oe.sbom
+    from pathlib import Path
+    from oe.rootfs import image_list_installed_packages
+
+    image_name = d.getVar("IMAGE_NAME")
+    image_link_name = d.getVar("IMAGE_LINK_NAME")
+    imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
+    img_spdxid = oe.sbom.get_image_spdxid(image_name)
+    packages = image_list_installed_packages(d)
+
+    combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
+
+    if image_link_name:
+        image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
+        image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
+        image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent))
+
+    def make_image_link(target_path, suffix):
+        if image_link_name:
+            link = imgdeploydir / (image_link_name + suffix)
+            link.symlink_to(os.path.relpath(target_path, link.parent))
+
+    spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
+    make_image_link(spdx_tar_path, ".spdx.tar.zst")
+    spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
+    make_image_link(spdx_index_path, ".spdx.index.json")
+}
+
+python sdk_host_combine_spdx() {
+    sdk_combine_spdx(d, "host")
+}
+
+python sdk_target_combine_spdx() {
+    sdk_combine_spdx(d, "target")
+}
+
+def sdk_combine_spdx(d, sdk_type):
+    import oe.sbom
+    from pathlib import Path
+    from oe.sdk import sdk_list_installed_packages
+
+    sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
+    sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
+    sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
+    sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
+    combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
+
+def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
     import os
     import oe.spdx
     import oe.sbom
     import io
     import json
-    from oe.rootfs import image_list_installed_packages
     from datetime import timezone, datetime
     from pathlib import Path
     import tarfile
     import bb.compress.zstd
 
     creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    image_name = d.getVar("IMAGE_NAME")
-    image_link_name = d.getVar("IMAGE_LINK_NAME")
-
     deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
-    imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
     source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
 
     doc = oe.spdx.SPDXDocument()
-    doc.name = image_name
+    doc.name = rootfs_name
     doc.documentNamespace = get_doc_namespace(d, doc)
     doc.creationInfo.created = creation_time
     doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
@@ -821,14 +870,12 @@ python image_combine_spdx() {
     image = oe.spdx.SPDXPackage()
     image.name = d.getVar("PN")
     image.versionInfo = d.getVar("PV")
-    image.SPDXID = oe.sbom.get_image_spdxid(image_name)
+    image.SPDXID = rootfs_spdxid
 
     doc.packages.append(image)
 
     spdx_package = oe.spdx.SPDXPackage()
 
-    packages = image_list_installed_packages(d)
-
     for name in sorted(packages.keys()):
         pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json")
         pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
@@ -856,7 +903,6 @@ python image_combine_spdx() {
         runtime_ref.checksum.algorithm = "SHA1"
         runtime_ref.checksum.checksumValue = runtime_doc_sha1
 
-        # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
         doc.externalDocumentRefs.append(runtime_ref)
         doc.add_relationship(
             image,
@@ -865,14 +911,10 @@ python image_combine_spdx() {
             comment="Runtime dependencies for %s" % name
         )
 
-    image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
+    image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json")
 
     with image_spdx_path.open("wb") as f:
-        doc.to_json(f, sort_keys=True)
-
-    if image_link_name:
-        image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
-        image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent))
+        doc.to_json(f, sort_keys=True, indent=4)
 
     num_threads = int(d.getVar("BB_NUMBER_THREADS"))
 
@@ -880,7 +922,7 @@ python image_combine_spdx() {
 
     index = {"documents": []}
 
-    spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
+    spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst")
     with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f:
         with tarfile.open(fileobj=f, mode="w|") as tar:
             def collect_spdx_document(path):
@@ -930,7 +972,7 @@ python image_combine_spdx() {
 
             index["documents"].sort(key=lambda x: x["filename"])
 
-            index_str = io.BytesIO(json.dumps(index, sort_keys=True).encode("utf-8"))
+            index_str = io.BytesIO(json.dumps(index, sort_keys=True, indent=4).encode("utf-8"))
 
             info = tarfile.TarInfo()
             info.name = "index.json"
@@ -942,17 +984,6 @@ python image_combine_spdx() {
 
             tar.addfile(info, fileobj=index_str)
 
-    def make_image_link(target_path, suffix):
-        if image_link_name:
-            link = imgdeploydir / (image_link_name + suffix)
-            link.symlink_to(os.path.relpath(target_path, link.parent))
-
-    make_image_link(spdx_tar_path, ".spdx.tar.zst")
-
-    spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
+    spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json")
     with spdx_index_path.open("w") as f:
-        json.dump(index, f, sort_keys=True)
-
-    make_image_link(spdx_index_path, ".spdx.index.json")
-}
-
+        json.dump(index, f, sort_keys=True, indent=4)
diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py
index 848812c0b7d..a975a3c9fc0 100644
--- a/meta/lib/oe/sbom.py
+++ b/meta/lib/oe/sbom.py
@@ -28,6 +28,10 @@ def get_image_spdxid(img):
     return "SPDXRef-Image-%s" % img
 
 
+def get_sdk_spdxid(sdk):
+    return "SPDXRef-SDK-%s" % sdk
+
+
 def write_doc(d, spdx_doc, subdir, spdx_deploy=None):
     from pathlib import Path
 
@@ -37,7 +41,7 @@ def write_doc(d, spdx_doc, subdir, spdx_deploy=None):
     dest = spdx_deploy / subdir / (spdx_doc.name + ".spdx.json")
     dest.parent.mkdir(exist_ok=True, parents=True)
     with dest.open("wb") as f:
-        doc_sha1 = spdx_doc.to_json(f, sort_keys=True)
+        doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=4)
 
     l = spdx_deploy / "by-namespace" / spdx_doc.documentNamespace.replace("/", "_")
     l.parent.mkdir(exist_ok=True, parents=True)
-- 
2.17.1

Re: [OE-core] [PATCH] create-spdx: add support for SDKs

[Saul Wold]

Overall I think this is going in the right direction, I need to review 
it a little deeper and check the actual output.

I am not sure that you tested this against master as you use the old _ 
override syntax vs using a :.

See note below.

Sau!


On 1/12/22 11:40, Andres Beltran wrote:
> Signed-off-by: Andres Beltran <abeltran@linux.microsoft.com>
> ---
>   meta/classes/create-spdx.bbclass | 95 +++++++++++++++++++++-----------
>   meta/lib/oe/sbom.py              |  6 +-
>   2 files changed, 68 insertions(+), 33 deletions(-)
> 
> diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
> index e44a204a8fc..d0f987315ee 100644
> --- a/meta/classes/create-spdx.bbclass
> +++ b/meta/classes/create-spdx.bbclass
> @@ -556,7 +556,7 @@ python do_create_spdx() {
>               oe.sbom.write_doc(d, package_doc, "packages")
>   }
>   # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
> -addtask do_create_spdx after do_package do_packagedata do_unpack before do_build do_rm_work
> +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
>   
>   SSTATETASKS += "do_create_spdx"
>   do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
> @@ -788,28 +788,77 @@ def spdx_get_src(d):
>   do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
>   
>   ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
> +
> +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
> +POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " sdk_host_combine_spdx; "
> +POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk = " sdk_target_combine_spdx; > +
You using the older _append syntax vs newer :append syntax in master


>   python image_combine_spdx() {
> +    import os
> +    import oe.sbom
> +    from pathlib import Path
> +    from oe.rootfs import image_list_installed_packages
> +
> +    image_name = d.getVar("IMAGE_NAME")
> +    image_link_name = d.getVar("IMAGE_LINK_NAME")
> +    imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
> +    img_spdxid = oe.sbom.get_image_spdxid(image_name)
> +    packages = image_list_installed_packages(d)
> +
> +    combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
> +
> +    if image_link_name:
> +        image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
> +        image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
> +        image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent))
> +
> +    def make_image_link(target_path, suffix):
> +        if image_link_name:
> +            link = imgdeploydir / (image_link_name + suffix)
> +            link.symlink_to(os.path.relpath(target_path, link.parent))
> +
> +    spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
> +    make_image_link(spdx_tar_path, ".spdx.tar.zst")
> +    spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
> +    make_image_link(spdx_index_path, ".spdx.index.json")
> +}
> +
> +python sdk_host_combine_spdx() {
> +    sdk_combine_spdx(d, "host")
> +}
> +
> +python sdk_target_combine_spdx() {
> +    sdk_combine_spdx(d, "target")
> +}
> +
> +def sdk_combine_spdx(d, sdk_type):
> +    import oe.sbom
> +    from pathlib import Path
> +    from oe.sdk import sdk_list_installed_packages
> +
> +    sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
> +    sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
> +    sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
> +    sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
> +    combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
> +
> +def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
>       import os
>       import oe.spdx
>       import oe.sbom
>       import io
>       import json
> -    from oe.rootfs import image_list_installed_packages
>       from datetime import timezone, datetime
>       from pathlib import Path
>       import tarfile
>       import bb.compress.zstd
>   
>       creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
> -    image_name = d.getVar("IMAGE_NAME")
> -    image_link_name = d.getVar("IMAGE_LINK_NAME")
> -
>       deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
> -    imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
>       source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
>   
>       doc = oe.spdx.SPDXDocument()
> -    doc.name = image_name
> +    doc.name = rootfs_name
>       doc.documentNamespace = get_doc_namespace(d, doc)
>       doc.creationInfo.created = creation_time
>       doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
> @@ -821,14 +870,12 @@ python image_combine_spdx() {
>       image = oe.spdx.SPDXPackage()
>       image.name = d.getVar("PN")
>       image.versionInfo = d.getVar("PV")
> -    image.SPDXID = oe.sbom.get_image_spdxid(image_name)
> +    image.SPDXID = rootfs_spdxid
>   
>       doc.packages.append(image)
>   
>       spdx_package = oe.spdx.SPDXPackage()
>   
> -    packages = image_list_installed_packages(d)
> -
>       for name in sorted(packages.keys()):
>           pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json")
>           pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
> @@ -856,7 +903,6 @@ python image_combine_spdx() {
>           runtime_ref.checksum.algorithm = "SHA1"
>           runtime_ref.checksum.checksumValue = runtime_doc_sha1
>   
> -        # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
>           doc.externalDocumentRefs.append(runtime_ref)
>           doc.add_relationship(
>               image,
> @@ -865,14 +911,10 @@ python image_combine_spdx() {
>               comment="Runtime dependencies for %s" % name
>           )
>   
> -    image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
> +    image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json")
>   
>       with image_spdx_path.open("wb") as f:
> -        doc.to_json(f, sort_keys=True)
> -
> -    if image_link_name:
> -        image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
> -        image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent))
> +        doc.to_json(f, sort_keys=True, indent=4)
>   
>       num_threads = int(d.getVar("BB_NUMBER_THREADS"))
>   
> @@ -880,7 +922,7 @@ python image_combine_spdx() {
>   
>       index = {"documents": []}
>   
> -    spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
> +    spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst")
>       with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f:
>           with tarfile.open(fileobj=f, mode="w|") as tar:
>               def collect_spdx_document(path):
> @@ -930,7 +972,7 @@ python image_combine_spdx() {
>   
>               index["documents"].sort(key=lambda x: x["filename"])
>   
> -            index_str = io.BytesIO(json.dumps(index, sort_keys=True).encode("utf-8"))
> +            index_str = io.BytesIO(json.dumps(index, sort_keys=True, indent=4).encode("utf-8"))
>   
>               info = tarfile.TarInfo()
>               info.name = "index.json"
> @@ -942,17 +984,6 @@ python image_combine_spdx() {
>   
>               tar.addfile(info, fileobj=index_str)
>   
> -    def make_image_link(target_path, suffix):
> -        if image_link_name:
> -            link = imgdeploydir / (image_link_name + suffix)
> -            link.symlink_to(os.path.relpath(target_path, link.parent))
> -
> -    make_image_link(spdx_tar_path, ".spdx.tar.zst")
> -
> -    spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
> +    spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json")
>       with spdx_index_path.open("w") as f:
> -        json.dump(index, f, sort_keys=True)
> -
> -    make_image_link(spdx_index_path, ".spdx.index.json")
> -}
> -
> +        json.dump(index, f, sort_keys=True, indent=4)
> diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py
> index 848812c0b7d..a975a3c9fc0 100644
> --- a/meta/lib/oe/sbom.py
> +++ b/meta/lib/oe/sbom.py
> @@ -28,6 +28,10 @@ def get_image_spdxid(img):
>       return "SPDXRef-Image-%s" % img
>   
>   
> +def get_sdk_spdxid(sdk):
> +    return "SPDXRef-SDK-%s" % sdk
> +
> +
>   def write_doc(d, spdx_doc, subdir, spdx_deploy=None):
>       from pathlib import Path
>   
> @@ -37,7 +41,7 @@ def write_doc(d, spdx_doc, subdir, spdx_deploy=None):
>       dest = spdx_deploy / subdir / (spdx_doc.name + ".spdx.json")
>       dest.parent.mkdir(exist_ok=True, parents=True)
>       with dest.open("wb") as f:
> -        doc_sha1 = spdx_doc.to_json(f, sort_keys=True)
> +        doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=4)
>   
>       l = spdx_deploy / "by-namespace" / spdx_doc.documentNamespace.replace("/", "_")
>       l.parent.mkdir(exist_ok=True, parents=True)
> 
> 
> 
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#160500): https://lists.openembedded.org/g/openembedded-core/message/160500
> Mute This Topic: https://lists.openembedded.org/mt/88381128/4950653
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [Saul.Wold@windriver.com]
> -=-=-=-=-=-=-=-=-=-=-=-
> 

-- 
Sau!

Re: [OE-core] [PATCH] create-spdx: add support for SDKs

[Andres Beltran]

On 1/14/2022 2:16 PM, Saul Wold wrote:
> Overall I think this is going in the right direction, I need to review 
> it a little deeper and check the actual output.
>
> I am not sure that you tested this against master as you use the old _ 
> override syntax vs using a :.
>
> See note below.
>
> Sau!
>
Thanks Saul! Yes, I was certainly using an older version. I will
correct the override syntax. Let me know if I should make any other
changes once you review it.

Re: [OE-core] [PATCH] create-spdx: add support for SDKs

[Joshua Watt]

On Wed, Jan 12, 2022 at 1:40 PM Andres Beltran
<abeltran@linux.microsoft.com> wrote:
>
> Currently, SPDX SBOMs are only created for images. Add support for
> SDKs. Fix json indent when outputting SBOMs for better readability.

Most of us just pipe the output through `jq` to view it (colors are
really helpful). These files are already quite large, do we really
need to add more padding?

>
> Signed-off-by: Andres Beltran <abeltran@linux.microsoft.com>
> ---
>  meta/classes/create-spdx.bbclass | 95 +++++++++++++++++++++-----------
>  meta/lib/oe/sbom.py              |  6 +-
>  2 files changed, 68 insertions(+), 33 deletions(-)
>
> diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
> index e44a204a8fc..d0f987315ee 100644
> --- a/meta/classes/create-spdx.bbclass
> +++ b/meta/classes/create-spdx.bbclass
> @@ -556,7 +556,7 @@ python do_create_spdx() {
>              oe.sbom.write_doc(d, package_doc, "packages")
>  }
>  # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
> -addtask do_create_spdx after do_package do_packagedata do_unpack before do_build do_rm_work
> +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
>
>  SSTATETASKS += "do_create_spdx"
>  do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
> @@ -788,28 +788,77 @@ def spdx_get_src(d):
>  do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
>
>  ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
> +
> +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
> +POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " sdk_host_combine_spdx; "
> +POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk = " sdk_target_combine_spdx; "
> +
>  python image_combine_spdx() {
> +    import os
> +    import oe.sbom
> +    from pathlib import Path
> +    from oe.rootfs import image_list_installed_packages
> +
> +    image_name = d.getVar("IMAGE_NAME")
> +    image_link_name = d.getVar("IMAGE_LINK_NAME")
> +    imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
> +    img_spdxid = oe.sbom.get_image_spdxid(image_name)
> +    packages = image_list_installed_packages(d)
> +
> +    combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
> +
> +    if image_link_name:
> +        image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
> +        image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
> +        image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent))
> +
> +    def make_image_link(target_path, suffix):
> +        if image_link_name:
> +            link = imgdeploydir / (image_link_name + suffix)
> +            link.symlink_to(os.path.relpath(target_path, link.parent))
> +
> +    spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
> +    make_image_link(spdx_tar_path, ".spdx.tar.zst")
> +    spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
> +    make_image_link(spdx_index_path, ".spdx.index.json")
> +}
> +
> +python sdk_host_combine_spdx() {
> +    sdk_combine_spdx(d, "host")
> +}
> +
> +python sdk_target_combine_spdx() {
> +    sdk_combine_spdx(d, "target")
> +}
> +
> +def sdk_combine_spdx(d, sdk_type):
> +    import oe.sbom
> +    from pathlib import Path
> +    from oe.sdk import sdk_list_installed_packages
> +
> +    sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
> +    sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
> +    sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
> +    sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
> +    combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
> +
> +def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
>      import os
>      import oe.spdx
>      import oe.sbom
>      import io
>      import json
> -    from oe.rootfs import image_list_installed_packages
>      from datetime import timezone, datetime
>      from pathlib import Path
>      import tarfile
>      import bb.compress.zstd
>
>      creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
> -    image_name = d.getVar("IMAGE_NAME")
> -    image_link_name = d.getVar("IMAGE_LINK_NAME")
> -
>      deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
> -    imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
>      source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
>
>      doc = oe.spdx.SPDXDocument()
> -    doc.name = image_name
> +    doc.name = rootfs_name
>      doc.documentNamespace = get_doc_namespace(d, doc)
>      doc.creationInfo.created = creation_time
>      doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
> @@ -821,14 +870,12 @@ python image_combine_spdx() {
>      image = oe.spdx.SPDXPackage()
>      image.name = d.getVar("PN")
>      image.versionInfo = d.getVar("PV")
> -    image.SPDXID = oe.sbom.get_image_spdxid(image_name)
> +    image.SPDXID = rootfs_spdxid
>
>      doc.packages.append(image)
>
>      spdx_package = oe.spdx.SPDXPackage()
>
> -    packages = image_list_installed_packages(d)
> -
>      for name in sorted(packages.keys()):
>          pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json")
>          pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
> @@ -856,7 +903,6 @@ python image_combine_spdx() {
>          runtime_ref.checksum.algorithm = "SHA1"
>          runtime_ref.checksum.checksumValue = runtime_doc_sha1
>
> -        # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
>          doc.externalDocumentRefs.append(runtime_ref)
>          doc.add_relationship(
>              image,
> @@ -865,14 +911,10 @@ python image_combine_spdx() {
>              comment="Runtime dependencies for %s" % name
>          )
>
> -    image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
> +    image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json")
>
>      with image_spdx_path.open("wb") as f:
> -        doc.to_json(f, sort_keys=True)
> -
> -    if image_link_name:
> -        image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
> -        image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent))
> +        doc.to_json(f, sort_keys=True, indent=4)
>
>      num_threads = int(d.getVar("BB_NUMBER_THREADS"))
>
> @@ -880,7 +922,7 @@ python image_combine_spdx() {
>
>      index = {"documents": []}
>
> -    spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
> +    spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst")
>      with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f:
>          with tarfile.open(fileobj=f, mode="w|") as tar:
>              def collect_spdx_document(path):
> @@ -930,7 +972,7 @@ python image_combine_spdx() {
>
>              index["documents"].sort(key=lambda x: x["filename"])
>
> -            index_str = io.BytesIO(json.dumps(index, sort_keys=True).encode("utf-8"))
> +            index_str = io.BytesIO(json.dumps(index, sort_keys=True, indent=4).encode("utf-8"))
>
>              info = tarfile.TarInfo()
>              info.name = "index.json"
> @@ -942,17 +984,6 @@ python image_combine_spdx() {
>
>              tar.addfile(info, fileobj=index_str)
>
> -    def make_image_link(target_path, suffix):
> -        if image_link_name:
> -            link = imgdeploydir / (image_link_name + suffix)
> -            link.symlink_to(os.path.relpath(target_path, link.parent))
> -
> -    make_image_link(spdx_tar_path, ".spdx.tar.zst")
> -
> -    spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
> +    spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json")
>      with spdx_index_path.open("w") as f:
> -        json.dump(index, f, sort_keys=True)
> -
> -    make_image_link(spdx_index_path, ".spdx.index.json")
> -}
> -
> +        json.dump(index, f, sort_keys=True, indent=4)
> diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py
> index 848812c0b7d..a975a3c9fc0 100644
> --- a/meta/lib/oe/sbom.py
> +++ b/meta/lib/oe/sbom.py
> @@ -28,6 +28,10 @@ def get_image_spdxid(img):
>      return "SPDXRef-Image-%s" % img
>
>
> +def get_sdk_spdxid(sdk):
> +    return "SPDXRef-SDK-%s" % sdk
> +
> +
>  def write_doc(d, spdx_doc, subdir, spdx_deploy=None):
>      from pathlib import Path
>
> @@ -37,7 +41,7 @@ def write_doc(d, spdx_doc, subdir, spdx_deploy=None):
>      dest = spdx_deploy / subdir / (spdx_doc.name + ".spdx.json")
>      dest.parent.mkdir(exist_ok=True, parents=True)
>      with dest.open("wb") as f:
> -        doc_sha1 = spdx_doc.to_json(f, sort_keys=True)
> +        doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=4)
>
>      l = spdx_deploy / "by-namespace" / spdx_doc.documentNamespace.replace("/", "_")
>      l.parent.mkdir(exist_ok=True, parents=True)
> --
> 2.17.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#160500): https://lists.openembedded.org/g/openembedded-core/message/160500
> Mute This Topic: https://lists.openembedded.org/mt/88381128/3616693
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [JPEWhacker@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

Re: [OE-core] [PATCH] create-spdx: add support for SDKs

[Andres Beltran]

On 1/14/2022 3:49 PM, Joshua Watt wrote:
>
> Most of us just pipe the output through `jq` to view it (colors are
> really helpful). These files are already quite large, do we really
> need to add more padding?

Ok thanks Joshua. I can certainly remove that change. I saw that the
json files didn't have any indentation out of a build, so added
some padding to make them more readable. But yes, we can certainly use
jq as well.