From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A78D8C433F5 for ; Fri, 14 Jan 2022 20:50:08 +0000 (UTC) Received: from mail-wm1-f43.google.com (mail-wm1-f43.google.com [209.85.128.43]) by mx.groups.io with SMTP id smtpd.web08.13244.1642193407950650771 for ; Fri, 14 Jan 2022 12:50:08 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20210112 header.b=NxI/1wci; spf=pass (domain: gmail.com, ip: 209.85.128.43, mailfrom: jpewhacker@gmail.com) Received: by mail-wm1-f43.google.com with SMTP id c2so2775310wml.1 for ; Fri, 14 Jan 2022 12:50:07 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=mime-version:references:in-reply-to:from:date:message-id:subject:to :cc; bh=Jq/LiDjK/PXDmMeBks4NDfWPRNY2PlJlzViWDqMOZu0=; b=NxI/1wcibBM80ugxayh7WjuCay0HDohPHT+ts7OAqK1Te0rmREfDcEBY05gGsjsJbX jfnfvo8IGS3FmHGCK+fT120HIuQLpW12shVmL86MCCNccBNDp9ewHCUcsLlLTrxpGhVD ey7Dqe3SLCxR5fTSd/QWFpEhtIsMIGHP1ks/apl5z2qZ33emBv8UNvNvXWAiRFGlcxDw l5FKPsf3CJ04sex8rf0lQH/qOVFAIyXjgllq9cowVxgZcnG4NpTis/Qn+mvZHNtE1Q0e m2lZpqZDV0CeCYkinm6rou6MU2ZwiOT8NGWl794MdBHgXHaBQbZsxBk2BWuKwlvIraFZ 2xWQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:mime-version:references:in-reply-to:from:date :message-id:subject:to:cc; bh=Jq/LiDjK/PXDmMeBks4NDfWPRNY2PlJlzViWDqMOZu0=; b=iO7aM0Fpmv3btMiggxZSP6e1llhkDMCzuSh8X2HHlkheZlFeVFStkhXAKUkKTEEipJ 0TnHIXm7vF43ZI8gbHf3iVvWhXPs4UZm9fQ0S3G0B9NR2GWt90u1ClVCrhQLQ3fVPca3 GbRf+L7hQchQf9O3+Io8OZDYeAfDe00ULAhwxbAp7ooR2Z1HCc/Job0532C9ZegsvQFB 04bXlIwIvoXUQ3kVo29bxireDfQAlQtiq8SmRZRca861brDSaA0S+rsgQVR1IVy0AXIV hrbrOBEDPbRrTxZP06aoXlGWW6GkFe32o1iNMZ9PVMmb6poBB1PVHzkHRCgjzJGPS+05 864w== X-Gm-Message-State: AOAM532ict1qx5IODuhjQtZ4HWISFXjDdaEdVZCBAOixaguwf5mXB0kp VVbosvFQyd2Ww0HCMBbvoo1hE1bJ9ftcUrxRR+M= X-Google-Smtp-Source: ABdhPJwNdAfEzOnB25S6zJWbsOZ+H9PXCjxuhTZsoJ3B99F4h+E86GsFIi9owHzfxrg3De4Vq9c8acssqEG6/i2eh24= X-Received: by 2002:adf:d1eb:: with SMTP id g11mr8227554wrd.94.1642193406007; Fri, 14 Jan 2022 12:50:06 -0800 (PST) MIME-Version: 1.0 References: <20220112194012.873-1-abeltran@linux.microsoft.com> In-Reply-To: <20220112194012.873-1-abeltran@linux.microsoft.com> From: Joshua Watt Date: Fri, 14 Jan 2022 14:49:54 -0600 Message-ID: Subject: Re: [OE-core] [PATCH] create-spdx: add support for SDKs To: Andres Beltran Cc: openembedded-core@lists.openembedded.org Content-Type: text/plain; charset="UTF-8" List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 14 Jan 2022 20:50:08 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/160592 On Wed, Jan 12, 2022 at 1:40 PM Andres Beltran wrote: > > Currently, SPDX SBOMs are only created for images. Add support for > SDKs. Fix json indent when outputting SBOMs for better readability. Most of us just pipe the output through `jq` to view it (colors are really helpful). These files are already quite large, do we really need to add more padding? > > Signed-off-by: Andres Beltran > --- > meta/classes/create-spdx.bbclass | 95 +++++++++++++++++++++----------- > meta/lib/oe/sbom.py | 6 +- > 2 files changed, 68 insertions(+), 33 deletions(-) > > diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass > index e44a204a8fc..d0f987315ee 100644 > --- a/meta/classes/create-spdx.bbclass > +++ b/meta/classes/create-spdx.bbclass > @@ -556,7 +556,7 @@ python do_create_spdx() { > oe.sbom.write_doc(d, package_doc, "packages") > } > # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source > -addtask do_create_spdx after do_package do_packagedata do_unpack before do_build do_rm_work > +addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work > > SSTATETASKS += "do_create_spdx" > do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" > @@ -788,28 +788,77 @@ def spdx_get_src(d): > do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" > > ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " > + > +do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" > +POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " sdk_host_combine_spdx; " > +POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk = " sdk_target_combine_spdx; " > + > python image_combine_spdx() { > + import os > + import oe.sbom > + from pathlib import Path > + from oe.rootfs import image_list_installed_packages > + > + image_name = d.getVar("IMAGE_NAME") > + image_link_name = d.getVar("IMAGE_LINK_NAME") > + imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) > + img_spdxid = oe.sbom.get_image_spdxid(image_name) > + packages = image_list_installed_packages(d) > + > + combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages) > + > + if image_link_name: > + image_spdx_path = imgdeploydir / (image_name + ".spdx.json") > + image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json") > + image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent)) > + > + def make_image_link(target_path, suffix): > + if image_link_name: > + link = imgdeploydir / (image_link_name + suffix) > + link.symlink_to(os.path.relpath(target_path, link.parent)) > + > + spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") > + make_image_link(spdx_tar_path, ".spdx.tar.zst") > + spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json") > + make_image_link(spdx_index_path, ".spdx.index.json") > +} > + > +python sdk_host_combine_spdx() { > + sdk_combine_spdx(d, "host") > +} > + > +python sdk_target_combine_spdx() { > + sdk_combine_spdx(d, "target") > +} > + > +def sdk_combine_spdx(d, sdk_type): > + import oe.sbom > + from pathlib import Path > + from oe.sdk import sdk_list_installed_packages > + > + sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type > + sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) > + sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) > + sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") > + combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages) > + > +def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages): > import os > import oe.spdx > import oe.sbom > import io > import json > - from oe.rootfs import image_list_installed_packages > from datetime import timezone, datetime > from pathlib import Path > import tarfile > import bb.compress.zstd > > creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") > - image_name = d.getVar("IMAGE_NAME") > - image_link_name = d.getVar("IMAGE_LINK_NAME") > - > deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) > - imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) > source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") > > doc = oe.spdx.SPDXDocument() > - doc.name = image_name > + doc.name = rootfs_name > doc.documentNamespace = get_doc_namespace(d, doc) > doc.creationInfo.created = creation_time > doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." > @@ -821,14 +870,12 @@ python image_combine_spdx() { > image = oe.spdx.SPDXPackage() > image.name = d.getVar("PN") > image.versionInfo = d.getVar("PV") > - image.SPDXID = oe.sbom.get_image_spdxid(image_name) > + image.SPDXID = rootfs_spdxid > > doc.packages.append(image) > > spdx_package = oe.spdx.SPDXPackage() > > - packages = image_list_installed_packages(d) > - > for name in sorted(packages.keys()): > pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") > pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) > @@ -856,7 +903,6 @@ python image_combine_spdx() { > runtime_ref.checksum.algorithm = "SHA1" > runtime_ref.checksum.checksumValue = runtime_doc_sha1 > > - # "OTHER" isn't ideal here, but I can't find a relationship that makes sense > doc.externalDocumentRefs.append(runtime_ref) > doc.add_relationship( > image, > @@ -865,14 +911,10 @@ python image_combine_spdx() { > comment="Runtime dependencies for %s" % name > ) > > - image_spdx_path = imgdeploydir / (image_name + ".spdx.json") > + image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json") > > with image_spdx_path.open("wb") as f: > - doc.to_json(f, sort_keys=True) > - > - if image_link_name: > - image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json") > - image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent)) > + doc.to_json(f, sort_keys=True, indent=4) > > num_threads = int(d.getVar("BB_NUMBER_THREADS")) > > @@ -880,7 +922,7 @@ python image_combine_spdx() { > > index = {"documents": []} > > - spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") > + spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") > with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: > with tarfile.open(fileobj=f, mode="w|") as tar: > def collect_spdx_document(path): > @@ -930,7 +972,7 @@ python image_combine_spdx() { > > index["documents"].sort(key=lambda x: x["filename"]) > > - index_str = io.BytesIO(json.dumps(index, sort_keys=True).encode("utf-8")) > + index_str = io.BytesIO(json.dumps(index, sort_keys=True, indent=4).encode("utf-8")) > > info = tarfile.TarInfo() > info.name = "index.json" > @@ -942,17 +984,6 @@ python image_combine_spdx() { > > tar.addfile(info, fileobj=index_str) > > - def make_image_link(target_path, suffix): > - if image_link_name: > - link = imgdeploydir / (image_link_name + suffix) > - link.symlink_to(os.path.relpath(target_path, link.parent)) > - > - make_image_link(spdx_tar_path, ".spdx.tar.zst") > - > - spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json") > + spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json") > with spdx_index_path.open("w") as f: > - json.dump(index, f, sort_keys=True) > - > - make_image_link(spdx_index_path, ".spdx.index.json") > -} > - > + json.dump(index, f, sort_keys=True, indent=4) > diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py > index 848812c0b7d..a975a3c9fc0 100644 > --- a/meta/lib/oe/sbom.py > +++ b/meta/lib/oe/sbom.py > @@ -28,6 +28,10 @@ def get_image_spdxid(img): > return "SPDXRef-Image-%s" % img > > > +def get_sdk_spdxid(sdk): > + return "SPDXRef-SDK-%s" % sdk > + > + > def write_doc(d, spdx_doc, subdir, spdx_deploy=None): > from pathlib import Path > > @@ -37,7 +41,7 @@ def write_doc(d, spdx_doc, subdir, spdx_deploy=None): > dest = spdx_deploy / subdir / (spdx_doc.name + ".spdx.json") > dest.parent.mkdir(exist_ok=True, parents=True) > with dest.open("wb") as f: > - doc_sha1 = spdx_doc.to_json(f, sort_keys=True) > + doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=4) > > l = spdx_deploy / "by-namespace" / spdx_doc.documentNamespace.replace("/", "_") > l.parent.mkdir(exist_ok=True, parents=True) > -- > 2.17.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#160500): https://lists.openembedded.org/g/openembedded-core/message/160500 > Mute This Topic: https://lists.openembedded.org/mt/88381128/3616693 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [JPEWhacker@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >