[PATCH v2 1/4] create-spdx: transform license list into a dict for faster lookups

[PATCH v2 1/4] create-spdx: transform license list into a dict for faster lookups

[Ross Burton]

spdx-licenses.json contains an array of licenses objects. As we'll be
searching it often, convert that to a dictionary when we parse it.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/create-spdx.bbclass | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index a590ab596ac..73ccb3c990f 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -44,7 +44,10 @@ python() {
         return
 
     with open(d.getVar("SPDX_LICENSES"), "r") as f:
-        d.setVar("SPDX_LICENSE_DATA", json.load(f))
+        data = json.load(f)
+        # Transform the license array to a dictionary
+        data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
+        d.setVar("SPDX_LICENSE_DATA", data)
 }
 
 def convert_license_to_spdx(lic, document, d):
@@ -55,9 +58,8 @@ def convert_license_to_spdx(lic, document, d):
     def add_extracted_license(ident, name, text):
         nonlocal document
 
-        for lic_data in license_data["licenses"]:
-            if lic_data["licenseId"] == ident:
-                return False
+        if ident in license_data["licenses"]:
+            return False
 
         spdx_lic = oe.spdx.SPDXExtractedLicensingInfo()
         spdx_lic.name = name
@@ -79,9 +81,8 @@ def convert_license_to_spdx(lic, document, d):
             return "OR"
 
         spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l
-        for lic_data in license_data["licenses"]:
-            if lic_data["licenseId"] == spdx_license:
-                return spdx_license
+        if spdx_license in license_data["licenses"]:
+            return spdx_license
 
         spdx_license = "LicenseRef-" + l
 
-- 
2.25.1

[PATCH v2 2/4] create-spdx: remove redundant test

[Ross Burton]

add_extracted_document() is only called if the license isn't known to
SPDX, so there's no need to check again.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/create-spdx.bbclass | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 73ccb3c990f..529dee22918 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -58,9 +58,6 @@ def convert_license_to_spdx(lic, document, d):
     def add_extracted_license(ident, name, text):
         nonlocal document
 
-        if ident in license_data["licenses"]:
-            return False
-
         spdx_lic = oe.spdx.SPDXExtractedLicensingInfo()
         spdx_lic.name = name
         spdx_lic.licenseId = ident
-- 
2.25.1

[PATCH v2 3/4] create-spdx: embed unknown license texts

[Ross Burton]

For licenses which are not known to SPDX, find and embed the actual
license text in an ExtractedLicesingInfo block.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/create-spdx.bbclass | 51 +++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 529dee22918..cbb9239991c 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -51,21 +51,49 @@ python() {
 }
 
 def convert_license_to_spdx(lic, document, d):
+    from pathlib import Path
     import oe.spdx
 
+    available_licenses = d.getVar("AVAILABLE_LICENSES").split()
     license_data = d.getVar("SPDX_LICENSE_DATA")
+    extracted = {}
 
-    def add_extracted_license(ident, name, text):
+    def add_extracted_license(ident, name):
         nonlocal document
 
-        spdx_lic = oe.spdx.SPDXExtractedLicensingInfo()
-        spdx_lic.name = name
-        spdx_lic.licenseId = ident
-        spdx_lic.extractedText = text
-
-        document.hasExtractedLicensingInfos.append(spdx_lic)
+        if name in extracted:
+            return
+
+        extracted_info = oe.spdx.SPDXExtractedLicensingInfo()
+        extracted_info.name = name
+        extracted_info.licenseId = ident
+
+        if name == "PD":
+            # Special-case this.
+            extracted_info.extractedText = "Software released to the public domain"
+        elif name in available_licenses:
+            # This license can be found in COMMON_LICENSE_DIR or LICENSE_PATH
+            for directory in [d.getVar('COMMON_LICENSE_DIR')] + d.getVar('LICENSE_PATH').split():
+                try:
+                    with (Path(directory) / name).open(errors="replace") as f:
+                        extracted_info.extractedText = f.read()
+                        break
+                except Exception as e:
+                    # Error out, as the license was in available_licenses so
+                    # should be on disk somewhere.
+                    bb.error(f"Cannot find text for license {name}: {e}")
+        else:
+            # If it's not SPDX, or PD, or in available licenses, then NO_GENERIC_LICENSE must be set
+            filename = d.getVarFlag('NO_GENERIC_LICENSE', name)
+            if filename:
+                filename = d.expand("${S}/" + filename)
+                with open(filename, errors="replace") as f:
+                    extracted_info.extractedText = f.read()
+            else:
+                bb.error(f"Cannot find any text for license {name}")
 
-        return True
+        extracted[name] = extracted_info
+        document.hasExtractedLicensingInfos.append(extracted_info)
 
     def convert(l):
         if l == "(" or l == ")":
@@ -82,12 +110,7 @@ def convert_license_to_spdx(lic, document, d):
             return spdx_license
 
         spdx_license = "LicenseRef-" + l
-
-        if l == "PD":
-            add_extracted_license(spdx_license, l, "Software released to the public domain")
-        elif add_extracted_license(spdx_license, l, "This software is licensed under the %s license" % l):
-            pass
-            #bb.warn("No SPDX License found for %s. Creating a place holder" % l)
+        add_extracted_license(spdx_license, l)
 
         return spdx_license
 
-- 
2.25.1

[PATCH v2 4/4] create-spex: don't duplicate license texts in each package

[Ross Burton]

Instead of putting the full license text for non-SPDX licenses into the
recipe and every package, use links to the recipe from the packages if
possible.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/create-spdx.bbclass | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index cbb9239991c..1e0b3605587 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -50,7 +50,7 @@ python() {
         d.setVar("SPDX_LICENSE_DATA", data)
 }
 
-def convert_license_to_spdx(lic, document, d):
+def convert_license_to_spdx(lic, document, d, existing={}):
     from pathlib import Path
     import oe.spdx
 
@@ -109,8 +109,11 @@ def convert_license_to_spdx(lic, document, d):
         if spdx_license in license_data["licenses"]:
             return spdx_license
 
-        spdx_license = "LicenseRef-" + l
-        add_extracted_license(spdx_license, l)
+        try:
+            spdx_license = existing[l]
+        except KeyError:
+            spdx_license = "LicenseRef-" + l
+            add_extracted_license(spdx_license, l)
 
         return spdx_license
 
@@ -462,7 +465,14 @@ python do_create_spdx() {
     doc_sha1 = oe.sbom.write_doc(d, doc, "recipes")
     dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
 
+    recipe_ref = oe.spdx.SPDXExternalDocumentRef()
+    recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
+    recipe_ref.spdxDocument = doc.documentNamespace
+    recipe_ref.checksum.algorithm = "SHA1"
+    recipe_ref.checksum.checksumValue = doc_sha1
+
     sources = collect_dep_sources(d, dep_recipes)
+    found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos}
 
     if not is_native:
         bb.build.exec_func("read_subpackage_metadata", d)
@@ -482,13 +492,6 @@ python do_create_spdx() {
             package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
             package_doc.creationInfo.creators.append("Organization: OpenEmbedded ()")
             package_doc.creationInfo.creators.append("Person: N/A ()")
-
-            recipe_ref = oe.spdx.SPDXExternalDocumentRef()
-            recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
-            recipe_ref.spdxDocument = doc.documentNamespace
-            recipe_ref.checksum.algorithm = "SHA1"
-            recipe_ref.checksum.checksumValue = doc_sha1
-
             package_doc.externalDocumentRefs.append(recipe_ref)
 
             package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE")
@@ -498,7 +501,7 @@ python do_create_spdx() {
             spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
             spdx_package.name = pkg_name
             spdx_package.versionInfo = d.getVar("PV")
-            spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d)
+            spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses)
 
             package_doc.packages.append(spdx_package)
 
-- 
2.25.1

Re: [OE-core] [PATCH v2 4/4] create-spex: don't duplicate license texts in each package

[Peter Kjellerstedt]

> -----Original Message-----
> From: openembedded-core@lists.openembedded.org <openembedded-
> core@lists.openembedded.org> On Behalf Of Ross Burton
> Sent: den 3 september 2021 18:01
> To: openembedded-core@lists.openembedded.org
> Subject: [OE-core] [PATCH v2 4/4] create-spex: don't duplicate license

Change "create-spex" to "create-spdx".

//Peter

> texts in each package
> 
> Instead of putting the full license text for non-SPDX licenses into the
> recipe and every package, use links to the recipe from the packages if
> possible.
> 
> Signed-off-by: Ross Burton <ross.burton@arm.com>
> ---
>  meta/classes/create-spdx.bbclass | 25 ++++++++++++++-----------
>  1 file changed, 14 insertions(+), 11 deletions(-)
> 
> diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-
> spdx.bbclass
> index cbb9239991c..1e0b3605587 100644
> --- a/meta/classes/create-spdx.bbclass
> +++ b/meta/classes/create-spdx.bbclass
> @@ -50,7 +50,7 @@ python() {
>          d.setVar("SPDX_LICENSE_DATA", data)
>  }
> 
> -def convert_license_to_spdx(lic, document, d):
> +def convert_license_to_spdx(lic, document, d, existing={}):
>      from pathlib import Path
>      import oe.spdx
> 
> @@ -109,8 +109,11 @@ def convert_license_to_spdx(lic, document, d):
>          if spdx_license in license_data["licenses"]:
>              return spdx_license
> 
> -        spdx_license = "LicenseRef-" + l
> -        add_extracted_license(spdx_license, l)
> +        try:
> +            spdx_license = existing[l]
> +        except KeyError:
> +            spdx_license = "LicenseRef-" + l
> +            add_extracted_license(spdx_license, l)
> 
>          return spdx_license
> 
> @@ -462,7 +465,14 @@ python do_create_spdx() {
>      doc_sha1 = oe.sbom.write_doc(d, doc, "recipes")
>      dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
> 
> +    recipe_ref = oe.spdx.SPDXExternalDocumentRef()
> +    recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
> +    recipe_ref.spdxDocument = doc.documentNamespace
> +    recipe_ref.checksum.algorithm = "SHA1"
> +    recipe_ref.checksum.checksumValue = doc_sha1
> +
>      sources = collect_dep_sources(d, dep_recipes)
> +    found_licenses = {license.name:recipe_ref.externalDocumentId + ":" +
> license.licenseId for license in doc.hasExtractedLicensingInfos}
> 
>      if not is_native:
>          bb.build.exec_func("read_subpackage_metadata", d)
> @@ -482,13 +492,6 @@ python do_create_spdx() {
>              package_doc.creationInfo.creators.append("Tool: OpenEmbedded
> Core create-spdx.bbclass")
>              package_doc.creationInfo.creators.append("Organization:
> OpenEmbedded ()")
>              package_doc.creationInfo.creators.append("Person: N/A ()")
> -
> -            recipe_ref = oe.spdx.SPDXExternalDocumentRef()
> -            recipe_ref.externalDocumentId = "DocumentRef-recipe-" +
> recipe.name
> -            recipe_ref.spdxDocument = doc.documentNamespace
> -            recipe_ref.checksum.algorithm = "SHA1"
> -            recipe_ref.checksum.checksumValue = doc_sha1
> -
>              package_doc.externalDocumentRefs.append(recipe_ref)
> 
>              package_license = d.getVar("LICENSE:%s" % package) or
> d.getVar("LICENSE")
> @@ -498,7 +501,7 @@ python do_create_spdx() {
>              spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
>              spdx_package.name = pkg_name
>              spdx_package.versionInfo = d.getVar("PV")
> -            spdx_package.licenseDeclared =
> convert_license_to_spdx(package_license, package_doc, d)
> +            spdx_package.licenseDeclared =
> convert_license_to_spdx(package_license, package_doc, d, found_licenses)
> 
>              package_doc.packages.append(spdx_package)
> 
> --
> 2.25.1