* [PATCH v2 1/4] create-spdx: transform license list into a dict for faster lookups
@ 2021-09-03 16:00 Ross Burton
2021-09-03 16:00 ` [PATCH v2 2/4] create-spdx: remove redundant test Ross Burton
` (2 more replies)
0 siblings, 3 replies; 5+ messages in thread
From: Ross Burton @ 2021-09-03 16:00 UTC (permalink / raw)
To: openembedded-core
spdx-licenses.json contains an array of licenses objects. As we'll be
searching it often, convert that to a dictionary when we parse it.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
meta/classes/create-spdx.bbclass | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index a590ab596ac..73ccb3c990f 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -44,7 +44,10 @@ python() {
return
with open(d.getVar("SPDX_LICENSES"), "r") as f:
- d.setVar("SPDX_LICENSE_DATA", json.load(f))
+ data = json.load(f)
+ # Transform the license array to a dictionary
+ data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
+ d.setVar("SPDX_LICENSE_DATA", data)
}
def convert_license_to_spdx(lic, document, d):
@@ -55,9 +58,8 @@ def convert_license_to_spdx(lic, document, d):
def add_extracted_license(ident, name, text):
nonlocal document
- for lic_data in license_data["licenses"]:
- if lic_data["licenseId"] == ident:
- return False
+ if ident in license_data["licenses"]:
+ return False
spdx_lic = oe.spdx.SPDXExtractedLicensingInfo()
spdx_lic.name = name
@@ -79,9 +81,8 @@ def convert_license_to_spdx(lic, document, d):
return "OR"
spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l
- for lic_data in license_data["licenses"]:
- if lic_data["licenseId"] == spdx_license:
- return spdx_license
+ if spdx_license in license_data["licenses"]:
+ return spdx_license
spdx_license = "LicenseRef-" + l
--
2.25.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH v2 2/4] create-spdx: remove redundant test
2021-09-03 16:00 [PATCH v2 1/4] create-spdx: transform license list into a dict for faster lookups Ross Burton
@ 2021-09-03 16:00 ` Ross Burton
2021-09-03 16:00 ` [PATCH v2 3/4] create-spdx: embed unknown license texts Ross Burton
2021-09-03 16:00 ` [PATCH v2 4/4] create-spex: don't duplicate license texts in each package Ross Burton
2 siblings, 0 replies; 5+ messages in thread
From: Ross Burton @ 2021-09-03 16:00 UTC (permalink / raw)
To: openembedded-core
add_extracted_document() is only called if the license isn't known to
SPDX, so there's no need to check again.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
meta/classes/create-spdx.bbclass | 3 ---
1 file changed, 3 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 73ccb3c990f..529dee22918 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -58,9 +58,6 @@ def convert_license_to_spdx(lic, document, d):
def add_extracted_license(ident, name, text):
nonlocal document
- if ident in license_data["licenses"]:
- return False
-
spdx_lic = oe.spdx.SPDXExtractedLicensingInfo()
spdx_lic.name = name
spdx_lic.licenseId = ident
--
2.25.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH v2 3/4] create-spdx: embed unknown license texts
2021-09-03 16:00 [PATCH v2 1/4] create-spdx: transform license list into a dict for faster lookups Ross Burton
2021-09-03 16:00 ` [PATCH v2 2/4] create-spdx: remove redundant test Ross Burton
@ 2021-09-03 16:00 ` Ross Burton
2021-09-03 16:00 ` [PATCH v2 4/4] create-spex: don't duplicate license texts in each package Ross Burton
2 siblings, 0 replies; 5+ messages in thread
From: Ross Burton @ 2021-09-03 16:00 UTC (permalink / raw)
To: openembedded-core
For licenses which are not known to SPDX, find and embed the actual
license text in an ExtractedLicesingInfo block.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
meta/classes/create-spdx.bbclass | 51 +++++++++++++++++++++++---------
1 file changed, 37 insertions(+), 14 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 529dee22918..cbb9239991c 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -51,21 +51,49 @@ python() {
}
def convert_license_to_spdx(lic, document, d):
+ from pathlib import Path
import oe.spdx
+ available_licenses = d.getVar("AVAILABLE_LICENSES").split()
license_data = d.getVar("SPDX_LICENSE_DATA")
+ extracted = {}
- def add_extracted_license(ident, name, text):
+ def add_extracted_license(ident, name):
nonlocal document
- spdx_lic = oe.spdx.SPDXExtractedLicensingInfo()
- spdx_lic.name = name
- spdx_lic.licenseId = ident
- spdx_lic.extractedText = text
-
- document.hasExtractedLicensingInfos.append(spdx_lic)
+ if name in extracted:
+ return
+
+ extracted_info = oe.spdx.SPDXExtractedLicensingInfo()
+ extracted_info.name = name
+ extracted_info.licenseId = ident
+
+ if name == "PD":
+ # Special-case this.
+ extracted_info.extractedText = "Software released to the public domain"
+ elif name in available_licenses:
+ # This license can be found in COMMON_LICENSE_DIR or LICENSE_PATH
+ for directory in [d.getVar('COMMON_LICENSE_DIR')] + d.getVar('LICENSE_PATH').split():
+ try:
+ with (Path(directory) / name).open(errors="replace") as f:
+ extracted_info.extractedText = f.read()
+ break
+ except Exception as e:
+ # Error out, as the license was in available_licenses so
+ # should be on disk somewhere.
+ bb.error(f"Cannot find text for license {name}: {e}")
+ else:
+ # If it's not SPDX, or PD, or in available licenses, then NO_GENERIC_LICENSE must be set
+ filename = d.getVarFlag('NO_GENERIC_LICENSE', name)
+ if filename:
+ filename = d.expand("${S}/" + filename)
+ with open(filename, errors="replace") as f:
+ extracted_info.extractedText = f.read()
+ else:
+ bb.error(f"Cannot find any text for license {name}")
- return True
+ extracted[name] = extracted_info
+ document.hasExtractedLicensingInfos.append(extracted_info)
def convert(l):
if l == "(" or l == ")":
@@ -82,12 +110,7 @@ def convert_license_to_spdx(lic, document, d):
return spdx_license
spdx_license = "LicenseRef-" + l
-
- if l == "PD":
- add_extracted_license(spdx_license, l, "Software released to the public domain")
- elif add_extracted_license(spdx_license, l, "This software is licensed under the %s license" % l):
- pass
- #bb.warn("No SPDX License found for %s. Creating a place holder" % l)
+ add_extracted_license(spdx_license, l)
return spdx_license
--
2.25.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH v2 4/4] create-spex: don't duplicate license texts in each package
2021-09-03 16:00 [PATCH v2 1/4] create-spdx: transform license list into a dict for faster lookups Ross Burton
2021-09-03 16:00 ` [PATCH v2 2/4] create-spdx: remove redundant test Ross Burton
2021-09-03 16:00 ` [PATCH v2 3/4] create-spdx: embed unknown license texts Ross Burton
@ 2021-09-03 16:00 ` Ross Burton
2021-09-03 22:14 ` [OE-core] " Peter Kjellerstedt
2 siblings, 1 reply; 5+ messages in thread
From: Ross Burton @ 2021-09-03 16:00 UTC (permalink / raw)
To: openembedded-core
Instead of putting the full license text for non-SPDX licenses into the
recipe and every package, use links to the recipe from the packages if
possible.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
meta/classes/create-spdx.bbclass | 25 ++++++++++++++-----------
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index cbb9239991c..1e0b3605587 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -50,7 +50,7 @@ python() {
d.setVar("SPDX_LICENSE_DATA", data)
}
-def convert_license_to_spdx(lic, document, d):
+def convert_license_to_spdx(lic, document, d, existing={}):
from pathlib import Path
import oe.spdx
@@ -109,8 +109,11 @@ def convert_license_to_spdx(lic, document, d):
if spdx_license in license_data["licenses"]:
return spdx_license
- spdx_license = "LicenseRef-" + l
- add_extracted_license(spdx_license, l)
+ try:
+ spdx_license = existing[l]
+ except KeyError:
+ spdx_license = "LicenseRef-" + l
+ add_extracted_license(spdx_license, l)
return spdx_license
@@ -462,7 +465,14 @@ python do_create_spdx() {
doc_sha1 = oe.sbom.write_doc(d, doc, "recipes")
dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
+ recipe_ref = oe.spdx.SPDXExternalDocumentRef()
+ recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
+ recipe_ref.spdxDocument = doc.documentNamespace
+ recipe_ref.checksum.algorithm = "SHA1"
+ recipe_ref.checksum.checksumValue = doc_sha1
+
sources = collect_dep_sources(d, dep_recipes)
+ found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos}
if not is_native:
bb.build.exec_func("read_subpackage_metadata", d)
@@ -482,13 +492,6 @@ python do_create_spdx() {
package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
package_doc.creationInfo.creators.append("Organization: OpenEmbedded ()")
package_doc.creationInfo.creators.append("Person: N/A ()")
-
- recipe_ref = oe.spdx.SPDXExternalDocumentRef()
- recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
- recipe_ref.spdxDocument = doc.documentNamespace
- recipe_ref.checksum.algorithm = "SHA1"
- recipe_ref.checksum.checksumValue = doc_sha1
-
package_doc.externalDocumentRefs.append(recipe_ref)
package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE")
@@ -498,7 +501,7 @@ python do_create_spdx() {
spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
spdx_package.name = pkg_name
spdx_package.versionInfo = d.getVar("PV")
- spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d)
+ spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses)
package_doc.packages.append(spdx_package)
--
2.25.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [OE-core] [PATCH v2 4/4] create-spex: don't duplicate license texts in each package
2021-09-03 16:00 ` [PATCH v2 4/4] create-spex: don't duplicate license texts in each package Ross Burton
@ 2021-09-03 22:14 ` Peter Kjellerstedt
0 siblings, 0 replies; 5+ messages in thread
From: Peter Kjellerstedt @ 2021-09-03 22:14 UTC (permalink / raw)
To: Ross Burton, openembedded-core
> -----Original Message-----
> From: openembedded-core@lists.openembedded.org <openembedded-
> core@lists.openembedded.org> On Behalf Of Ross Burton
> Sent: den 3 september 2021 18:01
> To: openembedded-core@lists.openembedded.org
> Subject: [OE-core] [PATCH v2 4/4] create-spex: don't duplicate license
Change "create-spex" to "create-spdx".
//Peter
> texts in each package
>
> Instead of putting the full license text for non-SPDX licenses into the
> recipe and every package, use links to the recipe from the packages if
> possible.
>
> Signed-off-by: Ross Burton <ross.burton@arm.com>
> ---
> meta/classes/create-spdx.bbclass | 25 ++++++++++++++-----------
> 1 file changed, 14 insertions(+), 11 deletions(-)
>
> diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-
> spdx.bbclass
> index cbb9239991c..1e0b3605587 100644
> --- a/meta/classes/create-spdx.bbclass
> +++ b/meta/classes/create-spdx.bbclass
> @@ -50,7 +50,7 @@ python() {
> d.setVar("SPDX_LICENSE_DATA", data)
> }
>
> -def convert_license_to_spdx(lic, document, d):
> +def convert_license_to_spdx(lic, document, d, existing={}):
> from pathlib import Path
> import oe.spdx
>
> @@ -109,8 +109,11 @@ def convert_license_to_spdx(lic, document, d):
> if spdx_license in license_data["licenses"]:
> return spdx_license
>
> - spdx_license = "LicenseRef-" + l
> - add_extracted_license(spdx_license, l)
> + try:
> + spdx_license = existing[l]
> + except KeyError:
> + spdx_license = "LicenseRef-" + l
> + add_extracted_license(spdx_license, l)
>
> return spdx_license
>
> @@ -462,7 +465,14 @@ python do_create_spdx() {
> doc_sha1 = oe.sbom.write_doc(d, doc, "recipes")
> dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
>
> + recipe_ref = oe.spdx.SPDXExternalDocumentRef()
> + recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
> + recipe_ref.spdxDocument = doc.documentNamespace
> + recipe_ref.checksum.algorithm = "SHA1"
> + recipe_ref.checksum.checksumValue = doc_sha1
> +
> sources = collect_dep_sources(d, dep_recipes)
> + found_licenses = {license.name:recipe_ref.externalDocumentId + ":" +
> license.licenseId for license in doc.hasExtractedLicensingInfos}
>
> if not is_native:
> bb.build.exec_func("read_subpackage_metadata", d)
> @@ -482,13 +492,6 @@ python do_create_spdx() {
> package_doc.creationInfo.creators.append("Tool: OpenEmbedded
> Core create-spdx.bbclass")
> package_doc.creationInfo.creators.append("Organization:
> OpenEmbedded ()")
> package_doc.creationInfo.creators.append("Person: N/A ()")
> -
> - recipe_ref = oe.spdx.SPDXExternalDocumentRef()
> - recipe_ref.externalDocumentId = "DocumentRef-recipe-" +
> recipe.name
> - recipe_ref.spdxDocument = doc.documentNamespace
> - recipe_ref.checksum.algorithm = "SHA1"
> - recipe_ref.checksum.checksumValue = doc_sha1
> -
> package_doc.externalDocumentRefs.append(recipe_ref)
>
> package_license = d.getVar("LICENSE:%s" % package) or
> d.getVar("LICENSE")
> @@ -498,7 +501,7 @@ python do_create_spdx() {
> spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
> spdx_package.name = pkg_name
> spdx_package.versionInfo = d.getVar("PV")
> - spdx_package.licenseDeclared =
> convert_license_to_spdx(package_license, package_doc, d)
> + spdx_package.licenseDeclared =
> convert_license_to_spdx(package_license, package_doc, d, found_licenses)
>
> package_doc.packages.append(spdx_package)
>
> --
> 2.25.1
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2021-09-03 22:15 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-03 16:00 [PATCH v2 1/4] create-spdx: transform license list into a dict for faster lookups Ross Burton
2021-09-03 16:00 ` [PATCH v2 2/4] create-spdx: remove redundant test Ross Burton
2021-09-03 16:00 ` [PATCH v2 3/4] create-spdx: embed unknown license texts Ross Burton
2021-09-03 16:00 ` [PATCH v2 4/4] create-spex: don't duplicate license texts in each package Ross Burton
2021-09-03 22:14 ` [OE-core] " Peter Kjellerstedt
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.