[PATCHv2 0/4] Add support to handle INCOMPATIBLE_LICENSE

[PATCHv2 0/4] Add support to handle INCOMPATIBLE_LICENSE

[Aníbal Limón]

The next changes are to add support for handle INCOMPATIBLE_LICENSE in 
manifest creation, in order to do it the license_create_manifest was 
reimplemented in python and also added some tweaks for save disk space
during build.

This patchset v2 fixes an issue when try to handle () into ManifestVisitor
for more info see next patches,
	license: Split visit_string in LicenseVisitor
	license: Add support for handle INCOMPATIBLE_LICENSE in manifest
    creation

The following changes since commit 4a711028c709d4bb1421e1637ae3fb0ac404fb45:

  oe-selftest: devtool: add a proper test to see if tap devices exist (2015-05-07 14:59:43 +0100)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib alimon/license-v2
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=alimon/license-v2

Aníbal Limón (4):
  license_class: Reimplemented manifest creation in python
  license_class: Generalize license_ok function
  license: Split visit_string in LicenseVisitor
  license: Add support for handle INCOMPATIBLE_LICENSE in manifest    
    creation

 meta/classes/license.bbclass | 205 +++++++++++++++++++++++--------------------
 meta/lib/oe/license.py       | 106 +++++++++++++++++++++-
 2 files changed, 211 insertions(+), 100 deletions(-)

-- 
1.8.4.5

[PATCHv2 1/4] license_class: Reimplemented manifest creation in python

[Aníbal Limón]

Reimplemented license_manifest_create from shell to python for
INCOMPATIBLE_LICENSE handle using oe.license module.

Optimizations are made to avoid license copy now uses a hardlink
and symbolic link this helps to save space during build.

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/classes/license.bbclass | 163 ++++++++++++++++++++++---------------------
 1 file changed, 82 insertions(+), 81 deletions(-)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index d9409a9..975867d 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -25,87 +25,88 @@ python write_package_manifest() {
         'w+').write(image_list_installed_packages(d))
 }
 
-license_create_manifest() {
-        # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
-        if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
-          exit 0
-        fi
-
-	INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
-	LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
-	# remove existing license.manifest file
-	if [ -f ${LICENSE_MANIFEST} ]; then
-		rm ${LICENSE_MANIFEST}
-	fi
-	touch ${LICENSE_MANIFEST}
-	for pkg in ${INSTALLED_PKGS}; do
-		filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
-		pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
-
-		# check to see if the package name exists in the manifest. if so, bail.
-		if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
-			continue
-		fi
-
-		pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
-		pkged_name="$(basename $(readlink ${filename}))"
-		pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; p }" ${filename})"
-		pkged_size="$(sed -n "/^PKGSIZE_${pkged_name}: /{ s/^PKGSIZE_${pkged_name}: //; p }" ${filename})"
-		if [ -z "${pkged_lic}" ]; then
-			# fallback checking value of LICENSE
-			pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; p }" ${filename})"
-		fi
-
-		echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
-		echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
-		echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
-		echo "LICENSE:" ${pkged_lic} >> ${LICENSE_MANIFEST}
-		echo "" >> ${LICENSE_MANIFEST}
-
-		# If the package doesn't contain any file, that is, its size is 0, the license
-		# isn't relevant as far as the final image is concerned. So doing license check
-		# doesn't make much sense, skip it.
-		if [ "$pkged_size" = "0" ]; then
-			continue
-		fi
-
-		lics="$(echo ${pkged_lic} | sed "s/[|&()*]/ /g" | sed "s/  */ /g" )"
-		for lic in ${lics}; do
-			# to reference a license file trim trailing + symbol
-			if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
-				bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
-			fi
-		done
-	done
-
-	# Two options here:
-	# - Just copy the manifest
-	# - Copy the manifest and the license directories
-	# With both options set we see a .5 M increase in core-image-minimal
-	if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
-		mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
-		cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
-		if [ "${COPY_LIC_DIRS}" = "1" ]; then
-			for pkg in ${INSTALLED_PKGS}; do
-				mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
-				pkged_pn="$(oe-pkgdata-util -p ${PKGDATA_DIR} lookup-recipe ${pkg})"
-				for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
-					# Really don't need to copy the generics as they're 
-					# represented in the manifest and in the actual pkg licenses
-					# Doing so would make your image quite a bit larger
-					if [ "${lic#generic_}" = "${lic}" ]; then
-						cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
-					else
-						if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
-							cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
-						fi
-						ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
-					fi
-				done
-			done
-		fi
-	fi
-
+python license_create_manifest() {
+    import re
+    import oe.packagedata
+
+    build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
+    if build_images_from_feeds == "1":
+        return 0
+
+    pkg_dic = {}
+    package_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                        d.getVar('IMAGE_NAME', True), 'package.manifest')
+    with open(package_manifest, "r") as package_file:
+        pkg_list = package_file.read().split()
+        for pkg in pkg_list:
+            pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
+                                    'runtime-reverse', pkg)
+            pkg_name = os.path.basename(os.readlink(pkg_info))
+
+            pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
+            if not "LICENSE" in pkg_dic[pkg_name].keys():
+                pkg_lic_name = "LICENSE_" + pkg_name
+                pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
+
+    license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                        d.getVar('IMAGE_NAME', True), 'license.manifest')
+    with open(license_manifest, "w") as license_file:
+        for pkg in sorted(pkg_dic):
+            license_file.write("PACKAGE NAME: %s\n" % pkg)
+            license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
+            license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+            license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
+
+            # If the package doesn't contain any file, that is, its size is 0, the license
+            # isn't relevant as far as the final image is concerned. So doing license check
+            # doesn't make much sense, skip it.
+            if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
+                continue
+
+            licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
+            licenses = re.sub('  *', ' ', licenses)
+            for lic in licenses.split():
+                lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                                        pkg_dic[pkg]["PN"], "generic_%s" % 
+                                        re.sub('\+', '', lic))
+                if not os.path.exists(lic_file):
+                   bb.warn("The license listed %s was not in the "\ 
+                            "licenses collected for recipe %s" 
+                            % (lic, pkg_dic[pkg]["PN"]))
+
+    # Two options here:
+    # - Just copy the manifest
+    # - Copy the manifest and the license directories
+    # With both options set we see a .5 M increase in core-image-minimal
+    copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
+    copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
+    if copy_lic_manifest == "1":
+        rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'), 
+                                'usr', 'share', 'common-licenses')
+        os.makedirs(rootfs_license_dir)
+        rootfs_license_manifest = os.path.join(rootfs_license_dir,
+                                                'license.manifest')
+        os.link(license_manifest, rootfs_license_manifest)
+
+        if copy_lic_dirs == "1":
+            for pkg in sorted(pkg_dic):
+                pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
+                os.makedirs(pkg_rootfs_license_dir)
+                pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                                            pkg_dic[pkg]["PN"]) 
+                licenses = os.listdir(pkg_license_dir)
+                for lic in licenses:
+                    rootfs_license = os.path.join(rootfs_license_dir, lic)
+                    pkg_license = os.path.join(pkg_license_dir, lic)
+                    pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
+
+                    if re.match("^generic_.*$", lic):
+                        if not os.path.exists(rootfs_license):
+                            os.link(pkg_license, rootfs_license)
+
+                        os.symlink(os.path.join('..', lic), pkg_rootfs_license)
+                    else:
+                        os.link(pkg_license, pkg_rootfs_license)
 }
 
 python do_populate_lic() {
-- 
1.8.4.5

[PATCHv2 2/4] license_class: Generalize license_ok function

[Aníbal Limón]

Add dont_want_licenses as parameter to license_ok function and move it
to oe.license module in order to use in other modules.

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/classes/license.bbclass | 21 ++++-----------------
 meta/lib/oe/license.py       | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 975867d..780b9d5 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -341,36 +341,23 @@ def incompatible_license(d, dont_want_licenses, package=None):
     take into consideration 'or' operand.  dont_want_licenses should be passed
     as canonical (SPDX) names.
     """
-    import re
     import oe.license
-    from fnmatch import fnmatchcase as fnmatch
     license = d.getVar("LICENSE_%s" % package, True) if package else None
     if not license:
         license = d.getVar('LICENSE', True)
 
-    def license_ok(license):
-        for dwl in dont_want_licenses:
-            # If you want to exclude license named generically 'X', we
-            # surely want to exclude 'X+' as well.  In consequence, we
-            # will exclude a trailing '+' character from LICENSE in
-            # case INCOMPATIBLE_LICENSE is not a 'X+' license.
-            lic = license
-            if not re.search('\+$', dwl):
-                lic = re.sub('\+', '', license)
-            if fnmatch(lic, dwl):
-                return False
-        return True
-
     # Handles an "or" or two license sets provided by
     # flattened_licenses(), pick one that works if possible.
     def choose_lic_set(a, b):
-        return a if all(license_ok(lic) for lic in a) else b
+        return a if all(oe.license.license_ok(lic, dont_want_licenses) \
+		 for lic in a) else b
 
     try:
         licenses = oe.license.flattened_licenses(license, choose_lic_set)
     except oe.license.LicenseError as exc:
         bb.fatal('%s: %s' % (d.getVar('P', True), exc))
-    return any(not license_ok(canonical_license(d, l)) for l in licenses)
+    return any(not oe.license.license_ok(canonical_license(d, l), \
+		dont_want_licenses) for l in licenses)
 
 def check_license_flags(d):
     """
diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py
index 31ca15b..bc146a28 100644
--- a/meta/lib/oe/license.py
+++ b/meta/lib/oe/license.py
@@ -5,6 +5,20 @@ import ast
 import re
 from fnmatch import fnmatchcase as fnmatch
 
+def license_ok(license, dont_want_licenses):
+    """ Return False if License exist in dont_want_licenses else True """
+    for dwl in dont_want_licenses:
+        # If you want to exclude license named generically 'X', we
+        # surely want to exclude 'X+' as well.  In consequence, we
+        # will exclude a trailing '+' character from LICENSE in
+        # case INCOMPATIBLE_LICENSE is not a 'X+' license.
+        lic = license
+        if not re.search('\+$', dwl):
+            lic = re.sub('\+', '', license)
+        if fnmatch(lic, dwl):
+            return False
+    return True
+
 class LicenseError(Exception):
     pass
 
-- 
1.8.4.5

[PATCHv2 3/4] license: Split visit_string in LicenseVisitor

[Aníbal Limón]

Create get_elements and visit_elements in LicenseVisitor based
on visit_string this allow to do modifications on elements before
parsing with AST.

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/lib/oe/license.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py
index bc146a28..254279d 100644
--- a/meta/lib/oe/license.py
+++ b/meta/lib/oe/license.py
@@ -44,8 +44,8 @@ license_operator = re.compile('([' + license_operator_chars + '])')
 license_pattern = re.compile('[a-zA-Z0-9.+_\-]+$')
 
 class LicenseVisitor(ast.NodeVisitor):
-    """Syntax tree visitor which can accept OpenEmbedded license strings"""
-    def visit_string(self, licensestr):
+    """Get elements based on OpenEmbedded license strings"""
+    def get_elements(self, licensestr):
         new_elements = []
         elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
         for pos, element in enumerate(elements):
@@ -57,7 +57,16 @@ class LicenseVisitor(ast.NodeVisitor):
                 raise InvalidLicense(element)
             new_elements.append(element)
 
-        self.visit(ast.parse(' '.join(new_elements)))
+        return new_elements
+
+    """Syntax tree visitor which can accept elements previously generated with
+    OpenEmbedded license string"""
+    def visit_elements(self, elements):
+        self.visit(ast.parse(' '.join(elements)))
+
+    """Syntax tree visitor which can accept OpenEmbedded license strings"""
+    def visit_string(self, licensestr):
+        self.visit_elements(self.get_elements(licensestr))
 
 class FlattenVisitor(LicenseVisitor):
     """Flatten a license tree (parsed from a string) by selecting one of each
-- 
1.8.4.5

[PATCHv2 4/4] license: Add support for handle INCOMPATIBLE_LICENSE in manifest creation

[Aníbal Limón]

When INCOMPATIBLE_LICENSE's is specified it need to be removed from
license.manifest and also avoid copy to target image.

Add ManifestVisitor that walk the license string searching for
INCOMPATIBLE_LICENSE's if found remove it.

[YOCTO #6765]

Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
---
 meta/classes/license.bbclass | 29 +++++++++++++++--
 meta/lib/oe/license.py       | 77 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+), 3 deletions(-)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 780b9d5..54ab123 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -29,6 +29,10 @@ python license_create_manifest() {
     import re
     import oe.packagedata
 
+    bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
+    bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
+    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
+
     build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
     if build_images_from_feeds == "1":
         return 0
@@ -52,6 +56,18 @@ python license_create_manifest() {
                         d.getVar('IMAGE_NAME', True), 'license.manifest')
     with open(license_manifest, "w") as license_file:
         for pkg in sorted(pkg_dic):
+            if bad_licenses:
+                try:
+                    (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
+                        oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
+                        bad_licenses, canonical_license, d)
+                except oe.license.LicenseError as exc:
+                    bb.fatal('%s: %s' % (d.getVar('P', True), exc))
+            else:
+                pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
+                pkg_dic[pkg]["LICENSES"] = re.sub('  *', ' ', pkg_dic[pkg]["LICENSES"])
+                pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
+
             license_file.write("PACKAGE NAME: %s\n" % pkg)
             license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
             license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
@@ -63,9 +79,7 @@ python license_create_manifest() {
             if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
                 continue
 
-            licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
-            licenses = re.sub('  *', ' ', licenses)
-            for lic in licenses.split():
+            for lic in pkg_dic[pkg]["LICENSES"]:
                 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
                                         pkg_dic[pkg]["PN"], "generic_%s" % 
                                         re.sub('\+', '', lic))
@@ -101,11 +115,20 @@ python license_create_manifest() {
                     pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
 
                     if re.match("^generic_.*$", lic):
+                        generic_lic = re.search("^generic_(.*)$", lic).group(1)
+                        if oe.license.license_ok(canonical_license(d,
+                            generic_lic), bad_licenses) == False:
+                            continue
+
                         if not os.path.exists(rootfs_license):
                             os.link(pkg_license, rootfs_license)
 
                         os.symlink(os.path.join('..', lic), pkg_rootfs_license)
                     else:
+                        if oe.license.license_ok(canonical_license(d,
+                            lic), bad_licenses) == False:
+                            continue
+
                         os.link(pkg_license, pkg_rootfs_license)
 }
 
diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py
index 254279d..f0f661c 100644
--- a/meta/lib/oe/license.py
+++ b/meta/lib/oe/license.py
@@ -138,3 +138,80 @@ def is_included(licensestr, whitelist=None, blacklist=None):
         return False, excluded
     else:
         return True, included
+
+class ManifestVisitor(LicenseVisitor):
+    """Walk license tree (parsed from a string) removing the incompatible
+    licenses specified"""
+    def __init__(self, dont_want_licenses, canonical_license, d):
+        self._dont_want_licenses = dont_want_licenses
+        self._canonical_license = canonical_license
+        self._d = d
+        self._operators = []
+
+        self.licenses = []
+        self.licensestr = ''
+
+        LicenseVisitor.__init__(self)
+
+    def visit(self, node):
+        if isinstance(node, ast.Str):
+            lic = node.s
+
+            if license_ok(self._canonical_license(self._d, lic),
+                    self._dont_want_licenses) == True:
+                if self._operators:
+                    ops = []
+                    for op in self._operators:
+                        if op == '[':
+                            ops.append(op)
+                        elif op == ']':
+                            ops.append(op)
+                        else:
+                            if not ops:
+                                ops.append(op)
+                            elif ops[-1] in ['[', ']']:
+                                ops.append(op)
+                            else:
+                                ops[-1] = op 
+
+                    for op in ops:
+                        if op == '[' or op == ']':
+                            self.licensestr += op
+                        elif self.licenses:
+                            self.licensestr += ' ' + op + ' '
+
+                    self._operators = []
+
+                self.licensestr += lic
+                self.licenses.append(lic)
+        elif isinstance(node, ast.BitAnd):
+            self._operators.append("&")
+        elif isinstance(node, ast.BitOr):
+            self._operators.append("|")
+        elif isinstance(node, ast.List):
+            self._operators.append("[")
+        elif isinstance(node, ast.Load):
+            self.licensestr += "]"
+
+        self.generic_visit(node)
+
+def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d):
+    """Given a license string and dont_want_licenses list,
+       return license string filtered and a list of licenses"""
+    manifest = ManifestVisitor(dont_want_licenses, canonical_license, d)
+
+    try:
+        elements = manifest.get_elements(licensestr)
+
+        # Replace '()' to '[]' for handle in ast as List and Load types.
+        elements = ['[' if e == '(' else e for e in elements]
+        elements = [']' if e == ')' else e for e in elements]
+
+        manifest.visit_elements(elements)
+    except SyntaxError as exc:
+        raise LicenseSyntaxError(licensestr, exc)
+
+    # Replace '[]' to '()' for output correct license.
+    manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')')
+
+    return (manifest.licensestr, manifest.licenses)
-- 
1.8.4.5