[PATCHv3 0/6] Create image manifest

[PATCHv3 0/6] Create image manifest

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

When building the rootfs the rootfs is created but the recipes
that deployed files in the image doesn't show up. Usually these
recipes include the bootloaders and the kernel.

With these patches a new manifest will be created that includes
the deployed recipes that were in the deployed. The format of the
manifest is different because the recipes doesn't install packages,
instead the recipes deploy files into the image

[YOCTO #6772]


Changes in v3:
- Create recipinfo file in every license folder to have the required
  information for the manifest file without using PKGDATA

The following changes since commit 1b25a70991589ed1f123015c16ee4806c46e3199:

  yocto-project-qs, ref-manual, poky.ent: CentOS Package updates (2015-11-18 16:42:22 +0000)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib mariano/bug6772v3
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug6772v3

Mariano Lopez (6):
  license.bbclass: Write recipeinfo file in license folder
  license.bbclass: Split license create manifest
  license.bbclass: Added get_boot_dependencies function
  license.bbclass: Added function get_deployed_dependencies
  license.bbclass: Add function get_deployed_files
  license.bbclass: Create image license manifest

 meta/classes/license.bbclass | 207 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 182 insertions(+), 25 deletions(-)

-- 
1.8.4.5

[PATCHv3 1/6] license.bbclass: Write recipeinfo file in license folder

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

Currently there is no way to get the recipe version when
creating the rootfs. It is needed because the manifest
file for the image has to contain this important piece
of information.

This change writes a new file in the license folder for
every recipe. This file is called "recipeinfo" and have
the information used to write the manifest file for the
recipes deployed next to the image.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index c616a20..7cf39e0 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -142,6 +142,10 @@ python do_populate_lic() {
     # The base directory we wrangle licenses to
     destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
     copy_license_files(lic_files_paths, destdir)
+    info = get_recipe_info(d)
+    with open(os.path.join(destdir, "recipeinfo"), "w") as f:
+        for key in sorted(info.keys()):
+            f.write("%s: %s\n" % (key, info[key]))
 }
 
 # it would be better to copy them in do_install_append, but find_license_filesa is python
@@ -156,6 +160,13 @@ python perform_packagecopy_prepend () {
         add_package_and_files(d)
 }
 
+def get_recipe_info(d):
+    info = {}
+    info["PV"] = d.getVar("PV", True)
+    info["PR"] = d.getVar("PR", True)
+    info["LICENSE"] = d.getVar("LICENSE")
+    return info
+
 def add_package_and_files(d):
     packages = d.getVar('PACKAGES', True)
     files = d.getVar('LICENSE_FILES_DIRECTORY', True)
-- 
1.8.4.5

[PATCHv3 2/6] license.bbclass: Split license create manifest

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This changes moves the writing of the licenses to a
separated function that could be called for other packages.

With these change it will be easier to reuse the writing of
the license for the packages deployed but not installed in
the rootfs.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 7cf39e0..62222f2 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -26,14 +26,9 @@ python write_package_manifest() {
 }
 
 python license_create_manifest() {
-    import re
     import oe.packagedata
     from oe.rootfs import image_list_installed_packages
 
-    bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
-    bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
-    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
-
     build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
     if build_images_from_feeds == "1":
         return 0
@@ -49,8 +44,18 @@ python license_create_manifest() {
             pkg_lic_name = "LICENSE_" + pkg_name
             pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
 
-    license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+    rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
                         d.getVar('IMAGE_NAME', True), 'license.manifest')
+    write_license_files(d, rootfs_license_manifest, pkg_dic)
+}
+
+def write_license_files(d, license_manifest, pkg_dic):
+    import re
+
+    bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
+    bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
+    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
+
     with open(license_manifest, "w") as license_file:
         for pkg in sorted(pkg_dic):
             if bad_licenses:
@@ -98,15 +103,16 @@ python license_create_manifest() {
     if copy_lic_manifest == "1":
         rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'), 
                                 'usr', 'share', 'common-licenses')
-        os.makedirs(rootfs_license_dir)
+        bb.utils.mkdirhier(rootfs_license_dir)
         rootfs_license_manifest = os.path.join(rootfs_license_dir,
-                                                'license.manifest')
-        os.link(license_manifest, rootfs_license_manifest)
+                os.path.split(license_manifest)[1])
+        if not os.path.exists(rootfs_license_manifest):
+            os.link(license_manifest, rootfs_license_manifest)
 
         if copy_lic_dirs == "1":
             for pkg in sorted(pkg_dic):
                 pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
-                os.makedirs(pkg_rootfs_license_dir)
+                bb.utils.mkdirhier(pkg_rootfs_license_dir)
                 pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
                                             pkg_dic[pkg]["PN"]) 
                 licenses = os.listdir(pkg_license_dir)
@@ -124,14 +130,16 @@ python license_create_manifest() {
                         if not os.path.exists(rootfs_license):
                             os.link(pkg_license, rootfs_license)
 
-                        os.symlink(os.path.join('..', lic), pkg_rootfs_license)
+                        if not os.path.exists(pkg_rootfs_license):
+                            os.symlink(os.path.join('..', lic), pkg_rootfs_license)
                     else:
-                        if oe.license.license_ok(canonical_license(d,
-                            lic), bad_licenses) == False:
+                        if (oe.license.license_ok(canonical_license(d,
+                                lic), bad_licenses) == False or
+                                os.path.exists(pkg_rootfs_license)):
                             continue
 
                         os.link(pkg_license, pkg_rootfs_license)
-}
+
 
 python do_populate_lic() {
     """
-- 
1.8.4.5

[PATCHv3 3/6] license.bbclass: Added get_boot_dependencies function

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This function gets the dependencies from the classes that
create a boot image, this is required because sometimes
the bootloader dependecy is in these classes. The current
classes covered are bootimg and bootdirectdisk because
these are the only clases that add dependencies.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 62222f2..afcc708 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -141,6 +141,42 @@ def write_license_files(d, license_manifest, pkg_dic):
                         os.link(pkg_license, pkg_rootfs_license)
 
 
+def get_boot_dependencies(d):
+    """
+    Return the dependencies from boot tasks
+    """
+
+    depends = ""
+    boot_depends_string = ""
+    taskdepdata = d.getVar("BB_TASKDEPDATA", True)
+    # Only bootimg and bootdirectdisk include the depends flag
+    boot_tasks = ["do_bootimg", "do_bootdirectdisk",]
+
+    for task in boot_tasks:
+        boot_depends_string = "%s %s" % (boot_depends_string,
+                d.getVarFlag(task, "depends", True) or "")
+    boot_depends = [dep.split(":")[0] for dep
+                in boot_depends_string.split()
+                if not dep.split(":")[0].endswith("-native")]
+    for dep in boot_depends:
+        info_file = os.path.join(d.getVar("LICENSE_DIRECTORY", True),
+                dep, "recipeinfo")
+        # If the recipe and dependency name is the same
+        if os.path.exists(info_file):
+            depends = "%s %s" % (depends, dep)
+        # We need to search for the provider of the dependency
+        else:
+            for taskdep in taskdepdata.itervalues():
+                # The fifth field contains what the task provides
+                if dep in taskdep[4]:
+                    info_file = os.path.join(
+                            d.getVar("LICENSE_DIRECTORY", True),
+                            taskdep[0], "recipeinfo")
+                    if os.path.exists(info_file):
+                        depends = "%s %s" % (depends, taskdep[0])
+                        break
+    return depends
+
 python do_populate_lic() {
     """
     Populate LICENSE_DIRECTORY with licenses.
-- 
1.8.4.5

[PATCHv3 4/6] license.bbclass: Added function get_deployed_dependencies

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This change introduce a new function to get the dependencies
that were deployed. It uses BB_TASKDEPDATAto get all the
dependencies of the current task, so it is possible to get
different packages depending at what point this function is
called.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 48 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index afcc708..f012783 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -141,6 +141,54 @@ def write_license_files(d, license_manifest, pkg_dic):
                         os.link(pkg_license, pkg_rootfs_license)
 
 
+def get_deployed_dependencies(d):
+    """
+    Get all the deployed dependencies of an image
+    """
+
+    deploy = {}
+    # Get all the dependencies for the current task (rootfs).
+    # Also get EXTRA_IMAGEDEPENDS because the bootloader is
+    # usually in this var and not listed in rootfs.
+    # At last, get the dependencies from boot classes because
+    # it might contain the bootloader.
+    taskdata = d.getVar("BB_TASKDEPDATA", True)
+    depends = list(set([dep[0] for dep
+                    in taskdata.itervalues()
+                    if not dep[0].endswith("-native")]))
+    extra_depends = d.getVar("EXTRA_IMAGEDEPENDS", True)
+    boot_depends = get_boot_dependencies(d)
+    depends.extend(extra_depends.split())
+    depends.extend(boot_depends.split())
+    depends = list(set(depends))
+
+    # To verify what was deployed it checks the rootfs dependencies against
+    # the SSTATE_MANIFESTS for "deploy" task.
+    # The manifest file name contains the arch. Because we are not running
+    # in the recipe context it is necessary to check every arch used.
+    sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS", True)
+    sstate_archs = d.getVar("SSTATE_ARCHS", True)
+    extra_archs = d.getVar("PACKAGE_EXTRA_ARCHS", True)
+    archs = list(set(("%s %s" % (sstate_archs, extra_archs)).split()))
+    for dep in depends:
+        # Some recipes have an arch on their own, so we try that first.
+        special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep, True)
+        if special_arch:
+            sstate_manifest_file = os.path.join(sstate_manifest_dir,
+                    "manifest-%s-%s.deploy" % (special_arch, dep))
+            if os.path.exists(sstate_manifest_file):
+                deploy[dep] = sstate_manifest_file
+                continue
+
+        for arch in archs:
+            sstate_manifest_file = os.path.join(sstate_manifest_dir,
+                    "manifest-%s-%s.deploy" % (arch, dep))
+            if os.path.exists(sstate_manifest_file):
+                deploy[dep] = sstate_manifest_file
+                break
+
+    return deploy
+
 def get_boot_dependencies(d):
     """
     Return the dependencies from boot tasks
-- 
1.8.4.5

[PATCHv3 5/6] license.bbclass: Add function get_deployed_files

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This function will get the files that were deployed using
the sstate-control manifest file. This will give a better
view of what was deployed next to the image.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index f012783..dbec4d7 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -225,6 +225,21 @@ def get_boot_dependencies(d):
                         break
     return depends
 
+def get_deployed_files(man_file):
+    """
+    This will get the files deployed from the sstate manifest
+    """
+
+    dep_files = ""
+    excluded_files = ["README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt"]
+    with open(man_file, "r") as manifest:
+        all_files = manifest.read()
+    for f in all_files.splitlines():
+        if ((not (os.path.islink(f) or os.path.isdir(f))) and
+                not os.path.basename(f) in excluded_files):
+            dep_files = "%s %s" % (dep_files, os.path.basename(f))
+    return dep_files
+
 python do_populate_lic() {
     """
     Populate LICENSE_DIRECTORY with licenses.
-- 
1.8.4.5

[PATCHv3 6/6] license.bbclass: Create image license manifest

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This change adds the license_deployed_manifest function
that will create the manifest for the packages deployed
next to the image but not installed in rootfs. Some
examples of these recipes would be the bootloaders, or
the kernel.

This new function was added to ROOTFS_POSTPROCESS_COMMAND
so it will run after every rootfs task.

This change also modify the write_license_files because
the image manifest is different from the root manifest.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 63 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 51 insertions(+), 12 deletions(-)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index dbec4d7..8d2bc0d 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -25,6 +25,10 @@ python write_package_manifest() {
         'w+').write(image_list_installed_packages(d))
 }
 
+python write_deploy_manifest() {
+    license_deployed_manifest(d)
+}
+
 python license_create_manifest() {
     import oe.packagedata
     from oe.rootfs import image_list_installed_packages
@@ -70,16 +74,24 @@ def write_license_files(d, license_manifest, pkg_dic):
                 pkg_dic[pkg]["LICENSES"] = re.sub('  *', ' ', pkg_dic[pkg]["LICENSES"])
                 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
 
-            license_file.write("PACKAGE NAME: %s\n" % pkg)
-            license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
-            license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
-            license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
-
-            # If the package doesn't contain any file, that is, its size is 0, the license
-            # isn't relevant as far as the final image is concerned. So doing license check
-            # doesn't make much sense, skip it.
-            if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
-                continue
+            if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
+                # Rootfs manifest
+                license_file.write("PACKAGE NAME: %s\n" % pkg)
+                license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
+                license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+                license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
+
+                # If the package doesn't contain any file, that is, its size is 0, the license
+                # isn't relevant as far as the final image is concerned. So doing license check
+                # doesn't make much sense, skip it.
+                if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
+                    continue
+            else:
+                # Image manifest
+                license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+                license_file.write("VERSION: %s\n" % pkg_dic[pkg]["PV"])
+                license_file.write("LICENSE: %s\n" % pkg_dic[pkg]["LICENSE"])
+                license_file.write("FILES:%s\n\n" % pkg_dic[pkg]["FILES"])
 
             for lic in pkg_dic[pkg]["LICENSES"]:
                 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
@@ -141,6 +153,33 @@ def write_license_files(d, license_manifest, pkg_dic):
                         os.link(pkg_license, pkg_rootfs_license)
 
 
+def license_deployed_manifest(d):
+    """
+    Write the license manifest for the deployed recipes.
+    The deployed recipes usually includes the bootloader
+    and extra files to boot the target.
+    """
+
+    dep_dic = {}
+    man_dic = {}
+    lic_dir = d.getVar("LICENSE_DIRECTORY", True)
+
+    dep_dic = get_deployed_dependencies(d)
+    for dep in dep_dic.keys():
+        man_dic[dep] = {}
+        # It is necessary to mark this will be used for image manifest
+        man_dic[dep]["IMAGE_MANIFEST"] = True
+        man_dic[dep]["PN"] = dep
+        man_dic[dep]["FILES"] = get_deployed_files(dep_dic[dep])
+        with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f:
+            for line in f.readlines():
+                key,val = line.split(": ", 1)
+                man_dic[dep][key] = val[:-1]
+
+    image_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+            d.getVar('IMAGE_NAME', True), 'image_license.manifest')
+    write_license_files(d, image_license_manifest, man_dic)
+
 def get_deployed_dependencies(d):
     """
     Get all the deployed dependencies of an image
@@ -227,7 +266,7 @@ def get_boot_dependencies(d):
 
 def get_deployed_files(man_file):
     """
-    This will get the files deployed from the sstate manifest
+    Get the files deployed from the sstate manifest
     """
 
     dep_files = ""
@@ -591,7 +630,7 @@ SSTATETASKS += "do_populate_lic"
 do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}"
 do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/"
 
-ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; "
+ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; write_deploy_manifest; license_create_manifest; "
 
 do_populate_lic_setscene[dirs] = "${LICSSTATEDIR}/${PN}"
 do_populate_lic_setscene[cleandirs] = "${LICSSTATEDIR}"
-- 
1.8.4.5

Re: [PATCHv3 1/6] license.bbclass: Write recipeinfo file in license folder

[Paul Eggleton]

Hi Mariano,

On Thursday 19 November 2015 14:25:14 mariano.lopez@linux.intel.com wrote:
> From: Mariano Lopez <mariano.lopez@linux.intel.com>
> 
> Currently there is no way to get the recipe version when
> creating the rootfs. It is needed because the manifest
> file for the image has to contain this important piece
> of information.
> 
> This change writes a new file in the license folder for
> every recipe. This file is called "recipeinfo" and have
> the information used to write the manifest file for the
> recipes deployed next to the image.
> 
> [YOCTO #6772]
> 
> Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
> ---
>  meta/classes/license.bbclass | 11 +++++++++++
>  1 file changed, 11 insertions(+)
> 
> diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
> index c616a20..7cf39e0 100644
> --- a/meta/classes/license.bbclass
> +++ b/meta/classes/license.bbclass
> @@ -142,6 +142,10 @@ python do_populate_lic() {
>      # The base directory we wrangle licenses to
>      destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN',
> True)) copy_license_files(lic_files_paths, destdir)
> +    info = get_recipe_info(d)
> +    with open(os.path.join(destdir, "recipeinfo"), "w") as f:
> +        for key in sorted(info.keys()):
> +            f.write("%s: %s\n" % (key, info[key]))
>  }
> 
>  # it would be better to copy them in do_install_append, but
> find_license_filesa is python @@ -156,6 +160,13 @@ python
> perform_packagecopy_prepend () {
>          add_package_and_files(d)
>  }
> 
> +def get_recipe_info(d):
> +    info = {}
> +    info["PV"] = d.getVar("PV", True)
> +    info["PR"] = d.getVar("PR", True)
> +    info["LICENSE"] = d.getVar("LICENSE")

This should be d.getVar("LICENSE", True) so that the value gets expanded 
(usually there aren't any variable references in LICENSE values but we can't 
discount it).

> +    return info
> +
>  def add_package_and_files(d):
>      packages = d.getVar('PACKAGES', True)
>      files = d.getVar('LICENSE_FILES_DIRECTORY', True)

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: [PATCHv3 3/6] license.bbclass: Added get_boot_dependencies function

[Paul Eggleton]

On Thursday 19 November 2015 14:25:16 mariano.lopez@linux.intel.com wrote:
> From: Mariano Lopez <mariano.lopez@linux.intel.com>
> 
> This function gets the dependencies from the classes that
> create a boot image, this is required because sometimes
> the bootloader dependecy is in these classes. The current
> classes covered are bootimg and bootdirectdisk because
> these are the only clases that add dependencies.
> 
> [YOCTO #6772]
> 
> Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
> ---
>  meta/classes/license.bbclass | 36 ++++++++++++++++++++++++++++++++++++
>  1 file changed, 36 insertions(+)
> 
> diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
> index 62222f2..afcc708 100644
> --- a/meta/classes/license.bbclass
> +++ b/meta/classes/license.bbclass
> @@ -141,6 +141,42 @@ def write_license_files(d, license_manifest, pkg_dic):
>                          os.link(pkg_license, pkg_rootfs_license)
> 
> 
> +def get_boot_dependencies(d):
> +    """
> +    Return the dependencies from boot tasks
> +    """
> +
> +    depends = ""
> +    boot_depends_string = ""
> +    taskdepdata = d.getVar("BB_TASKDEPDATA", True)
> +    # Only bootimg and bootdirectdisk include the depends flag
> +    boot_tasks = ["do_bootimg", "do_bootdirectdisk",]
> +
> +    for task in boot_tasks:
> +        boot_depends_string = "%s %s" % (boot_depends_string,
> +                d.getVarFlag(task, "depends", True) or "")
> +    boot_depends = [dep.split(":")[0] for dep
> +                in boot_depends_string.split()
> +                if not dep.split(":")[0].endswith("-native")]
> +    for dep in boot_depends:
> +        info_file = os.path.join(d.getVar("LICENSE_DIRECTORY", True),
> +                dep, "recipeinfo")
> +        # If the recipe and dependency name is the same
> +        if os.path.exists(info_file):
> +            depends = "%s %s" % (depends, dep)
> +        # We need to search for the provider of the dependency
> +        else:
> +            for taskdep in taskdepdata.itervalues():
> +                # The fifth field contains what the task provides
> +                if dep in taskdep[4]:
> +                    info_file = os.path.join(
> +                            d.getVar("LICENSE_DIRECTORY", True),
> +                            taskdep[0], "recipeinfo")
> +                    if os.path.exists(info_file):
> +                        depends = "%s %s" % (depends, taskdep[0])
> +                        break
> +    return depends

Where a function returns a list of items it should return an actual list 
rather than a space-separated string; this gives the caller a little more 
flexibility on how it processes the returned list. (Obviously here the list is 
going to be written out immediately as a space-separated string, but that's as 
easy as ' '.join(listname) in the calling function.)

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: [PATCHv3 5/6] license.bbclass: Add function get_deployed_files

[Paul Eggleton]

On Thursday 19 November 2015 14:25:18 mariano.lopez@linux.intel.com wrote:
> From: Mariano Lopez <mariano.lopez@linux.intel.com>
> 
> This function will get the files that were deployed using
> the sstate-control manifest file. This will give a better
> view of what was deployed next to the image.
> 
> [YOCTO #6772]
> 
> Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
> ---
>  meta/classes/license.bbclass | 15 +++++++++++++++
>  1 file changed, 15 insertions(+)
> 
> diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
> index f012783..dbec4d7 100644
> --- a/meta/classes/license.bbclass
> +++ b/meta/classes/license.bbclass
> @@ -225,6 +225,21 @@ def get_boot_dependencies(d):
>                          break
>      return depends
> 
> +def get_deployed_files(man_file):
> +    """
> +    This will get the files deployed from the sstate manifest
> +    """
> +
> +    dep_files = ""
> +    excluded_files = ["README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt"]
> +    with open(man_file, "r") as manifest:
> +        all_files = manifest.read()
> +    for f in all_files.splitlines():
> +        if ((not (os.path.islink(f) or os.path.isdir(f))) and
> +                not os.path.basename(f) in excluded_files):
> +            dep_files = "%s %s" % (dep_files, os.path.basename(f))
> +    return dep_files

As per get_boot_dependencies(), best to return a proper list here.

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

[PATCHv4 1/6] license.bbclass: Write recipeinfo file in license folder

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

Currently there is no way to get the recipe version when
creating the rootfs. It is needed because the manifest
file for the image has to contain this important piece
of information.

This change writes a new file in the license folder for
every recipe. This file is called "recipeinfo" and have
the information used to write the manifest file for the
recipes deployed next to the image.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index c616a20..76f40c9 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -142,6 +142,10 @@ python do_populate_lic() {
     # The base directory we wrangle licenses to
     destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
     copy_license_files(lic_files_paths, destdir)
+    info = get_recipe_info(d)
+    with open(os.path.join(destdir, "recipeinfo"), "w") as f:
+        for key in sorted(info.keys()):
+            f.write("%s: %s\n" % (key, info[key]))
 }
 
 # it would be better to copy them in do_install_append, but find_license_filesa is python
@@ -156,6 +160,13 @@ python perform_packagecopy_prepend () {
         add_package_and_files(d)
 }
 
+def get_recipe_info(d):
+    info = {}
+    info["PV"] = d.getVar("PV", True)
+    info["PR"] = d.getVar("PR", True)
+    info["LICENSE"] = d.getVar("LICENSE", True)
+    return info
+
 def add_package_and_files(d):
     packages = d.getVar('PACKAGES', True)
     files = d.getVar('LICENSE_FILES_DIRECTORY', True)
-- 
1.8.4.5

[PATCHv4 2/6] license.bbclass: Split license create manifest

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This changes moves the writing of the licenses to a
separated function that could be called for other packages.

With these change it will be easier to reuse the writing of
the license for the packages deployed but not installed in
the rootfs.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 76f40c9..83ae771 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -26,14 +26,9 @@ python write_package_manifest() {
 }
 
 python license_create_manifest() {
-    import re
     import oe.packagedata
     from oe.rootfs import image_list_installed_packages
 
-    bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
-    bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
-    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
-
     build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
     if build_images_from_feeds == "1":
         return 0
@@ -49,8 +44,18 @@ python license_create_manifest() {
             pkg_lic_name = "LICENSE_" + pkg_name
             pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
 
-    license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+    rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
                         d.getVar('IMAGE_NAME', True), 'license.manifest')
+    write_license_files(d, rootfs_license_manifest, pkg_dic)
+}
+
+def write_license_files(d, license_manifest, pkg_dic):
+    import re
+
+    bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
+    bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
+    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
+
     with open(license_manifest, "w") as license_file:
         for pkg in sorted(pkg_dic):
             if bad_licenses:
@@ -98,15 +103,16 @@ python license_create_manifest() {
     if copy_lic_manifest == "1":
         rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'), 
                                 'usr', 'share', 'common-licenses')
-        os.makedirs(rootfs_license_dir)
+        bb.utils.mkdirhier(rootfs_license_dir)
         rootfs_license_manifest = os.path.join(rootfs_license_dir,
-                                                'license.manifest')
-        os.link(license_manifest, rootfs_license_manifest)
+                os.path.split(license_manifest)[1])
+        if not os.path.exists(rootfs_license_manifest):
+            os.link(license_manifest, rootfs_license_manifest)
 
         if copy_lic_dirs == "1":
             for pkg in sorted(pkg_dic):
                 pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
-                os.makedirs(pkg_rootfs_license_dir)
+                bb.utils.mkdirhier(pkg_rootfs_license_dir)
                 pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
                                             pkg_dic[pkg]["PN"]) 
                 licenses = os.listdir(pkg_license_dir)
@@ -124,14 +130,16 @@ python license_create_manifest() {
                         if not os.path.exists(rootfs_license):
                             os.link(pkg_license, rootfs_license)
 
-                        os.symlink(os.path.join('..', lic), pkg_rootfs_license)
+                        if not os.path.exists(pkg_rootfs_license):
+                            os.symlink(os.path.join('..', lic), pkg_rootfs_license)
                     else:
-                        if oe.license.license_ok(canonical_license(d,
-                            lic), bad_licenses) == False:
+                        if (oe.license.license_ok(canonical_license(d,
+                                lic), bad_licenses) == False or
+                                os.path.exists(pkg_rootfs_license)):
                             continue
 
                         os.link(pkg_license, pkg_rootfs_license)
-}
+
 
 python do_populate_lic() {
     """
-- 
1.8.4.5

[PATCHv4 3/6] license.bbclass: Added get_boot_dependencies function

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This function gets the dependencies from the classes that
create a boot image, this is required because sometimes
the bootloader dependecy is in these classes. The current
classes covered are bootimg and bootdirectdisk because
these are the only clases that add dependencies.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 83ae771..457ad85 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -141,6 +141,42 @@ def write_license_files(d, license_manifest, pkg_dic):
                         os.link(pkg_license, pkg_rootfs_license)
 
 
+def get_boot_dependencies(d):
+    """
+    Return the dependencies from boot tasks
+    """
+
+    depends = []
+    boot_depends_string = ""
+    taskdepdata = d.getVar("BB_TASKDEPDATA", True)
+    # Only bootimg and bootdirectdisk include the depends flag
+    boot_tasks = ["do_bootimg", "do_bootdirectdisk",]
+
+    for task in boot_tasks:
+        boot_depends_string = "%s %s" % (boot_depends_string,
+                d.getVarFlag(task, "depends", True) or "")
+    boot_depends = [dep.split(":")[0] for dep
+                in boot_depends_string.split()
+                if not dep.split(":")[0].endswith("-native")]
+    for dep in boot_depends:
+        info_file = os.path.join(d.getVar("LICENSE_DIRECTORY", True),
+                dep, "recipeinfo")
+        # If the recipe and dependency name is the same
+        if os.path.exists(info_file):
+            depends.append(dep)
+        # We need to search for the provider of the dependency
+        else:
+            for taskdep in taskdepdata.itervalues():
+                # The fifth field contains what the task provides
+                if dep in taskdep[4]:
+                    info_file = os.path.join(
+                            d.getVar("LICENSE_DIRECTORY", True),
+                            taskdep[0], "recipeinfo")
+                    if os.path.exists(info_file):
+                        depends.append(taskdep[0])
+                        break
+    return depends
+
 python do_populate_lic() {
     """
     Populate LICENSE_DIRECTORY with licenses.
-- 
1.8.4.5

[PATCHv4 4/6] license.bbclass: Added function get_deployed_dependencies

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This change introduce a new function to get the dependencies
that were deployed. It uses BB_TASKDEPDATAto get all the
dependencies of the current task, so it is possible to get
different packages depending at what point this function is
called.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 48 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 457ad85..11a5004 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -141,6 +141,54 @@ def write_license_files(d, license_manifest, pkg_dic):
                         os.link(pkg_license, pkg_rootfs_license)
 
 
+def get_deployed_dependencies(d):
+    """
+    Get all the deployed dependencies of an image
+    """
+
+    deploy = {}
+    # Get all the dependencies for the current task (rootfs).
+    # Also get EXTRA_IMAGEDEPENDS because the bootloader is
+    # usually in this var and not listed in rootfs.
+    # At last, get the dependencies from boot classes because
+    # it might contain the bootloader.
+    taskdata = d.getVar("BB_TASKDEPDATA", True)
+    depends = list(set([dep[0] for dep
+                    in taskdata.itervalues()
+                    if not dep[0].endswith("-native")]))
+    extra_depends = d.getVar("EXTRA_IMAGEDEPENDS", True)
+    boot_depends = get_boot_dependencies(d)
+    depends.extend(extra_depends.split())
+    depends.extend(boot_depends)
+    depends = list(set(depends))
+
+    # To verify what was deployed it checks the rootfs dependencies against
+    # the SSTATE_MANIFESTS for "deploy" task.
+    # The manifest file name contains the arch. Because we are not running
+    # in the recipe context it is necessary to check every arch used.
+    sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS", True)
+    sstate_archs = d.getVar("SSTATE_ARCHS", True)
+    extra_archs = d.getVar("PACKAGE_EXTRA_ARCHS", True)
+    archs = list(set(("%s %s" % (sstate_archs, extra_archs)).split()))
+    for dep in depends:
+        # Some recipes have an arch on their own, so we try that first.
+        special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep, True)
+        if special_arch:
+            sstate_manifest_file = os.path.join(sstate_manifest_dir,
+                    "manifest-%s-%s.deploy" % (special_arch, dep))
+            if os.path.exists(sstate_manifest_file):
+                deploy[dep] = sstate_manifest_file
+                continue
+
+        for arch in archs:
+            sstate_manifest_file = os.path.join(sstate_manifest_dir,
+                    "manifest-%s-%s.deploy" % (arch, dep))
+            if os.path.exists(sstate_manifest_file):
+                deploy[dep] = sstate_manifest_file
+                break
+
+    return deploy
+
 def get_boot_dependencies(d):
     """
     Return the dependencies from boot tasks
-- 
1.8.4.5

[PATCHv4 5/6] license.bbclass: Add function get_deployed_files

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This function will get the files that were deployed using
the sstate-control manifest file. This will give a better
view of what was deployed next to the image.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 11a5004..b7e14c77d 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -225,6 +225,21 @@ def get_boot_dependencies(d):
                         break
     return depends
 
+def get_deployed_files(man_file):
+    """
+    Get the files deployed from the sstate manifest
+    """
+
+    dep_files = []
+    excluded_files = ["README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt"]
+    with open(man_file, "r") as manifest:
+        all_files = manifest.read()
+    for f in all_files.splitlines():
+        if ((not (os.path.islink(f) or os.path.isdir(f))) and
+                not os.path.basename(f) in excluded_files):
+            dep_files.append(os.path.basename(f))
+    return dep_files
+
 python do_populate_lic() {
     """
     Populate LICENSE_DIRECTORY with licenses.
-- 
1.8.4.5

[PATCHv4 6/6] license.bbclass: Create image license manifest

[mariano.lopez]

From: Mariano Lopez <mariano.lopez@linux.intel.com>

This change adds the license_deployed_manifest function
that will create the manifest for the packages deployed
next to the image but not installed in rootfs. Some
examples of these recipes would be the bootloaders, or
the kernel.

This new function was added to ROOTFS_POSTPROCESS_COMMAND
so it will run after every rootfs task.

This change also modify the write_license_files because
the image manifest is different from the root manifest.

[YOCTO #6772]

Signed-off-by: Mariano Lopez <mariano.lopez@linux.intel.com>
---
 meta/classes/license.bbclass | 62 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 51 insertions(+), 11 deletions(-)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index b7e14c77d..f3cbb7d 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -25,6 +25,10 @@ python write_package_manifest() {
         'w+').write(image_list_installed_packages(d))
 }
 
+python write_deploy_manifest() {
+    license_deployed_manifest(d)
+}
+
 python license_create_manifest() {
     import oe.packagedata
     from oe.rootfs import image_list_installed_packages
@@ -70,16 +74,24 @@ def write_license_files(d, license_manifest, pkg_dic):
                 pkg_dic[pkg]["LICENSES"] = re.sub('  *', ' ', pkg_dic[pkg]["LICENSES"])
                 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
 
-            license_file.write("PACKAGE NAME: %s\n" % pkg)
-            license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
-            license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
-            license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
-
-            # If the package doesn't contain any file, that is, its size is 0, the license
-            # isn't relevant as far as the final image is concerned. So doing license check
-            # doesn't make much sense, skip it.
-            if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
-                continue
+            if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
+                # Rootfs manifest
+                license_file.write("PACKAGE NAME: %s\n" % pkg)
+                license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
+                license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+                license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
+
+                # If the package doesn't contain any file, that is, its size is 0, the license
+                # isn't relevant as far as the final image is concerned. So doing license check
+                # doesn't make much sense, skip it.
+                if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
+                    continue
+            else:
+                # Image manifest
+                license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+                license_file.write("VERSION: %s\n" % pkg_dic[pkg]["PV"])
+                license_file.write("LICENSE: %s\n" % pkg_dic[pkg]["LICENSE"])
+                license_file.write("FILES: %s\n\n" % pkg_dic[pkg]["FILES"])
 
             for lic in pkg_dic[pkg]["LICENSES"]:
                 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
@@ -141,6 +153,34 @@ def write_license_files(d, license_manifest, pkg_dic):
                         os.link(pkg_license, pkg_rootfs_license)
 
 
+def license_deployed_manifest(d):
+    """
+    Write the license manifest for the deployed recipes.
+    The deployed recipes usually includes the bootloader
+    and extra files to boot the target.
+    """
+
+    dep_dic = {}
+    man_dic = {}
+    lic_dir = d.getVar("LICENSE_DIRECTORY", True)
+
+    dep_dic = get_deployed_dependencies(d)
+    for dep in dep_dic.keys():
+        man_dic[dep] = {}
+        # It is necessary to mark this will be used for image manifest
+        man_dic[dep]["IMAGE_MANIFEST"] = True
+        man_dic[dep]["PN"] = dep
+        man_dic[dep]["FILES"] = \
+            " ".join(get_deployed_files(dep_dic[dep]))
+        with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f:
+            for line in f.readlines():
+                key,val = line.split(": ", 1)
+                man_dic[dep][key] = val[:-1]
+
+    image_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+            d.getVar('IMAGE_NAME', True), 'image_license.manifest')
+    write_license_files(d, image_license_manifest, man_dic)
+
 def get_deployed_dependencies(d):
     """
     Get all the deployed dependencies of an image
@@ -591,7 +631,7 @@ SSTATETASKS += "do_populate_lic"
 do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}"
 do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/"
 
-ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; "
+ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; write_deploy_manifest; license_create_manifest; "
 
 do_populate_lic_setscene[dirs] = "${LICSSTATEDIR}/${PN}"
 do_populate_lic_setscene[cleandirs] = "${LICSSTATEDIR}"
-- 
1.8.4.5