[PATCH 1/7] image/kernelsrc/packagegroups/recipes: Remove uneeded noexec tasks

[PATCH 1/7] image/kernelsrc/packagegroups/recipes: Remove uneeded noexec tasks

[Richard Purdie]

We used to have issues removing tasks like do_fetch due to implications
for targets like world and universe. These have now been resolved.

Removing uneeded tasks has advantages compared to noexec since it means
that accidentally left in dependencies are no longer needed/processed
(e.g. do_patch depends on quilt-native).

This cleans up a number of cases which local analysis highlighted as
being unneeded leading to slightly cleaner task graphs.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/image.bbclass                   | 16 ++++++++--------
 meta/classes/kernelsrc.bbclass               |  6 +++---
 meta/classes/packagegroup.bbclass            | 12 ++++++------
 meta/recipes-core/glibc/glibc-collateral.inc |  6 +++---
 meta/recipes-core/meta/meta-environment.bb   | 10 +++++-----
 meta/recipes-core/meta/meta-world-pkgdata.bb | 12 ++++++------
 meta/recipes-core/meta/package-index.bb      | 24 ++++++++++++------------
 7 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 76146c9..b4e5432 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -615,15 +615,15 @@ MULTILIBRE_ALLOW_REP =. "${base_bindir}|${base_sbindir}|${bindir}|${sbindir}|${l
 MULTILIB_CHECK_FILE = "${WORKDIR}/multilib_check.py"
 MULTILIB_TEMP_ROOTFS = "${WORKDIR}/multilib"
 
-do_fetch[noexec] = "1"
-do_unpack[noexec] = "1"
-do_patch[noexec] = "1"
-do_configure[noexec] = "1"
-do_compile[noexec] = "1"
-do_install[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
 deltask do_populate_sysroot
-do_package[noexec] = "1"
-do_package_qa[noexec] = "1"
+deltask do_package
+deltask do_package_qa
 do_packagedata[noexec] = "1"
 do_package_write_ipk[noexec] = "1"
 do_package_write_deb[noexec] = "1"
diff --git a/meta/classes/kernelsrc.bbclass b/meta/classes/kernelsrc.bbclass
index 9efd46a..675d40e 100644
--- a/meta/classes/kernelsrc.bbclass
+++ b/meta/classes/kernelsrc.bbclass
@@ -1,7 +1,7 @@
 S = "${STAGING_KERNEL_DIR}"
-do_fetch[noexec] = "1"
-do_unpack[depends] += "virtual/kernel:do_patch"
-do_unpack[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+do_patch[depends] += "virtual/kernel:do_patch"
 do_patch[noexec] = "1"
 do_package[depends] += "virtual/kernel:do_populate_sysroot"
 KERNEL_VERSION = "${@get_kernelversion_file("${STAGING_KERNEL_BUILDDIR}")}"
diff --git a/meta/classes/packagegroup.bbclass b/meta/classes/packagegroup.bbclass
index ad6c8b0..4af4a01 100644
--- a/meta/classes/packagegroup.bbclass
+++ b/meta/classes/packagegroup.bbclass
@@ -40,12 +40,12 @@ python () {
 DEPCHAIN_DBGDEFAULTDEPS = "1"
 
 # We only need the packaging tasks - disable the rest
-do_fetch[noexec] = "1"
-do_unpack[noexec] = "1"
-do_patch[noexec] = "1"
-do_configure[noexec] = "1"
-do_compile[noexec] = "1"
-do_install[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
 deltask do_populate_sysroot
 
 python () {
diff --git a/meta/recipes-core/glibc/glibc-collateral.inc b/meta/recipes-core/glibc/glibc-collateral.inc
index 60655eb..6b32e4a 100644
--- a/meta/recipes-core/glibc/glibc-collateral.inc
+++ b/meta/recipes-core/glibc/glibc-collateral.inc
@@ -9,9 +9,9 @@ HOMEPAGE = "http://www.gnu.org/software/libc/index.html"
 # http://lists.openembedded.org/pipermail/openembedded-core/2015-January/100679.html
 ARM_INSTRUCTION_SET = "arm"
 
-do_fetch[noexec] = "1"
-do_unpack[noexec] = "1"
-do_patch[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
 do_configure[noexec] = "1"
 do_compile[noexec] = "1"
 
diff --git a/meta/recipes-core/meta/meta-environment.bb b/meta/recipes-core/meta/meta-environment.bb
index a2ed229..47086fb 100644
--- a/meta/recipes-core/meta/meta-environment.bb
+++ b/meta/recipes-core/meta/meta-environment.bb
@@ -71,9 +71,9 @@ FILES_${PN}= " \
     ${SDKPATH}/* \
     "
 
-do_fetch[noexec] = "1"
-do_unpack[noexec] = "1"
-do_patch[noexec] = "1"
-do_configure[noexec] = "1"
-do_compile[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
 do_populate_sysroot[noexec] = "1"
diff --git a/meta/recipes-core/meta/meta-world-pkgdata.bb b/meta/recipes-core/meta/meta-world-pkgdata.bb
index 617f96a..02abd8a 100644
--- a/meta/recipes-core/meta/meta-world-pkgdata.bb
+++ b/meta/recipes-core/meta/meta-world-pkgdata.bb
@@ -25,12 +25,12 @@ python do_collect_packagedata() {
     oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
 }
 
-do_fetch[noexec] = "1"
-do_unpack[noexec] = "1"
-do_patch[noexec] = "1"
-do_configure[noexec] = "1"
-do_compile[noexec] = "1"
-do_install[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
 
 do_prepare_recipe_sysroot[deptask] = ""
 
diff --git a/meta/recipes-core/meta/package-index.bb b/meta/recipes-core/meta/package-index.bb
index 3c46b49..fe022ff 100644
--- a/meta/recipes-core/meta/package-index.bb
+++ b/meta/recipes-core/meta/package-index.bb
@@ -4,18 +4,18 @@ LICENSE = "MIT"
 INHIBIT_DEFAULT_DEPS = "1"
 PACKAGES = ""
 
-do_fetch[noexec] = "1"
-do_unpack[noexec] = "1"
-do_patch[noexec] = "1"
-do_configure[noexec] = "1"
-do_compile[noexec] = "1"
-do_install[noexec] = "1"
-do_package[noexec] = "1"
-do_packagedata[noexec] = "1"
-do_package_write_ipk[noexec] = "1"
-do_package_write_rpm[noexec] = "1"
-do_package_write_deb[noexec] = "1"
-do_populate_sysroot[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
+deltask do_package
+deltask do_packagedata
+deltask do_package_write_ipk
+deltask do_package_write_rpm
+deltask do_package_write_deb
+deltask do_populate_sysroot
 
 do_package_index[nostamp] = "1"
 do_package_index[depends] += "${PACKAGEINDEXDEPS}"
-- 
2.7.4

[PATCH 2/7] allarch: Drop STAGING_DIR_HOST expansion

[Richard Purdie]

Now that STAGINGDIR_HOST doesn't contain MACHINE, we no longer need to expand
the value. Pre-expansion can mean components like PV can be expanded too early
and cause problems for certain use cases.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/allarch.bbclass | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/meta/classes/allarch.bbclass b/meta/classes/allarch.bbclass
index a774841..a7ce024 100644
--- a/meta/classes/allarch.bbclass
+++ b/meta/classes/allarch.bbclass
@@ -2,10 +2,6 @@
 # This class is used for architecture independent recipes/data files (usually scripts)
 #
 
-# Expand STAGING_DIR_HOST since for cross-canadian/native/nativesdk, this will
-# point elsewhere after these changes.
-STAGING_DIR_HOST := "${STAGING_DIR_HOST}"
-
 PACKAGE_ARCH = "all"
 
 python () {
-- 
2.7.4

[PATCH 3/7] gzip/pigz/expat/image: Simplify gzip-native/pigz-native

[Richard Purdie]

With recipe specific sysroots, the gzip-replacement-native dance/class
is obsolete, simplify the code accordingly.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/gzipnative.bbclass          | 5 -----
 meta/classes/image.bbclass               | 2 --
 meta/classes/image_types.bbclass         | 2 +-
 meta/recipes-core/expat/expat.inc        | 6 +++---
 meta/recipes-extended/gzip/gzip_1.8.bb   | 1 -
 meta/recipes-extended/pigz/pigz_2.3.4.bb | 2 --
 6 files changed, 4 insertions(+), 14 deletions(-)
 delete mode 100644 meta/classes/gzipnative.bbclass

diff --git a/meta/classes/gzipnative.bbclass b/meta/classes/gzipnative.bbclass
deleted file mode 100644
index 326cbbb..0000000
--- a/meta/classes/gzipnative.bbclass
+++ /dev/null
@@ -1,5 +0,0 @@
-EXTRANATIVEPATH += "pigz-native gzip-native"
-DEPENDS += "gzip-native"
-
-# tar may get run by do_unpack or do_populate_lic which could call gzip
-do_unpack[depends] += "gzip-native:do_populate_sysroot"
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index b4e5432..0ffe99f 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -9,8 +9,6 @@ TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}"
 TOOLCHAIN_TARGET_TASK_ATTEMPTONLY += "${PACKAGE_INSTALL_ATTEMPTONLY}"
 POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_sysroot_relativelinks; "
 
-inherit gzipnative
-
 LICENSE = "MIT"
 PACKAGES = ""
 DEPENDS += "${MLPREFIX}qemuwrapper-cross depmodwrapper-cross"
diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index ddd52f1..005d1e2 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -359,7 +359,7 @@ CONVERSION_CMD_sha384sum = "sha384sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}
 CONVERSION_CMD_sha512sum = "sha512sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha512sum"
 CONVERSION_CMD_bmap = "bmaptool create ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.bmap"
 CONVERSION_DEPENDS_lzma = "xz-native"
-CONVERSION_DEPENDS_gz = ""
+CONVERSION_DEPENDS_gz = "pigz-native"
 CONVERSION_DEPENDS_bz2 = "pbzip2-native"
 CONVERSION_DEPENDS_xz = "xz-native"
 CONVERSION_DEPENDS_lz4 = "lz4-native"
diff --git a/meta/recipes-core/expat/expat.inc b/meta/recipes-core/expat/expat.inc
index fe9d7e7..9fa0ca2 100644
--- a/meta/recipes-core/expat/expat.inc
+++ b/meta/recipes-core/expat/expat.inc
@@ -7,12 +7,12 @@ LICENSE = "MIT"
 SRC_URI = "${SOURCEFORGE_MIRROR}/expat/expat-${PV}.tar.bz2 \
            file://autotools.patch \
 	  "
-
-inherit autotools lib_package gzipnative
+inherit autotools lib_package
 
 # This package uses an archive format known to have issue with some
 # versions of gzip
-do_unpack[depends] += "gzip-native:do_populate_sysroot"
+DEPENDS += "pigz-native"
+do_unpack[depends] += "pigz-native:do_populate_sysroot"
 
 do_configure_prepend () {
 	rm -f ${S}/conftools/libtool.m4
diff --git a/meta/recipes-extended/gzip/gzip_1.8.bb b/meta/recipes-extended/gzip/gzip_1.8.bb
index a19b965..11be846 100644
--- a/meta/recipes-extended/gzip/gzip_1.8.bb
+++ b/meta/recipes-extended/gzip/gzip_1.8.bb
@@ -9,7 +9,6 @@ LIC_FILES_CHKSUM = "file://COPYING;md5=d32239bcb673463ab874e80d47fae504 \
                     file://gzip.h;beginline=8;endline=20;md5=6e47caaa630e0c8bf9f1bc8d94a8ed0e"
 
 PROVIDES_append_class-native = " gzip-replacement-native"
-NATIVE_PACKAGE_PATH_SUFFIX = "/${PN}"
 
 BBCLASSEXTEND = "native"
 
diff --git a/meta/recipes-extended/pigz/pigz_2.3.4.bb b/meta/recipes-extended/pigz/pigz_2.3.4.bb
index 87b61bf..53c56c5 100644
--- a/meta/recipes-extended/pigz/pigz_2.3.4.bb
+++ b/meta/recipes-extended/pigz/pigz_2.3.4.bb
@@ -7,7 +7,5 @@ SRC_URI = "https://github.com/madler/pigz/archive/v${PV}.tar.gz"
 SRC_URI[md5sum] = "c109057050b15edf3eb9bb4d0805235e"
 SRC_URI[sha256sum] = "763f2fdb203aa0b7b640e63385e38e5dd4e5aaa041bc8e42aa96f2ef156b06e8"
 
-NATIVE_PACKAGE_PATH_SUFFIX = "/${PN}"
-
 BBCLASSEXTEND = "native nativesdk"
 
-- 
2.7.4

[PATCH 4/7] sstate: Allow sstate_clean_manifest to take a prefix

[Richard Purdie]

Manifest files containing the same duplicated prefix are wasteful on space
and ultimately this costs build time. Add support for manifest files with
common prefixes removed and use the prefix if the path isn't absolute.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/sstate.bbclass | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 0fdeb9d..31789da 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -426,7 +426,7 @@ def sstate_clean_cachefiles(d):
         ss = sstate_state_fromvars(ld, task)
         sstate_clean_cachefile(ss, ld)
 
-def sstate_clean_manifest(manifest, d):
+def sstate_clean_manifest(manifest, d, prefix=None):
     import oe.path
 
     mfile = open(manifest)
@@ -435,6 +435,8 @@ def sstate_clean_manifest(manifest, d):
 
     for entry in entries:
         entry = entry.strip()
+        if prefix and not entry.startswith("/"):
+            entry = prefix + "/" + entry
         bb.debug(2, "Removing manifest: %s" % entry)
         # We can race against another package populating directories as we're removing them
         # so we ignore errors here.
-- 
2.7.4

[PATCH 5/7] staging: Drop common prefix WORKDIR from manifest files

[Richard Purdie]

Manifest files containing the same duplicated prefix are wasteful on space
and ultimately this costs build time. Drop the WORKDIR prefix from the manifest
files since this small change mounts up a lot.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/staging.bbclass | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index 1fe60ac..1b618fb 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -360,6 +360,7 @@ python extend_recipe_sysroot() {
 
     taskdepdata = d.getVar("BB_TASKDEPDATA", False)
     mytaskname = d.getVar("BB_RUNTASK")
+    workdir = d.getVar("WORKDIR")
     #bb.warn(str(taskdepdata))
     pn = d.getVar("PN")
 
@@ -504,7 +505,7 @@ python extend_recipe_sysroot() {
                 continue
             else:
                 bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash))
-                sstate_clean_manifest(depdir + "/" + lnk, d)
+                sstate_clean_manifest(depdir + "/" + lnk, d, workdir)
                 os.unlink(depdir + "/" + c)
         elif os.path.lexists(depdir + "/" + c):
             os.unlink(depdir + "/" + c)
@@ -559,7 +560,7 @@ python extend_recipe_sysroot() {
                     else:
                         dest = staging_copyfile(l, destsysroot, fixme[''], postinsts, stagingdir)
                     if dest:
-                        m.write(dest + "\n")
+                        m.write(dest.replace(workdir + "/", "") + "\n")
 
     for f in fixme:
         if f == '':
-- 
2.7.4

[PATCH 6/7] staging: Allow removal of stale sysroot objects

[Richard Purdie]

The main sysroot components of unreachable build targets will be removed by the core
code. This currently doesn't trigger a removal in the individual workdirs. This
adds in symlinking between the complete stamps and the component sysroot meaning
we can detect when someting was removed and hence remove it from the sysroot.

This fixes cases where DISTRO_FEATURES like systemd are changed amongst other
things and makes builds more robust against configuration changes.

If a dependency is rebuild, that is caught by checksum comparision code elsewhere
in this function as before.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/staging.bbclass | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index 1b618fb..cc7767a 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -490,6 +490,18 @@ python extend_recipe_sysroot() {
     fixme['native'] = []
     postinsts = []
     multilibs = {}
+    manifests = {}
+
+    for f in os.listdir(depdir):
+        if not f.endswith(".complete"):
+            continue
+        f = depdir + "/" + f
+        if os.path.islink(f) and not os.path.exists(f):
+            bb.note("%s no longer exists, removing from sysroot" % f)
+            lnk = os.readlink(f.replace(".complete", ""))
+            sstate_clean_manifest(depdir + "/" + lnk, d, workdir)
+            os.unlink(f)
+            os.unlink(f.replace(".complete", ""))
 
     for dep in configuredeps:
         c = setscenedeps[dep][0]
@@ -547,6 +559,7 @@ python extend_recipe_sysroot() {
             bb.warn("Manifest %s not found?" % manifest)
         else:
             with open(manifest, "r") as f, open(taskmanifest, 'w') as m:
+                manifests[dep] = manifest
                 for l in f:
                     l = l.strip()
                     if l.endswith("/"):
@@ -573,9 +586,9 @@ python extend_recipe_sysroot() {
     for p in postinsts:
         subprocess.check_output(p, shell=True)
 
-    for dep in configuredeps:
+    for dep in manifests:
         c = setscenedeps[dep][0]
-        open(depdir + "/" + c + ".complete", "w").close()
+        os.symlink(manifests[dep], depdir + "/" + c + ".complete")
 
     bb.utils.unlockfile(lock)
 }
-- 
2.7.4

[PATCH 7/7] libxml2: Drop docs in native case

[Richard Purdie]

With rss, moving these around was having an increasing overhead and we don't
need them in the native case so remove them.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/recipes-core/libxml/libxml2_2.9.4.bb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/meta/recipes-core/libxml/libxml2_2.9.4.bb b/meta/recipes-core/libxml/libxml2_2.9.4.bb
index 0287a0a..93b9ba0 100644
--- a/meta/recipes-core/libxml/libxml2_2.9.4.bb
+++ b/meta/recipes-core/libxml/libxml2_2.9.4.bb
@@ -76,4 +76,9 @@ do_install_ptest () {
 	cp -r ${WORKDIR}/xmlconf ${D}${PTEST_PATH}
 }
 
+do_install_append_class-native () {
+	# Docs are not needed in the native case
+	rm ${D}${datadir}/gtk-doc -rf
+}
+
 BBCLASSEXTEND = "native nativesdk"
-- 
2.7.4