[PATCH 1/6] glibc: Separate locale files to their own sstate task

[PATCH 1/6] glibc: Separate locale files to their own sstate task

[Richard Purdie]

Putting the locale and script files into the sysroot for use by their
specific recipes used to be a simple way to share the files. With RSS,
we don't want to copy these into many different recipes so put these
files in their own sstate task.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/recipes-core/glibc/glibc-collateral.inc |  2 +-
 meta/recipes-core/glibc/glibc-initial.inc    |  7 +------
 meta/recipes-core/glibc/glibc-locale.inc     |  2 +-
 meta/recipes-core/glibc/glibc-package.inc    | 30 +++++++++++++++++-----------
 meta/recipes-core/glibc/glibc-scripts.inc    |  2 +-
 5 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/meta/recipes-core/glibc/glibc-collateral.inc b/meta/recipes-core/glibc/glibc-collateral.inc
index 6b32e4a..37f27ca 100644
--- a/meta/recipes-core/glibc/glibc-collateral.inc
+++ b/meta/recipes-core/glibc/glibc-collateral.inc
@@ -15,7 +15,7 @@ deltask do_patch
 do_configure[noexec] = "1"
 do_compile[noexec] = "1"
 
-do_install[depends] += "virtual/${MLPREFIX}libc:do_populate_sysroot"
+do_install[depends] += "virtual/${MLPREFIX}libc:do_stash_locale"
 
 COMPATIBLE_HOST_libc-musl_class-target = "null"
 COMPATIBLE_HOST_libc-uclibc_class-target = "null"
diff --git a/meta/recipes-core/glibc/glibc-initial.inc b/meta/recipes-core/glibc/glibc-initial.inc
index 7380bdb..f94603c 100644
--- a/meta/recipes-core/glibc/glibc-initial.inc
+++ b/meta/recipes-core/glibc/glibc-initial.inc
@@ -43,7 +43,7 @@ do_install () {
 	fi
 }
 
-do_install_locale() {
+do_stash_locale() {
 	:
 }
 
@@ -51,9 +51,4 @@ do_siteconfig () {
 	:
 }
 
-# We don't install any scripts so there is nothing to evacuate
-do_evacuate_scripts () {
-	:
-}
-
 inherit nopackages
diff --git a/meta/recipes-core/glibc/glibc-locale.inc b/meta/recipes-core/glibc/glibc-locale.inc
index 50b191c..9c99741 100644
--- a/meta/recipes-core/glibc/glibc-locale.inc
+++ b/meta/recipes-core/glibc/glibc-locale.inc
@@ -66,7 +66,7 @@ DESCRIPTION_localedef = "glibc: compile locale definition files"
 FILES_${MLPREFIX}glibc-gconv = "${libdir}/gconv/*"
 FILES_localedef = "${bindir}/localedef"
 
-LOCALETREESRC = "${STAGING_INCDIR}/glibc-locale-internal-${MULTIMACH_TARGET_SYS}"
+LOCALETREESRC = "${STAGING_DIR}-components/${PACKAGE_ARCH}/glibc-stash-locale"
 
 do_install () {
 	mkdir -p ${D}${bindir} ${D}${datadir} ${D}${libdir}
diff --git a/meta/recipes-core/glibc/glibc-package.inc b/meta/recipes-core/glibc/glibc-package.inc
index 481a00e..557b958 100644
--- a/meta/recipes-core/glibc/glibc-package.inc
+++ b/meta/recipes-core/glibc/glibc-package.inc
@@ -145,8 +145,11 @@ do_install_append_aarch64 () {
 	fi
 }
 
-do_install_locale () {
-	dest=${D}/${includedir}/glibc-locale-internal-${MULTIMACH_TARGET_SYS}
+LOCALESTASH = "${WORKDIR}/stashed-locale"
+bashscripts = "mtrace sotruss xtrace"
+
+do_stash_locale () {
+	dest=${LOCALESTASH}
 	install -d ${dest}${base_libdir} ${dest}${bindir} ${dest}${libdir} ${dest}${datadir}
 	if [ "${base_libdir}" != "${libdir}" ]; then
 		cp -fpPR ${D}${base_libdir}/* ${dest}${base_libdir}
@@ -166,14 +169,8 @@ do_install_locale () {
 	cp -fpPR ${D}${datadir}/* ${dest}${datadir}
 	rm -rf ${D}${datadir}/locale/
 	cp -fpPR ${WORKDIR}/SUPPORTED ${dest}
-}
-
-addtask do_install_locale after do_install before do_populate_sysroot do_package
 
-bashscripts = "mtrace sotruss xtrace"
-
-do_evacuate_scripts () {
-	target=${D}${includedir}/glibc-scripts-internal-${MULTIMACH_TARGET_SYS}
+	target=${dest}/scripts
 	mkdir -p $target
 	for i in ${bashscripts}; do
 		if [ -f ${D}${bindir}/$i ]; then
@@ -182,13 +179,22 @@ do_evacuate_scripts () {
 	done
 }
 
-addtask evacuate_scripts after do_install before do_populate_sysroot do_package
+addtask do_stash_locale after do_install before do_populate_sysroot do_package
+do_stash_locale[dirs] = "${B}"
+do_stash_locale[cleandirs] = "${LOCALESTASH}"
+SSTATETASKS += "do_stash_locale"
+do_stash_locale[sstate-inputdirs] = "${LOCALESTASH}"
+do_stash_locale[sstate-outputdirs] = "${STAGING_DIR}-components/${PACKAGE_ARCH}/glibc-stash-locale"
+do_stash_locale[sstate-fixmedir] = "${STAGING_DIR}-components/${PACKAGE_ARCH}/glibc-stash-locale"
+
+python do_stash_locale_setscene () {
+    sstate_setscene(d)
+}
+addtask do_stash_locale_setscene
 
 PACKAGE_PREPROCESS_FUNCS += "glibc_package_preprocess"
 
 glibc_package_preprocess () {
-	rm -rf ${PKGD}/${includedir}/glibc-locale-internal-${MULTIMACH_TARGET_SYS}
-	rm -rf ${PKGD}/${includedir}/glibc-scripts-internal-${MULTIMACH_TARGET_SYS}
 	for i in ${bashscripts}; do
 	    rm -f ${PKGD}${bindir}/$i
 	done
diff --git a/meta/recipes-core/glibc/glibc-scripts.inc b/meta/recipes-core/glibc/glibc-scripts.inc
index 66fdee4..42616f0 100644
--- a/meta/recipes-core/glibc/glibc-scripts.inc
+++ b/meta/recipes-core/glibc/glibc-scripts.inc
@@ -4,7 +4,7 @@ SUMMARY = "utility scripts provided by glibc"
 DESCRIPTION = "utility scripts provided by glibc"
 RDEPENDS_${PN} = "bash glibc-mtrace"
 
-SRC = "${STAGING_INCDIR}/glibc-scripts-internal-${MULTIMACH_TARGET_SYS}"
+SRC = "${STAGING_DIR}-components/${PACKAGE_ARCH}/glibc-stash-locale/scripts"
 
 bashscripts = "sotruss xtrace"
 
-- 
2.7.4

[PATCH 2/6] staging: Don't put ptest files into the sysroot

[Richard Purdie]

This wastes space/time as the ptest files aren't going to be used in the
sysroot so add them to the blacklist.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/staging.bbclass | 1 +
 1 file changed, 1 insertion(+)

diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index 93d31eb..b92bdaf 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -31,6 +31,7 @@ SYSROOT_DIRS_BLACKLIST = " \
     ${datadir}/applications \
     ${datadir}/fonts \
     ${datadir}/pixmaps \
+    ${libdir}/${PN}/ptest \
 "
 
 sysroot_stage_dir() {
-- 
2.7.4

[PATCH 3/6] perl-native: Remove .packinfo file

[Richard Purdie]

This is large and needs fixups to relocate it in each case. We can drop it, save
the work and the ~150MB disk space its various copies take up. Its not needed
for anything that I can see.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/recipes-devtools/perl/perl-native_5.24.0.bb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/meta/recipes-devtools/perl/perl-native_5.24.0.bb b/meta/recipes-devtools/perl/perl-native_5.24.0.bb
index af2ad7b..4216257 100644
--- a/meta/recipes-devtools/perl/perl-native_5.24.0.bb
+++ b/meta/recipes-devtools/perl/perl-native_5.24.0.bb
@@ -106,6 +106,10 @@ do_install () {
 	for f in `grep -Il '#! *${bindir}/perl' ${D}/${bindir}/*`; do
 		sed -i -e 's|${bindir}/perl|/usr/bin/env nativeperl|' $f
 	done
+
+	# The packlist is large with hardcoded paths meaning it needs relocating
+	# so just remove it.
+	rm ${D}${libdir}/perl/${PV}/.packlist
 }
 
 SYSROOT_PREPROCESS_FUNCS += "perl_sysroot_create_wrapper"
-- 
2.7.4

[PATCH 4/6] staging: Add shared manifest support

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/staging.bbclass | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index b92bdaf..bd798ba 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -479,6 +479,7 @@ python extend_recipe_sysroot() {
     bb.note("\n".join(msgbuf))
 
     stagingdir = d.getVar("STAGING_DIR")
+    sharedmanifests = stagingdir + "-components/manifests"
     recipesysroot = d.getVar("RECIPE_SYSROOT")
     recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE")
     current_variant = d.getVar("BBEXTENDVARIANT")
@@ -493,6 +494,7 @@ python extend_recipe_sysroot() {
 
     depdir = recipesysrootnative + "/installeddeps"
     bb.utils.mkdirhier(depdir)
+    bb.utils.mkdirhier(sharedmanifests)
 
     lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock")
 
@@ -588,6 +590,27 @@ python extend_recipe_sysroot() {
                         dest = staging_copyfile(l, destsysroot, fixme[''], postinsts, stagingdir, seendirs)
                     if dest:
                         m.write(dest.replace(workdir + "/", "") + "\n")
+            # Having multiple identical manifests in each sysroot eats diskspace so
+            # create a shared pool of them.
+            sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest)
+            if not os.path.exists(sharedm):
+                smlock = bb.utils.lockfile(sharedm + ".lock")
+                # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other
+                # but python can lose file handles so we need to do this under a lock.
+                try:
+                    if not os.path.exists(sharedm):
+                        os.rename(taskmanifest, sharedm)
+                except OSError:
+                   pass
+                bb.utils.unlockfile(smlock)
+            if os.path.exists(sharedm):
+                # If we're crossing mount points we'll not reach here.
+                if os.path.exists(taskmanifest):
+                    if os.path.getsize(sharedm) != os.path.getsize(taskmanifest):
+                        # Order of entries can differ, overall size shouldn't
+                        raise Exception("Manifests %s and %s differ in size and shouldn't?" % (sharedm, taskmanifest))
+                    os.unlink(taskmanifest)
+                os.link(sharedm, taskmanifest)
 
     for f in fixme:
         if f == '':
-- 
2.7.4

[PATCH 5/6] staging: Improve file creation resiliance

[Richard Purdie]

If you abort a build mid execution of the staging extend_sysroot function
there are currently races and cleanup of that function may fail.

This change splits the code into separate phases so that the manifests are
manipulated before files are installed, meaning we should be able to reverse
actions if builds fail, crash or are interrupted.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/staging.bbclass | 88 ++++++++++++++++++++++++--------------------
 1 file changed, 49 insertions(+), 39 deletions(-)

diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index bd798ba..0cb46bf 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -245,17 +245,9 @@ python do_populate_sysroot_setscene () {
 }
 addtask do_populate_sysroot_setscene
 
-def staging_copyfile(c, target, fixme, postinsts, stagingdir, seendirs):
+def staging_copyfile(c, target, dest, postinsts, seendirs):
     import errno
 
-    if c.endswith("/fixmepath"):
-        fixme.append(c)
-        return None
-    if c.endswith("/fixmepath.cmd"):
-        return None
-    #bb.warn(c)
-    dest = c.replace(stagingdir, "")
-    dest = target + "/" + "/".join(dest.split("/")[3:])
     destdir = os.path.dirname(dest)
     if destdir not in seendirs:
         bb.utils.mkdirhier(destdir)
@@ -282,9 +274,7 @@ def staging_copyfile(c, target, fixme, postinsts, stagingdir, seendirs):
                 raise
     return dest
 
-def staging_copydir(c, target, stagingdir, seendirs):
-    dest = c.replace(stagingdir, "")
-    dest = target + "/" + "/".join(dest.split("/")[3:])
+def staging_copydir(c, target, dest, seendirs):
     if dest not in seendirs:
         bb.utils.mkdirhier(dest)
         seendirs.add(dest)
@@ -338,11 +328,18 @@ def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
             with open(manifest, "r") as f:
                 for l in f:
                     l = l.strip()
+                    if l.endswith("/fixmepath"):
+                        fixme.append(l)
+                        continue
+                    if l.endswith("/fixmepath.cmd"):
+                        continue
+                    dest = l.replace(stagingdir, "")
+                    dest = targetdir + "/" + "/".join(dest.split("/")[3:])
                     if l.endswith("/"):
-                        staging_copydir(l, targetdir, stagingdir, seendirs)
+                        staging_copydir(l, targetdir, dest, seendirs)
                         continue
                     try:
-                        staging_copyfile(l, targetdir, fixme, postinsts, stagingdir, seendirs)
+                        staging_copyfile(l, targetdir, dest, postinsts, seendirs)
                     except FileExistsError:
                         continue
 
@@ -365,6 +362,8 @@ def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
 python extend_recipe_sysroot() {
     import copy
     import subprocess
+    import errno
+    import collections
 
     taskdepdata = d.getVar("BB_TASKDEPDATA", False)
     mytaskname = d.getVar("BB_RUNTASK")
@@ -574,43 +573,54 @@ python extend_recipe_sysroot() {
         if not os.path.exists(manifest):
             bb.warn("Manifest %s not found?" % manifest)
         else:
-            with open(manifest, "r") as f, open(taskmanifest, 'w') as m:
+            newmanifest = collections.OrderedDict()
+            if native:
+                fm = fixme['native']
+                targetdir = recipesysrootnative
+            else:
+                fm = fixme['']
+                targetdir = destsysroot
+            with open(manifest, "r") as f:
                 manifests[dep] = manifest
                 for l in f:
                     l = l.strip()
-                    if l.endswith("/"):
-                        if native:
-                            dest = staging_copydir(l, recipesysrootnative, stagingdir, seendirs)
-                        else:
-                            dest = staging_copydir(l, destsysroot, stagingdir, seendirs)
+                    if l.endswith("/fixmepath"):
+                        fm.append(l)
+                        continue
+                    if l.endswith("/fixmepath.cmd"):
                         continue
-                    if native:
-                        dest = staging_copyfile(l, recipesysrootnative, fixme['native'], postinsts, stagingdir, seendirs)
-                    else:
-                        dest = staging_copyfile(l, destsysroot, fixme[''], postinsts, stagingdir, seendirs)
-                    if dest:
-                        m.write(dest.replace(workdir + "/", "") + "\n")
+                    dest = l.replace(stagingdir, "")
+                    dest = targetdir + "/" + "/".join(dest.split("/")[3:])
+                    newmanifest[l] = dest
             # Having multiple identical manifests in each sysroot eats diskspace so
-            # create a shared pool of them.
+            # create a shared pool of them and hardlink if we can.
+            # We create the manifest in advance so that if something fails during installation,
+            # or the build is interrupted, subsequent exeuction can cleanup.
             sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest)
             if not os.path.exists(sharedm):
                 smlock = bb.utils.lockfile(sharedm + ".lock")
                 # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other
                 # but python can lose file handles so we need to do this under a lock.
-                try:
-                    if not os.path.exists(sharedm):
-                        os.rename(taskmanifest, sharedm)
-                except OSError:
-                   pass
+                if not os.path.exists(sharedm):
+                    with open(sharedm, 'w') as m:
+                       for l in newmanifest:
+                           dest = newmanifest[l]
+                           m.write(dest.replace(workdir + "/", "") + "\n")
                 bb.utils.unlockfile(smlock)
-            if os.path.exists(sharedm):
-                # If we're crossing mount points we'll not reach here.
-                if os.path.exists(taskmanifest):
-                    if os.path.getsize(sharedm) != os.path.getsize(taskmanifest):
-                        # Order of entries can differ, overall size shouldn't
-                        raise Exception("Manifests %s and %s differ in size and shouldn't?" % (sharedm, taskmanifest))
-                    os.unlink(taskmanifest)
+            try:
                 os.link(sharedm, taskmanifest)
+            except OSError as err:
+                if err.errno == errno.EXDEV:
+                    bb.utils.copyfile(sharedm, taskmanifest)
+                else:
+                    raise
+            # Finally actually install the files
+            for l in newmanifest:
+                    dest = newmanifest[l]
+                    if l.endswith("/"):
+                        staging_copydir(l, targetdir, dest, seendirs)
+                        continue
+                    staging_copyfile(l, targetdir, dest, postinsts, seendirs)
 
     for f in fixme:
         if f == '':
-- 
2.7.4

[PATCH 6/6] staging: Add missing parameter to debug message

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/staging.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index 0cb46bf..0752009 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -464,7 +464,7 @@ python extend_recipe_sysroot() {
                 taskdeps[datadep] = setscenedeps[datadep][:2]
                 retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf)
                 if retval:
-                    msgbuf.append("Skipping setscene dependency %s for installation into the sysroot")
+                    msgbuf.append("Skipping setscene dependency %s for installation into the sysroot" % datadep)
                     continue
                 done.append(datadep)
                 new.append(datadep)
-- 
2.7.4

Re: [PATCH 3/6] perl-native: Remove .packinfo file

[Martin Jansa]

[-- Attachment #1: Type: text/plain, Size: 1474 bytes --]

On Mon, Feb 06, 2017 at 03:52:11PM +0000, Richard Purdie wrote:
> This is large and needs fixups to relocate it in each case. We can drop it, save
> the work and the ~150MB disk space its various copies take up. Its not needed
> for anything that I can see.

-1

The subject says .packinfo, but the change is for .packlist.

> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/recipes-devtools/perl/perl-native_5.24.0.bb | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/meta/recipes-devtools/perl/perl-native_5.24.0.bb b/meta/recipes-devtools/perl/perl-native_5.24.0.bb
> index af2ad7b..4216257 100644
> --- a/meta/recipes-devtools/perl/perl-native_5.24.0.bb
> +++ b/meta/recipes-devtools/perl/perl-native_5.24.0.bb
> @@ -106,6 +106,10 @@ do_install () {
>  	for f in `grep -Il '#! *${bindir}/perl' ${D}/${bindir}/*`; do
>  		sed -i -e 's|${bindir}/perl|/usr/bin/env nativeperl|' $f
>  	done
> +
> +	# The packlist is large with hardcoded paths meaning it needs relocating
> +	# so just remove it.
> +	rm ${D}${libdir}/perl/${PV}/.packlist
>  }
>  
>  SYSROOT_PREPROCESS_FUNCS += "perl_sysroot_create_wrapper"
> -- 
> 2.7.4
> 
> -- 
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core

-- 
Martin 'JaMa' Jansa     jabber: Martin.Jansa@gmail.com

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 201 bytes --]

Re: [PATCH 3/6] perl-native: Remove .packinfo file

[Khem Raj]

On Mon, Feb 6, 2017 at 7:52 AM, Richard Purdie
<richard.purdie@linuxfoundation.org> wrote:
> This is large and needs fixups to relocate it in each case. We can drop it, save
> the work and the ~150MB disk space its various copies take up. Its not needed
> for anything that I can see.

I agree, I think we should even consider removing them from target and
nativesdk perl.

>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/recipes-devtools/perl/perl-native_5.24.0.bb | 4 ++++
>  1 file changed, 4 insertions(+)
>
> diff --git a/meta/recipes-devtools/perl/perl-native_5.24.0.bb b/meta/recipes-devtools/perl/perl-native_5.24.0.bb
> index af2ad7b..4216257 100644
> --- a/meta/recipes-devtools/perl/perl-native_5.24.0.bb
> +++ b/meta/recipes-devtools/perl/perl-native_5.24.0.bb
> @@ -106,6 +106,10 @@ do_install () {
>         for f in `grep -Il '#! *${bindir}/perl' ${D}/${bindir}/*`; do
>                 sed -i -e 's|${bindir}/perl|/usr/bin/env nativeperl|' $f
>         done
> +
> +       # The packlist is large with hardcoded paths meaning it needs relocating
> +       # so just remove it.
> +       rm ${D}${libdir}/perl/${PV}/.packlist
>  }
>
>  SYSROOT_PREPROCESS_FUNCS += "perl_sysroot_create_wrapper"
> --
> 2.7.4
>
> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core

Re: [PATCH 3/6] perl-native: Remove .packinfo file

[Richard Purdie]

On Mon, 2017-02-06 at 20:26 +0100, Martin Jansa wrote:
> On Mon, Feb 06, 2017 at 03:52:11PM +0000, Richard Purdie wrote:
> > 
> > This is large and needs fixups to relocate it in each case. We can
> > drop it, save
> > the work and the ~150MB disk space its various copies take up. Its
> > not needed
> > for anything that I can see.
> -1
> 
> The subject says .packinfo, but the change is for .packlist.

Sorry, I've fixed the commit message on -next.

Cheers,

Richard