[PATCH 01/14] vim: Backport fix for CVE-2021-3770

[PATCH 01/14] vim: Backport fix for CVE-2021-3770

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 ...1e135a16091c93f6f5f7525a5c58fb7ca9f9.patch | 207 ++++++++++++++++++
 meta/recipes-support/vim/vim.inc              |   2 +
 2 files changed, 209 insertions(+)
 create mode 100644 meta/recipes-support/vim/files/b7081e135a16091c93f6f5f7525a5c58fb7ca9f9.patch

diff --git a/meta/recipes-support/vim/files/b7081e135a16091c93f6f5f7525a5c58fb7ca9f9.patch b/meta/recipes-support/vim/files/b7081e135a16091c93f6f5f7525a5c58fb7ca9f9.patch
new file mode 100644
index 00000000000..1cee7595021
--- /dev/null
+++ b/meta/recipes-support/vim/files/b7081e135a16091c93f6f5f7525a5c58fb7ca9f9.patch
@@ -0,0 +1,207 @@
+From b7081e135a16091c93f6f5f7525a5c58fb7ca9f9 Mon Sep 17 00:00:00 2001
+From: Bram Moolenaar <Bram@vim.org>
+Date: Sat, 4 Sep 2021 18:47:28 +0200
+Subject: [PATCH] patch 8.2.3402: invalid memory access when using :retab with
+ large value
+
+Problem:    Invalid memory access when using :retab with large value.
+Solution:   Check the number is positive.
+
+CVE: CVE-2021-3770
+Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
+Upstream-Status: Backport [https://github.com/vim/vim/commit/b7081e135a16091c93f6f5f7525a5c58fb7ca9f9]
+---
+ src/indent.c               | 34 +++++++++++++++++++++-------------
+ src/option.c               | 12 ++++++------
+ src/optionstr.c            |  4 ++--
+ src/testdir/test_retab.vim |  3 +++
+ src/version.c              |  2 ++
+ 5 files changed, 34 insertions(+), 21 deletions(-)
+
+Index: git/src/indent.c
+===================================================================
+--- git.orig/src/indent.c
++++ git/src/indent.c
+@@ -18,18 +18,19 @@
+ /*
+  * Set the integer values corresponding to the string setting of 'vartabstop'.
+  * "array" will be set, caller must free it if needed.
++ * Return FAIL for an error.
+  */
+     int
+ tabstop_set(char_u *var, int **array)
+ {
+-    int valcount = 1;
+-    int t;
+-    char_u *cp;
++    int	    valcount = 1;
++    int	    t;
++    char_u  *cp;
+ 
+     if (var[0] == NUL || (var[0] == '0' && var[1] == NUL))
+     {
+ 	*array = NULL;
+-	return TRUE;
++	return OK;
+     }
+ 
+     for (cp = var; *cp != NUL; ++cp)
+@@ -43,8 +44,8 @@ tabstop_set(char_u *var, int **array)
+ 		if (cp != end)
+ 		    emsg(_(e_positive));
+ 		else
+-		    emsg(_(e_invarg));
+-		return FALSE;
++		    semsg(_(e_invarg2), cp);
++		return FAIL;
+ 	    }
+ 	}
+ 
+@@ -55,26 +56,33 @@ tabstop_set(char_u *var, int **array)
+ 	    ++valcount;
+ 	    continue;
+ 	}
+-	emsg(_(e_invarg));
+-	return FALSE;
++	semsg(_(e_invarg2), var);
++	return FAIL;
+     }
+ 
+     *array = ALLOC_MULT(int, valcount + 1);
+     if (*array == NULL)
+-	return FALSE;
++	return FAIL;
+     (*array)[0] = valcount;
+ 
+     t = 1;
+     for (cp = var; *cp != NUL;)
+     {
+-	(*array)[t++] = atoi((char *)cp);
+-	while (*cp  != NUL && *cp != ',')
++	int n = atoi((char *)cp);
++
++	if (n < 0 || n > 9999)
++	{
++	    semsg(_(e_invarg2), cp);
++	    return FAIL;
++	}
++	(*array)[t++] = n;
++	while (*cp != NUL && *cp != ',')
+ 	    ++cp;
+ 	if (*cp != NUL)
+ 	    ++cp;
+     }
+ 
+-    return TRUE;
++    return OK;
+ }
+ 
+ /*
+@@ -1556,7 +1564,7 @@ ex_retab(exarg_T *eap)
+ 
+ #ifdef FEAT_VARTABS
+     new_ts_str = eap->arg;
+-    if (!tabstop_set(eap->arg, &new_vts_array))
++    if (tabstop_set(eap->arg, &new_vts_array) == FAIL)
+ 	return;
+     while (vim_isdigit(*(eap->arg)) || *(eap->arg) == ',')
+ 	++(eap->arg);
+Index: git/src/option.c
+===================================================================
+--- git.orig/src/option.c
++++ git/src/option.c
+@@ -2292,9 +2292,9 @@ didset_options2(void)
+ #endif
+ #ifdef FEAT_VARTABS
+     vim_free(curbuf->b_p_vsts_array);
+-    tabstop_set(curbuf->b_p_vsts, &curbuf->b_p_vsts_array);
++    (void)tabstop_set(curbuf->b_p_vsts, &curbuf->b_p_vsts_array);
+     vim_free(curbuf->b_p_vts_array);
+-    tabstop_set(curbuf->b_p_vts,  &curbuf->b_p_vts_array);
++    (void)tabstop_set(curbuf->b_p_vts,  &curbuf->b_p_vts_array);
+ #endif
+ }
+ 
+@@ -5756,7 +5756,7 @@ buf_copy_options(buf_T *buf, int flags)
+ 	    buf->b_p_vsts = vim_strsave(p_vsts);
+ 	    COPY_OPT_SCTX(buf, BV_VSTS);
+ 	    if (p_vsts && p_vsts != empty_option)
+-		tabstop_set(p_vsts, &buf->b_p_vsts_array);
++		(void)tabstop_set(p_vsts, &buf->b_p_vsts_array);
+ 	    else
+ 		buf->b_p_vsts_array = 0;
+ 	    buf->b_p_vsts_nopaste = p_vsts_nopaste
+@@ -5914,7 +5914,7 @@ buf_copy_options(buf_T *buf, int flags)
+ 		buf->b_p_isk = save_p_isk;
+ #ifdef FEAT_VARTABS
+ 		if (p_vts && p_vts != empty_option && !buf->b_p_vts_array)
+-		    tabstop_set(p_vts, &buf->b_p_vts_array);
++		    (void)tabstop_set(p_vts, &buf->b_p_vts_array);
+ 		else
+ 		    buf->b_p_vts_array = NULL;
+ #endif
+@@ -5929,7 +5929,7 @@ buf_copy_options(buf_T *buf, int flags)
+ 		buf->b_p_vts = vim_strsave(p_vts);
+ 		COPY_OPT_SCTX(buf, BV_VTS);
+ 		if (p_vts && p_vts != empty_option && !buf->b_p_vts_array)
+-		    tabstop_set(p_vts, &buf->b_p_vts_array);
++		    (void)tabstop_set(p_vts, &buf->b_p_vts_array);
+ 		else
+ 		    buf->b_p_vts_array = NULL;
+ #endif
+@@ -6634,7 +6634,7 @@ paste_option_changed(void)
+ 	    if (buf->b_p_vsts_array)
+ 		vim_free(buf->b_p_vsts_array);
+ 	    if (buf->b_p_vsts && buf->b_p_vsts != empty_option)
+-		tabstop_set(buf->b_p_vsts, &buf->b_p_vsts_array);
++		(void)tabstop_set(buf->b_p_vsts, &buf->b_p_vsts_array);
+ 	    else
+ 		buf->b_p_vsts_array = 0;
+ #endif
+Index: git/src/optionstr.c
+===================================================================
+--- git.orig/src/optionstr.c
++++ git/src/optionstr.c
+@@ -2166,7 +2166,7 @@ did_set_string_option(
+ 	    if (errmsg == NULL)
+ 	    {
+ 		int *oldarray = curbuf->b_p_vsts_array;
+-		if (tabstop_set(*varp, &(curbuf->b_p_vsts_array)))
++		if (tabstop_set(*varp, &(curbuf->b_p_vsts_array)) == OK)
+ 		{
+ 		    if (oldarray)
+ 			vim_free(oldarray);
+@@ -2205,7 +2205,7 @@ did_set_string_option(
+ 	    {
+ 		int *oldarray = curbuf->b_p_vts_array;
+ 
+-		if (tabstop_set(*varp, &(curbuf->b_p_vts_array)))
++		if (tabstop_set(*varp, &(curbuf->b_p_vts_array)) == OK)
+ 		{
+ 		    vim_free(oldarray);
+ #ifdef FEAT_FOLDING
+Index: git/src/testdir/test_retab.vim
+===================================================================
+--- git.orig/src/testdir/test_retab.vim
++++ git/src/testdir/test_retab.vim
+@@ -74,4 +74,7 @@ endfunc
+ func Test_retab_error()
+   call assert_fails('retab -1',  'E487:')
+   call assert_fails('retab! -1', 'E487:')
++  call assert_fails('ret -1000', 'E487:')
++  call assert_fails('ret 10000', 'E475:')
++  call assert_fails('ret 80000000000000000000', 'E475:')
+ endfunc
+Index: git/src/version.c
+===================================================================
+--- git.orig/src/version.c
++++ git/src/version.c
+@@ -743,6 +743,8 @@ static char *(features[]) =
+ static int included_patches[] =
+ {   /* Add new patch number below this line */
+ /**/
++    3402,
++/**/
+     0
+ };
+ 
diff --git a/meta/recipes-support/vim/vim.inc b/meta/recipes-support/vim/vim.inc
index 17322885dc6..7e9225fbcb5 100644
--- a/meta/recipes-support/vim/vim.inc
+++ b/meta/recipes-support/vim/vim.inc
@@ -17,7 +17,9 @@ SRC_URI = "git://github.com/vim/vim.git \
            file://0001-src-Makefile-improve-reproducibility.patch \
            file://no-path-adjust.patch \
            file://racefix.patch \
+           file://b7081e135a16091c93f6f5f7525a5c58fb7ca9f9.patch \
 "
+
 SRCREV = "98056533b96b6b5d8849641de93185dd7bcadc44"
 
 # Do not consider .z in x.y.z, as that is updated with every commit
-- 
2.32.0

[PATCH 02/14] libgcrypt: Upgrade 1.9.3 -> 1.9.4

[Richard Purdie]

Includes a fix for CVE-2021-40528.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 .../libgcrypt/{libgcrypt_1.9.3.bb => libgcrypt_1.9.4.bb}        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename meta/recipes-support/libgcrypt/{libgcrypt_1.9.3.bb => libgcrypt_1.9.4.bb} (96%)

diff --git a/meta/recipes-support/libgcrypt/libgcrypt_1.9.3.bb b/meta/recipes-support/libgcrypt/libgcrypt_1.9.4.bb
similarity index 96%
rename from meta/recipes-support/libgcrypt/libgcrypt_1.9.3.bb
rename to meta/recipes-support/libgcrypt/libgcrypt_1.9.4.bb
index dee936dbc4c..4bc1dd85126 100644
--- a/meta/recipes-support/libgcrypt/libgcrypt_1.9.3.bb
+++ b/meta/recipes-support/libgcrypt/libgcrypt_1.9.4.bb
@@ -27,7 +27,7 @@ SRC_URI = "${GNUPG_MIRROR}/libgcrypt/libgcrypt-${PV}.tar.bz2 \
            file://0004-tests-Makefile.am-fix-undefined-reference-to-pthread.patch \
            file://0001-Makefile.am-add-a-missing-space.patch \
            "
-SRC_URI[sha256sum] = "97ebe4f94e2f7e35b752194ce15a0f3c66324e0ff6af26659bbfb5ff2ec328fd"
+SRC_URI[sha256sum] = "ea849c83a72454e3ed4267697e8ca03390aee972ab421e7df69dfe42b65caaf7"
 
 # Below whitelisted CVEs are disputed and not affecting crypto libraries for any distro.
 CVE_CHECK_WHITELIST += "CVE-2018-12433 CVE-2018-12438"
-- 
2.32.0

[PATCH 03/14] sqlite3: Exclude CVE-2021-36690 from cve checks

[Richard Purdie]

Issue is in an experimental extension we don't have/use. Could also
be windows only.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/recipes-support/sqlite/sqlite3_3.36.0.bb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/meta/recipes-support/sqlite/sqlite3_3.36.0.bb b/meta/recipes-support/sqlite/sqlite3_3.36.0.bb
index f5d75e8e4c2..30c9445be1a 100644
--- a/meta/recipes-support/sqlite/sqlite3_3.36.0.bb
+++ b/meta/recipes-support/sqlite/sqlite3_3.36.0.bb
@@ -10,3 +10,5 @@ SRC_URI[sha256sum] = "bd90c3eb96bee996206b83be7065c9ce19aef38c3f4fb53073ada0d0b6
 CVE_CHECK_WHITELIST += "CVE-2019-19242"
 # This is believed to be iOS specific (https://groups.google.com/g/sqlite-dev/c/U7OjAbZO6LA)
 CVE_CHECK_WHITELIST += "CVE-2015-3717"
+# Issue in an experimental extension we don't have/use. Fixed by https://sqlite.org/src/info/b1e0c22ec981cf5f
+CVE_CHECK_WHITELIST += "CVE-2021-36690"
-- 
2.32.0

[PATCH 04/14] recipes: Add missing pkgconfig inherit

[Richard Purdie]

Various recipes were missing a pkgconfig inherit or pkgconfig-native
dependency despite using pkgconfig.

Add the inherit to igt-gpu-tools/gdb/libmodulemd/libwpe/xwayland/waffle
shaderc/iputils/wpebackend-fdo/lttng-ust.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/recipes-devtools/gdb/gdb-cross-canadian.inc         | 1 +
 meta/recipes-devtools/gdb/gdb.inc                        | 2 +-
 meta/recipes-devtools/libmodulemd/libmodulemd_git.bb     | 2 +-
 meta/recipes-extended/iputils/iputils_20210722.bb        | 2 +-
 meta/recipes-graphics/igt-gpu-tools/igt-gpu-tools_git.bb | 2 +-
 meta/recipes-graphics/shaderc/shaderc_2021.1.bb          | 2 +-
 meta/recipes-graphics/waffle/waffle_1.6.1.bb             | 2 +-
 meta/recipes-graphics/xwayland/xwayland_21.1.2.bb        | 2 +-
 meta/recipes-kernel/lttng/lttng-ust_2.13.0.bb            | 2 +-
 meta/recipes-sato/webkit/libwpe_1.10.1.bb                | 2 +-
 meta/recipes-sato/webkit/wpebackend-fdo_1.10.0.bb        | 2 +-
 11 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/meta/recipes-devtools/gdb/gdb-cross-canadian.inc b/meta/recipes-devtools/gdb/gdb-cross-canadian.inc
index 670534fb5f0..dfacba29a01 100644
--- a/meta/recipes-devtools/gdb/gdb-cross-canadian.inc
+++ b/meta/recipes-devtools/gdb/gdb-cross-canadian.inc
@@ -1,5 +1,6 @@
 inherit cross-canadian
 inherit python3-dir
+inherit pkgconfig
 
 SUMMARY = "GNU debugger (cross-canadian gdb for ${TARGET_ARCH} target)"
 PN = "gdb-cross-canadian-${TRANSLATED_TARGET_ARCH}"
diff --git a/meta/recipes-devtools/gdb/gdb.inc b/meta/recipes-devtools/gdb/gdb.inc
index 2b0ae1655f6..2c95ed3ca0f 100644
--- a/meta/recipes-devtools/gdb/gdb.inc
+++ b/meta/recipes-devtools/gdb/gdb.inc
@@ -1,6 +1,6 @@
 require gdb-common.inc
 
-inherit gettext
+inherit gettext pkgconfig
 
 #LDFLAGS:append = " -s"
 #export CFLAGS:append=" -L${STAGING_LIBDIR}"
diff --git a/meta/recipes-devtools/libmodulemd/libmodulemd_git.bb b/meta/recipes-devtools/libmodulemd/libmodulemd_git.bb
index 963e881d6c1..ee8ce4d6a67 100644
--- a/meta/recipes-devtools/libmodulemd/libmodulemd_git.bb
+++ b/meta/recipes-devtools/libmodulemd/libmodulemd_git.bb
@@ -11,7 +11,7 @@ SRCREV = "1a032da198333ee77bdbe4be65e60eb4115ea73f"
 
 S = "${WORKDIR}/git"
 
-inherit meson gobject-introspection
+inherit meson gobject-introspection pkgconfig
 
 EXTRA_OEMESON = "-Dwith_py3=false -Dwith_docs=false -Drpmio=disabled -Dlibmagic=disabled -Dwith_manpages=disabled -Dgobject_overrides_dir_py3=${PYTHON_SITEPACKAGES_DIR}/gi/overrides"
 
diff --git a/meta/recipes-extended/iputils/iputils_20210722.bb b/meta/recipes-extended/iputils/iputils_20210722.bb
index a9821e556f0..e1940b77b5a 100644
--- a/meta/recipes-extended/iputils/iputils_20210722.bb
+++ b/meta/recipes-extended/iputils/iputils_20210722.bb
@@ -37,7 +37,7 @@ PACKAGECONFIG[tftpd] = "-DBUILD_TFTPD=true, -DBUILD_TFTPD=false,"
 PACKAGECONFIG[traceroute6] = "-DBUILD_TRACEROUTE6=true,-DBUILD_TRACEROUTE6=false,"
 PACKAGECONFIG[docs] = "-DBUILD_HTML_MANS=true -DBUILD_MANS=true,-DBUILD_HTML_MANS=false -DBUILD_MANS=false, libxslt"
 
-inherit meson systemd update-alternatives
+inherit meson systemd update-alternatives pkgconfig
 
 EXTRA_OEMESON += "--prefix=${root_prefix}/ -DSKIP_TESTS=true"
 
diff --git a/meta/recipes-graphics/igt-gpu-tools/igt-gpu-tools_git.bb b/meta/recipes-graphics/igt-gpu-tools/igt-gpu-tools_git.bb
index 79fab7a28a5..1a00eca7d3a 100644
--- a/meta/recipes-graphics/igt-gpu-tools/igt-gpu-tools_git.bb
+++ b/meta/recipes-graphics/igt-gpu-tools/igt-gpu-tools_git.bb
@@ -7,7 +7,7 @@ LIC_FILES_CHKSUM = "file://COPYING;md5=67bfee4df38fa6ecbe3a675c552d4c08"
 
 LICENSE = "MIT"
 
-inherit meson
+inherit meson pkgconfig
 
 SRCREV = "203def046b466fb2da67f9f15552d84e1c0b41f2"
 PV = "1.26"
diff --git a/meta/recipes-graphics/shaderc/shaderc_2021.1.bb b/meta/recipes-graphics/shaderc/shaderc_2021.1.bb
index e15b5794ee7..ad3f62ce17d 100644
--- a/meta/recipes-graphics/shaderc/shaderc_2021.1.bb
+++ b/meta/recipes-graphics/shaderc/shaderc_2021.1.bb
@@ -14,7 +14,7 @@ SRC_URI = "git://github.com/google/shaderc.git;protocol=https;branch=main \
 UPSTREAM_CHECK_GITTAGREGEX = "^v(?P<pver>\d+(\.\d+)+)$"
 S = "${WORKDIR}/git"
 
-inherit cmake python3native
+inherit cmake python3native pkgconfig
 
 DEPENDS = "spirv-headers spirv-tools glslang"
 
diff --git a/meta/recipes-graphics/waffle/waffle_1.6.1.bb b/meta/recipes-graphics/waffle/waffle_1.6.1.bb
index fa5efc12c45..395494018a5 100644
--- a/meta/recipes-graphics/waffle/waffle_1.6.1.bb
+++ b/meta/recipes-graphics/waffle/waffle_1.6.1.bb
@@ -15,7 +15,7 @@ SRC_URI[sha256sum] = "31565649ff0e2d8dff1b8f7f2264ab7a78452063c7e04adfc4ce03e64b
 
 UPSTREAM_CHECK_URI = "http://www.waffle-gl.org/releases.html"
 
-inherit meson features_check lib_package bash-completion
+inherit meson features_check lib_package bash-completion pkgconfig
 
 DEPENDS:append = " python3"
 
diff --git a/meta/recipes-graphics/xwayland/xwayland_21.1.2.bb b/meta/recipes-graphics/xwayland/xwayland_21.1.2.bb
index 3df6fd1ff90..6aba65935bf 100644
--- a/meta/recipes-graphics/xwayland/xwayland_21.1.2.bb
+++ b/meta/recipes-graphics/xwayland/xwayland_21.1.2.bb
@@ -14,7 +14,7 @@ SRC_URI[sha256sum] = "b81cbdd5ad60b8b7ad8c3ecc7ec2a28c9bf021448670735cebb501f08b
 
 UPSTREAM_CHECK_REGEX = "xwayland-(?P<pver>\d+(\.(?!90\d)\d+)+)\.tar"
 
-inherit meson features_check
+inherit meson features_check pkgconfig
 REQUIRED_DISTRO_FEATURES = "x11 opengl"
 
 DEPENDS += "xorgproto xtrans pixman libxkbfile libxfont2 wayland wayland-native wayland-protocols libdrm libepoxy"
diff --git a/meta/recipes-kernel/lttng/lttng-ust_2.13.0.bb b/meta/recipes-kernel/lttng/lttng-ust_2.13.0.bb
index 2074b8c3186..3b0e205ab86 100644
--- a/meta/recipes-kernel/lttng/lttng-ust_2.13.0.bb
+++ b/meta/recipes-kernel/lttng/lttng-ust_2.13.0.bb
@@ -11,7 +11,7 @@ PYTHON_OPTION = "am_cv_python_pyexecdir='${PYTHON_SITEPACKAGES_DIR}' \
                  PYTHON_INCLUDE='-I${STAGING_INCDIR}/python${PYTHON_BASEVERSION}${PYTHON_ABI}' \
 "
 
-inherit autotools lib_package manpages python3native
+inherit autotools lib_package manpages python3native pkgconfig
 
 include lttng-platforms.inc
 
diff --git a/meta/recipes-sato/webkit/libwpe_1.10.1.bb b/meta/recipes-sato/webkit/libwpe_1.10.1.bb
index 4e77481bc8f..68214ffbad0 100644
--- a/meta/recipes-sato/webkit/libwpe_1.10.1.bb
+++ b/meta/recipes-sato/webkit/libwpe_1.10.1.bb
@@ -6,7 +6,7 @@ LICENSE = "BSD-2-Clause"
 LIC_FILES_CHKSUM = "file://COPYING;md5=371a616eb4903c6cb79e9893a5f615cc"
 DEPENDS = "virtual/egl libxkbcommon"
 
-inherit cmake features_check
+inherit cmake features_check pkgconfig
 
 REQUIRED_DISTRO_FEATURES = "opengl"
 
diff --git a/meta/recipes-sato/webkit/wpebackend-fdo_1.10.0.bb b/meta/recipes-sato/webkit/wpebackend-fdo_1.10.0.bb
index 29ffb9091fe..f2d640f07a5 100644
--- a/meta/recipes-sato/webkit/wpebackend-fdo_1.10.0.bb
+++ b/meta/recipes-sato/webkit/wpebackend-fdo_1.10.0.bb
@@ -8,7 +8,7 @@ DEPENDS = "glib-2.0 libxkbcommon wayland virtual/egl libwpe libepoxy"
 
 DEPENDS:append:class-target = " wayland-native"
 
-inherit meson features_check
+inherit meson features_check pkgconfig
 
 REQUIRED_DISTRO_FEATURES = "opengl"
 
-- 
2.32.0

[PATCH 05/14] lttng-tools: Add missing DEPENDS on bison-native

[Richard Purdie]

This was being provided by other pieces of the dependency chain but is
specifically required by configure and could fail if those pieces come
from sstate. Fix such builds by adding the missing dependency.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb b/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb
index 95bb2519d7d..47cab42fcf5 100644
--- a/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb
+++ b/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb
@@ -12,7 +12,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=40ef17463fbd6f377db3c47b1cbaded8 \
 
 include lttng-platforms.inc
 
-DEPENDS = "liburcu popt libxml2 util-linux"
+DEPENDS = "liburcu popt libxml2 util-linux bison-native"
 RDEPENDS:${PN} = "libgcc"
 RRECOMMENDS:${PN} += "${LTTNGMODULES}"
 RDEPENDS:${PN}-ptest += "make perl bash gawk babeltrace procps perl-module-overloading coreutils util-linux kmod ${LTTNGMODULES} sed python3-core grep"
-- 
2.32.0

Re: [OE-core] [PATCH 05/14] lttng-tools: Add missing DEPENDS on bison-native

[Denys Dmytriyenko]

Steve,

Can we get this backported to dunfell, please? Thanks!


On Mon, Sep 20, 2021 at 01:46:12PM +0100, Richard Purdie wrote:
> This was being provided by other pieces of the dependency chain but is
> specifically required by configure and could fail if those pieces come
> from sstate. Fix such builds by adding the missing dependency.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb b/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb
> index 95bb2519d7d..47cab42fcf5 100644
> --- a/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb
> +++ b/meta/recipes-kernel/lttng/lttng-tools_2.13.0.bb
> @@ -12,7 +12,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=40ef17463fbd6f377db3c47b1cbaded8 \
>  
>  include lttng-platforms.inc
>  
> -DEPENDS = "liburcu popt libxml2 util-linux"
> +DEPENDS = "liburcu popt libxml2 util-linux bison-native"
>  RDEPENDS:${PN} = "libgcc"
>  RRECOMMENDS:${PN} += "${LTTNGMODULES}"
>  RDEPENDS:${PN}-ptest += "make perl bash gawk babeltrace procps perl-module-overloading coreutils util-linux kmod ${LTTNGMODULES} sed python3-core grep"
> -- 
> 2.32.0
> 

-- 
Regards,
Denys Dmytriyenko <denis@denix.org>
PGP: 0x420902729A92C964 - https://denix.org/0x420902729A92C964
Fingerprint: 25FC E4A5 8A72 2F69 1186  6D76 4209 0272 9A92 C964

[PATCH 06/14] image/qemu: Add explict depends for qemu-helper addto_recipe_sysroot task

[Richard Purdie]

The populate_sysroot task isn't enough for qemu-helper-native, we need
it's addto_recipe_sysroot task. This corrects what amounts to bad
dependency information to be explicit.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/image.bbclass         | 5 ++++-
 meta/conf/machine/include/qemu.inc | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index d76895178fb..1d88ccd8192 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -138,7 +138,10 @@ python () {
     def extraimage_getdepends(task):
         deps = ""
         for dep in (d.getVar('EXTRA_IMAGEDEPENDS') or "").split():
-            deps += " %s:%s" % (dep, task)
+            if ":" in dep:
+                deps += " %s " % (dep)
+            else:
+                deps += " %s:%s" % (dep, task)
         return deps
 
     d.appendVarFlag('do_image_complete', 'depends', extraimage_getdepends('do_populate_sysroot'))
diff --git a/meta/conf/machine/include/qemu.inc b/meta/conf/machine/include/qemu.inc
index c7136da7116..642c322abcb 100644
--- a/meta/conf/machine/include/qemu.inc
+++ b/meta/conf/machine/include/qemu.inc
@@ -21,7 +21,7 @@ RDEPENDS:${KERNEL_PACKAGE_NAME}-base = ""
 # Use a common kernel recipe for all QEMU machines
 PREFERRED_PROVIDER_virtual/kernel ??= "linux-yocto"
 
-EXTRA_IMAGEDEPENDS += "qemu-system-native qemu-helper-native"
+EXTRA_IMAGEDEPENDS += "qemu-system-native qemu-helper-native:do_addto_recipe_sysroot"
 
 # Provide the nfs server kernel module for all qemu images
 KERNEL_FEATURES:append:pn-linux-yocto = " features/nfsd/nfsd-enable.scc"
-- 
2.32.0

[PATCH 07/14] staging: Mark deploy an sstate task

[Richard Purdie]

Deploy is a tricky task as it isn't in SSTATETASKS as it isn't always
present. We do need to ensure dependency chains around do_deploy tasks
are correctly handled as sstate tasks though. For now add to the list of
sstate tasks manually (like the other locale task reference).

Without this, missing manifest files could be reported now that do_deploy
tasks no longer have their dependencies added by the depvalid function.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/staging.bbclass | 1 +
 1 file changed, 1 insertion(+)

diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index af3397bab6f..65a6cd5120b 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -306,6 +306,7 @@ python extend_recipe_sysroot() {
     sstatetasks = d.getVar("SSTATETASKS").split()
     # Add recipe specific tasks referenced by setscene_depvalid()
     sstatetasks.append("do_stash_locale")
+    sstatetasks.append("do_deploy")
 
     def print_dep_tree(deptree):
         data = ""
-- 
2.32.0

[PATCH 08/14] sstate: Ensure deploy tasks don't pull in toolchains

[Richard Purdie]

If an image is built from sstate, the cross toolchain was being pulled in.
This was due to the sstate dependencies for deploy tasks not being considered
in the "depvalid" logic.

do_deploy tasks do not need their populate_sysroot dependencies when installed
from sstate so skip these. This reduces the sstate pulled in by an image
build from sstate significantly.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/sstate.bbclass | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 29679e6a5ee..498dfc681cf 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1090,8 +1090,8 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
         # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
         if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
             return False
-        # do_package/packagedata/package_qa don't need do_populate_sysroot
-        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
+        # do_package/packagedata/package_qa/deploy don't need do_populate_sysroot
+        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa', 'do_deploy']:
             continue
         # Native/Cross packages don't exist and are noexec anyway
         if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
-- 
2.32.0

[PATCH 09/14] sstate: Avoid deploy_source_date_epoch sstate when unneeded

[Richard Purdie]

This sstate task is only needed when depended upon, it can be skipped
if there are no tasks running that directly depend upon it.

This reduced the number of sstate tasks in something like an image
build.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/sstate.bbclass | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 498dfc681cf..5accc13a89b 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -1066,6 +1066,10 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
     if taskdependees[task][1] == "do_populate_lic":
         return True
 
+    # We only need to trigger deploy_source_date_epoch through direct dependencies
+    if taskdependees[task][1] == "do_deploy_source_date_epoch":
+        return True
+
     # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
     if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
         return True
-- 
2.32.0

[RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive do_build task dependencies

[Richard Purdie]

This is a controversial change which removes the recursive dependencies
from the do_build target of packaging tasks of recipes.

Currently this means when you "bitbake <image>" or "bitbake <recipe>",
the packaging tasks run for all packaging backends enabled for all recipes
in the dependency chain. The same therefore then applies to images.

We don't actually need that, it is a convinience thing. Removing it
massively simplifies the task graph and causes much fewer tasks to execute
in many common scenarios. It also means less sstate is fetched for
example when building an image.

This means when building a recipe all package formats would be built
but when building an image, only the format used by the image would be
generated. This should be an improvement in most cases but some CI systems
may need to be explict about what they're building.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/package_deb.bbclass | 5 +----
 meta/classes/package_ipk.bbclass | 4 +---
 meta/classes/package_rpm.bbclass | 4 +---
 3 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index eca43e17876..77caaa55c8f 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -315,10 +315,7 @@ python do_package_write_deb () {
 do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}"
 do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}"
 do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
-addtask package_write_deb after do_packagedata do_package
-
+addtask package_write_deb after do_packagedata do_package before do_build
 
 PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot"
 PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot"
-
-do_build[recrdeptask] += "do_package_write_deb"
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index c3b53854e8b..998e18ba6c1 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -274,9 +274,7 @@ python do_package_write_ipk () {
 do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}"
 do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}"
 do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
-addtask package_write_ipk after do_packagedata do_package
+addtask package_write_ipk after do_packagedata do_package before do_build
 
 PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot"
 PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot"
-
-do_build[recrdeptask] += "do_package_write_ipk"
diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index 88d861c0e75..60f8299d965 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -748,9 +748,7 @@ python do_package_write_rpm () {
 do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}"
 do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}"
 do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
-addtask package_write_rpm after do_packagedata do_package
+addtask package_write_rpm after do_packagedata do_package before do_build
 
 PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot"
 PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot"
-
-do_build[recrdeptask] += "do_package_write_rpm"
-- 
2.32.0

Re: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive do_build task dependencies

[Peter Kjellerstedt]

> -----Original Message-----
> From: openembedded-core@lists.openembedded.org <openembedded-
> core@lists.openembedded.org> On Behalf Of Richard Purdie
> Sent: den 20 september 2021 14:46
> To: openembedded-core@lists.openembedded.org
> Subject: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive
> do_build task dependencies
> 
> This is a controversial change which removes the recursive dependencies
> from the do_build target of packaging tasks of recipes.
> 
> Currently this means when you "bitbake <image>" or "bitbake <recipe>",
> the packaging tasks run for all packaging backends enabled for all recipes
> in the dependency chain. The same therefore then applies to images.
> 
> We don't actually need that, it is a convinience thing. Removing it
> massively simplifies the task graph and causes much fewer tasks to execute
> in many common scenarios. It also means less sstate is fetched for
> example when building an image.
> 
> This means when building a recipe all package formats would be built
> but when building an image, only the format used by the image would be
> generated. This should be an improvement in most cases but some CI systems
> may need to be explict about what they're building.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/classes/package_deb.bbclass | 5 +----
>  meta/classes/package_ipk.bbclass | 4 +---
>  meta/classes/package_rpm.bbclass | 4 +---
>  3 files changed, 3 insertions(+), 10 deletions(-)
> 
> diff --git a/meta/classes/package_deb.bbclass
> b/meta/classes/package_deb.bbclass
> index eca43e17876..77caaa55c8f 100644
> --- a/meta/classes/package_deb.bbclass
> +++ b/meta/classes/package_deb.bbclass
> @@ -315,10 +315,7 @@ python do_package_write_deb () {
>  do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}"
>  do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}"
>  do_package_write_deb[depends] +=
> "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> 'do_populate_sysroot')}"
> -addtask package_write_deb after do_packagedata do_package
> -
> +addtask package_write_deb after do_packagedata do_package before do_build
> 
>  PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot"
>  PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot"
> -
> -do_build[recrdeptask] += "do_package_write_deb"
> diff --git a/meta/classes/package_ipk.bbclass
> b/meta/classes/package_ipk.bbclass
> index c3b53854e8b..998e18ba6c1 100644
> --- a/meta/classes/package_ipk.bbclass
> +++ b/meta/classes/package_ipk.bbclass
> @@ -274,9 +274,7 @@ python do_package_write_ipk () {
>  do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}"
>  do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}"
>  do_package_write_ipk[depends] +=
> "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> 'do_populate_sysroot')}"
> -addtask package_write_ipk after do_packagedata do_package
> +addtask package_write_ipk after do_packagedata do_package before do_build
> 
>  PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot"
>  PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot"
> -
> -do_build[recrdeptask] += "do_package_write_ipk"
> diff --git a/meta/classes/package_rpm.bbclass
> b/meta/classes/package_rpm.bbclass
> index 88d861c0e75..60f8299d965 100644
> --- a/meta/classes/package_rpm.bbclass
> +++ b/meta/classes/package_rpm.bbclass
> @@ -748,9 +748,7 @@ python do_package_write_rpm () {
>  do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}"
>  do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}"
>  do_package_write_rpm[depends] +=
> "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> 'do_populate_sysroot')}"
> -addtask package_write_rpm after do_packagedata do_package
> +addtask package_write_rpm after do_packagedata do_package before do_build
> 
>  PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot"
>  PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot"
> -
> -do_build[recrdeptask] += "do_package_write_rpm"
> --
> 2.32.0

I just wanted to give some positive feedback now that this has been 
integrated. This patch alone reduced the number of edges in the task 
graph by 50% for one of our typical products (from 127721 to 64785). 
That is pretty impressive!

And to add to this, the later patch that removes the use of the meta 
class reduced the number of nodes in the task graph by 10% (from 17122 
to 15313). Also pretty impressive for such a small change.

//Peter

Re: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive do_build task dependencies

[Richard Purdie]

On Thu, 2021-09-23 at 21:41 +0000, Peter Kjellerstedt wrote:
> > -----Original Message-----
> > From: openembedded-core@lists.openembedded.org <openembedded-
> > core@lists.openembedded.org> On Behalf Of Richard Purdie
> > Sent: den 20 september 2021 14:46
> > To: openembedded-core@lists.openembedded.org
> > Subject: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive
> > do_build task dependencies
> > 
> > This is a controversial change which removes the recursive dependencies
> > from the do_build target of packaging tasks of recipes.
> > 
> > Currently this means when you "bitbake <image>" or "bitbake <recipe>",
> > the packaging tasks run for all packaging backends enabled for all recipes
> > in the dependency chain. The same therefore then applies to images.
> > 
> > We don't actually need that, it is a convinience thing. Removing it
> > massively simplifies the task graph and causes much fewer tasks to execute
> > in many common scenarios. It also means less sstate is fetched for
> > example when building an image.
> > 
> > This means when building a recipe all package formats would be built
> > but when building an image, only the format used by the image would be
> > generated. This should be an improvement in most cases but some CI systems
> > may need to be explict about what they're building.
> > 
> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > ---
> >  meta/classes/package_deb.bbclass | 5 +----
> >  meta/classes/package_ipk.bbclass | 4 +---
> >  meta/classes/package_rpm.bbclass | 4 +---
> >  3 files changed, 3 insertions(+), 10 deletions(-)
> > 
> > diff --git a/meta/classes/package_deb.bbclass
> > b/meta/classes/package_deb.bbclass
> > index eca43e17876..77caaa55c8f 100644
> > --- a/meta/classes/package_deb.bbclass
> > +++ b/meta/classes/package_deb.bbclass
> > @@ -315,10 +315,7 @@ python do_package_write_deb () {
> >  do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}"
> >  do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}"
> >  do_package_write_deb[depends] +=
> > "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> > 'do_populate_sysroot')}"
> > -addtask package_write_deb after do_packagedata do_package
> > -
> > +addtask package_write_deb after do_packagedata do_package before do_build
> > 
> >  PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot"
> >  PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot"
> > -
> > -do_build[recrdeptask] += "do_package_write_deb"
> > diff --git a/meta/classes/package_ipk.bbclass
> > b/meta/classes/package_ipk.bbclass
> > index c3b53854e8b..998e18ba6c1 100644
> > --- a/meta/classes/package_ipk.bbclass
> > +++ b/meta/classes/package_ipk.bbclass
> > @@ -274,9 +274,7 @@ python do_package_write_ipk () {
> >  do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}"
> >  do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}"
> >  do_package_write_ipk[depends] +=
> > "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> > 'do_populate_sysroot')}"
> > -addtask package_write_ipk after do_packagedata do_package
> > +addtask package_write_ipk after do_packagedata do_package before do_build
> > 
> >  PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot"
> >  PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot"
> > -
> > -do_build[recrdeptask] += "do_package_write_ipk"
> > diff --git a/meta/classes/package_rpm.bbclass
> > b/meta/classes/package_rpm.bbclass
> > index 88d861c0e75..60f8299d965 100644
> > --- a/meta/classes/package_rpm.bbclass
> > +++ b/meta/classes/package_rpm.bbclass
> > @@ -748,9 +748,7 @@ python do_package_write_rpm () {
> >  do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}"
> >  do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}"
> >  do_package_write_rpm[depends] +=
> > "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> > 'do_populate_sysroot')}"
> > -addtask package_write_rpm after do_packagedata do_package
> > +addtask package_write_rpm after do_packagedata do_package before do_build
> > 
> >  PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot"
> >  PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot"
> > -
> > -do_build[recrdeptask] += "do_package_write_rpm"
> > --
> > 2.32.0
> 
> I just wanted to give some positive feedback now that this has been 
> integrated. This patch alone reduced the number of edges in the task 
> graph by 50% for one of our typical products (from 127721 to 64785). 
> That is pretty impressive!
> 
> And to add to this, the later patch that removes the use of the meta 
> class reduced the number of nodes in the task graph by 10% (from 17122 
> to 15313). Also pretty impressive for such a small change.

Thanks, it is nice to hear something working out well like that! :)

I've been working towards this for a while but we've never been quite ready for
the final pieces until now.

Cheers,

Richard

Re: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive do_build task dependencies

[Khem Raj]

On Thu, Sep 23, 2021 at 2:42 PM Peter Kjellerstedt
<peter.kjellerstedt@axis.com> wrote:
>
> > -----Original Message-----
> > From: openembedded-core@lists.openembedded.org <openembedded-
> > core@lists.openembedded.org> On Behalf Of Richard Purdie
> > Sent: den 20 september 2021 14:46
> > To: openembedded-core@lists.openembedded.org
> > Subject: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive
> > do_build task dependencies
> >
> > This is a controversial change which removes the recursive dependencies
> > from the do_build target of packaging tasks of recipes.
> >
> > Currently this means when you "bitbake <image>" or "bitbake <recipe>",
> > the packaging tasks run for all packaging backends enabled for all recipes
> > in the dependency chain. The same therefore then applies to images.
> >
> > We don't actually need that, it is a convinience thing. Removing it
> > massively simplifies the task graph and causes much fewer tasks to execute
> > in many common scenarios. It also means less sstate is fetched for
> > example when building an image.
> >
> > This means when building a recipe all package formats would be built
> > but when building an image, only the format used by the image would be
> > generated. This should be an improvement in most cases but some CI systems
> > may need to be explict about what they're building.
> >
> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > ---
> >  meta/classes/package_deb.bbclass | 5 +----
> >  meta/classes/package_ipk.bbclass | 4 +---
> >  meta/classes/package_rpm.bbclass | 4 +---
> >  3 files changed, 3 insertions(+), 10 deletions(-)
> >
> > diff --git a/meta/classes/package_deb.bbclass
> > b/meta/classes/package_deb.bbclass
> > index eca43e17876..77caaa55c8f 100644
> > --- a/meta/classes/package_deb.bbclass
> > +++ b/meta/classes/package_deb.bbclass
> > @@ -315,10 +315,7 @@ python do_package_write_deb () {
> >  do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}"
> >  do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}"
> >  do_package_write_deb[depends] +=
> > "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> > 'do_populate_sysroot')}"
> > -addtask package_write_deb after do_packagedata do_package
> > -
> > +addtask package_write_deb after do_packagedata do_package before do_build
> >
> >  PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot"
> >  PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot"
> > -
> > -do_build[recrdeptask] += "do_package_write_deb"
> > diff --git a/meta/classes/package_ipk.bbclass
> > b/meta/classes/package_ipk.bbclass
> > index c3b53854e8b..998e18ba6c1 100644
> > --- a/meta/classes/package_ipk.bbclass
> > +++ b/meta/classes/package_ipk.bbclass
> > @@ -274,9 +274,7 @@ python do_package_write_ipk () {
> >  do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}"
> >  do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}"
> >  do_package_write_ipk[depends] +=
> > "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> > 'do_populate_sysroot')}"
> > -addtask package_write_ipk after do_packagedata do_package
> > +addtask package_write_ipk after do_packagedata do_package before do_build
> >
> >  PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot"
> >  PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot"
> > -
> > -do_build[recrdeptask] += "do_package_write_ipk"
> > diff --git a/meta/classes/package_rpm.bbclass
> > b/meta/classes/package_rpm.bbclass
> > index 88d861c0e75..60f8299d965 100644
> > --- a/meta/classes/package_rpm.bbclass
> > +++ b/meta/classes/package_rpm.bbclass
> > @@ -748,9 +748,7 @@ python do_package_write_rpm () {
> >  do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}"
> >  do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}"
> >  do_package_write_rpm[depends] +=
> > "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'),
> > 'do_populate_sysroot')}"
> > -addtask package_write_rpm after do_packagedata do_package
> > +addtask package_write_rpm after do_packagedata do_package before do_build
> >
> >  PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot"
> >  PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot"
> > -
> > -do_build[recrdeptask] += "do_package_write_rpm"
> > --
> > 2.32.0
>
> I just wanted to give some positive feedback now that this has been
> integrated. This patch alone reduced the number of edges in the task
> graph by 50% for one of our typical products (from 127721 to 64785).
> That is pretty impressive!
>
> And to add to this, the later patch that removes the use of the meta
> class reduced the number of nodes in the task graph by 10% (from 17122
> to 15313). Also pretty impressive for such a small change.
>

this is quite encouraging, dont go by number of lines of change :)
btw. did you also try with other change which reduces native dependencies,

> //Peter
>
> 
>

Re: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive do_build task dependencies

[Martin Jansa]

[-- Attachment #1: Type: text/plain, Size: 963 bytes --]

On Fri, Sep 24, 2021 at 6:51 AM Khem Raj <raj.khem@gmail.com> wrote:

> this is quite encouraging, dont go by number of lines of change :)
> btw. did you also try with other change which reduces native dependencies,
>

I've tried layer.conf change in our builds and it causes _a lot_ of
pkgconfig issues, but in most cases they are easy to spot in failed task
log and easy to fix. With ~ 100 fixed recipes I can build the images again
:). I've already merged bunch of these fixes in various layers and queued
it for others. I've seen a lot of these fixes for meta-oe layers from Khem
as well, so I think it's also getting ready. But possibly less impressive
improvement and more fixes still needed for other external layers. Maybe we
could add pkgconfig inherit in e.g. cmake.bbclass, that would at least in
my case eliminate maybe 90% cases where I needed to add pkgconfig inherit
(because native pkgconfig is called from CMake's FindPkgConfig).

[-- Attachment #2: Type: text/html, Size: 1239 bytes --]

Re: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive do_build task dependencies

[Richard Purdie]

On Fri, 2021-09-24 at 09:58 +0200, Martin Jansa wrote:
> On Fri, Sep 24, 2021 at 6:51 AM Khem Raj <raj.khem@gmail.com> wrote:
> > this is quite encouraging, dont go by number of lines of change :)
> > btw. did you also try with other change which reduces native dependencies,
> > 
> 
>  
> I've tried layer.conf change in our builds and it causes _a lot_ of pkgconfig
> issues, but in most cases they are easy to spot in failed task log and easy to
> fix. With ~ 100 fixed recipes I can build the images again :). I've already
> merged bunch of these fixes in various layers and queued it for others. I've
> seen a lot of these fixes for meta-oe layers from Khem as well, so I think
> it's also getting ready. But possibly less impressive improvement and more
> fixes still needed for other external layers. Maybe we could add pkgconfig
> inherit in e.g. cmake.bbclass, that would at least in my case eliminate maybe
> 90% cases where I needed to add pkgconfig inherit (because native pkgconfig is
> called from CMake's FindPkgConfig). 

That could be an option, depends if there are many recipes which use cmake but
don't use pkgconfig.

FWIW I'm still planning to hold the layer.conf change causing that until after
release, unless I hear people saying the issues are mostly resolved.

I've merged the recursive build dependency changes as whilst it is an invasive
change, I think it puts sstate use and hence the project in a much better
position for the future.

Cheers,

Richard

Re: [OE-core] [RFC PATCH 10/14] package_ipk/deb/rpm: Drop recursive do_build task dependencies

[Khem Raj]

On 9/24/21 12:58 AM, Martin Jansa wrote:
> On Fri, Sep 24, 2021 at 6:51 AM Khem Raj <raj.khem@gmail.com 
> <mailto:raj.khem@gmail.com>> wrote:
> 
>     this is quite encouraging, dont go by number of lines of change :)
>     btw. did you also try with other change which reduces native
>     dependencies,
> 
> I've tried layer.conf change in our builds and it causes _a lot_ of 
> pkgconfig issues, but in most cases they are easy to spot in failed task 
> log and easy to fix. With ~ 100 fixed recipes I can build the images 
> again :). I've already merged bunch of these fixes in various layers and 
> queued it for others. I've seen a lot of these fixes for meta-oe layers 
> from Khem as well, so I think it's also getting ready. But possibly less 
> impressive improvement and more fixes still needed for other external 
> layers. Maybe we could add pkgconfig inherit in e.g. cmake.bbclass, that 
> would at least in my case eliminate maybe 90% cases where I needed to 
> add pkgconfig inherit (because native pkgconfig is called from CMake's 
> FindPkgConfig).

meta-openembedded layers are now fixed for the pkgcconfig issues. So we 
should be in a sightly better shape

[RFC PATCH 11/14] populate_sdk_base/images: Drop use of 'meta' class and hence do_build dependencies

[Richard Purdie]

The 'meta' is old and not very useful. It empties PACKAGES and creates
recursive do_build dependencies.

We shouldn't need such recursive build dependencies any more so simplify the code.

This does cause behaviour changes as some dependencies are no longer built.
It did show up issues with qemu-helper-native handling for example but those
issues look like real races and the underlying dependency chains were not correct.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/populate_sdk_base.bbclass | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
index ccfe2232895..49e166e697d 100644
--- a/meta/classes/populate_sdk_base.bbclass
+++ b/meta/classes/populate_sdk_base.bbclass
@@ -1,4 +1,6 @@
-inherit meta image-postinst-intercepts image-artifact-names
+PACKAGES = ""
+
+inherit image-postinst-intercepts image-artifact-names
 
 # Wildcards specifying complementary packages to install for every package that has been explicitly
 # installed into the rootfs
-- 
2.32.0

Re: [OE-core] [RFC PATCH 11/14] populate_sdk_base/images: Drop use of 'meta' class and hence do_build dependencies

[ChenQi]

[-- Attachment #1: Type: text/plain, Size: 2031 bytes --]

Hi Richard,

I like this change. However, it causes problem of 'rm_work'.
Now some native recipes don't get cleaned up with 'rm_work' enabled.

e.g.
INHERIT += "rm_work"
bitbake core-image-minimal
A few native recipes don't get clean up such as alsa-lib-native, 
createrepo-c-native, dnf-native, etc.

Do you have any idea how to fix it? Or should we accept the current 
situation?

Regards,
Qi

On 09/20/2021 08:46 PM, Richard Purdie wrote:
> The 'meta' is old and not very useful. It empties PACKAGES and creates
> recursive do_build dependencies.
>
> We shouldn't need such recursive build dependencies any more so simplify the code.
>
> This does cause behaviour changes as some dependencies are no longer built.
> It did show up issues with qemu-helper-native handling for example but those
> issues look like real races and the underlying dependency chains were not correct.
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>   meta/classes/populate_sdk_base.bbclass | 4 +++-
>   1 file changed, 3 insertions(+), 1 deletion(-)
>
> diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
> index ccfe2232895..49e166e697d 100644
> --- a/meta/classes/populate_sdk_base.bbclass
> +++ b/meta/classes/populate_sdk_base.bbclass
> @@ -1,4 +1,6 @@
> -inherit meta image-postinst-intercepts image-artifact-names
> +PACKAGES = ""
> +
> +inherit image-postinst-intercepts image-artifact-names
>   
>   # Wildcards specifying complementary packages to install for every package that has been explicitly
>   # installed into the rootfs
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#156182): https://lists.openembedded.org/g/openembedded-core/message/156182
> Mute This Topic: https://lists.openembedded.org/mt/85739582/3618072
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [Qi.Chen@windriver.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>


[-- Attachment #2: Type: text/html, Size: 3190 bytes --]

Re: [OE-core] [RFC PATCH 11/14] populate_sdk_base/images: Drop use of 'meta' class and hence do_build dependencies

[Richard Purdie]

On Wed, 2021-10-27 at 10:43 +0800, ChenQi wrote:
> 
>  I like this change. However, it causes problem of 'rm_work'.
>  Now some native recipes don't get cleaned up with 'rm_work' enabled.
>  
>  e.g.
>  INHERIT += "rm_work"
>  bitbake core-image-minimal
>  A few native recipes don't get clean up such as alsa-lib-native, createrepo-
> c-native, dnf-native, etc.
>  
>  Do you have any idea how to fix it? Or should we accept the current
> situation?
>  

I think the issue is that populate_sdk is an odd task in that it doesn't run
before do_build (nor should it). rm_work will therefore not get triggered since
the class does:

addtask rm_work_all before do_build

What might work is something like:

do_populate_sdk[recrdeptask] = "do_rm_work"

but that may have some horrible side effects, I'm not sure.

Cheers,

Richard

[PATCH 12/14] buildtools-tarball/uninative-tarball/meta-ide-support: Drop useless meta class

[Richard Purdie]

The class adds an emtpy PACKAGES setting but most code now uses the
nopackages class which is much clearer. It also adds recursive do_build
dependencies which don't really serve any useful purpose any more.

Simplify the code and drop the class use.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/recipes-core/meta/buildtools-tarball.bb | 1 -
 meta/recipes-core/meta/meta-ide-support.bb   | 2 +-
 meta/recipes-core/meta/testexport-tarball.bb | 1 -
 meta/recipes-core/meta/uninative-tarball.bb  | 1 -
 4 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/meta/recipes-core/meta/buildtools-tarball.bb b/meta/recipes-core/meta/buildtools-tarball.bb
index 6e96cf6c326..fb28930656b 100644
--- a/meta/recipes-core/meta/buildtools-tarball.bb
+++ b/meta/recipes-core/meta/buildtools-tarball.bb
@@ -49,7 +49,6 @@ RDEPENDS = "${TOOLCHAIN_HOST_TASK}"
 
 EXCLUDE_FROM_WORLD = "1"
 
-inherit meta
 inherit populate_sdk
 inherit toolchain-scripts-base
 inherit nopackages
diff --git a/meta/recipes-core/meta/meta-ide-support.bb b/meta/recipes-core/meta/meta-ide-support.bb
index 768f6f4bb67..37e65a49cac 100644
--- a/meta/recipes-core/meta/meta-ide-support.bb
+++ b/meta/recipes-core/meta/meta-ide-support.bb
@@ -6,7 +6,7 @@ DEPENDS = "virtual/libc gdb-cross-${TARGET_ARCH} qemu-native qemu-helper-native
 PR = "r3"
 RM_WORK_EXCLUDE += "${PN}"
 
-inherit meta toolchain-scripts nopackages
+inherit toolchain-scripts nopackages
 
 do_populate_ide_support () {
   toolchain_create_tree_env_script
diff --git a/meta/recipes-core/meta/testexport-tarball.bb b/meta/recipes-core/meta/testexport-tarball.bb
index de7663b38c0..7cb84a01049 100644
--- a/meta/recipes-core/meta/testexport-tarball.bb
+++ b/meta/recipes-core/meta/testexport-tarball.bb
@@ -26,7 +26,6 @@ RDEPENDS = "${TOOLCHAIN_HOST_TASK}"
 
 EXCLUDE_FROM_WORLD = "1"
 
-inherit meta
 inherit populate_sdk
 inherit toolchain-scripts-base
 inherit nopackages
diff --git a/meta/recipes-core/meta/uninative-tarball.bb b/meta/recipes-core/meta/uninative-tarball.bb
index bbdaebad947..a21d08b591c 100644
--- a/meta/recipes-core/meta/uninative-tarball.bb
+++ b/meta/recipes-core/meta/uninative-tarball.bb
@@ -36,7 +36,6 @@ RDEPENDS = "${TOOLCHAIN_HOST_TASK}"
 
 EXCLUDE_FROM_WORLD = "1"
 
-inherit meta
 inherit populate_sdk
 inherit nopackages
 
-- 
2.32.0

[PATCH 13/14] meta: Drop useless class

[Richard Purdie]

This class was added by me back in 2007 and has changed one for a whitespace issue
since. It only has two lines and neither are particularly useful, one was replaced
by the nopackages class, the other adding recursive dependencies also is now
mainly problematic adding tons of unneeded dependencies. The name is hard to
understand and the class doesn't have a clear purpose. Drop it.

Remove the references in devtool (which may be the one reason to keep it around
but in my view still not worth it).

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/meta.bbclass               | 4 ----
 meta/lib/oeqa/selftest/cases/devtool.py | 2 +-
 scripts/lib/devtool/standard.py         | 4 ----
 3 files changed, 1 insertion(+), 9 deletions(-)
 delete mode 100644 meta/classes/meta.bbclass

diff --git a/meta/classes/meta.bbclass b/meta/classes/meta.bbclass
deleted file mode 100644
index 5e6890238b1..00000000000
--- a/meta/classes/meta.bbclass
+++ /dev/null
@@ -1,4 +0,0 @@
-
-PACKAGES = ""
-
-do_build[recrdeptask] = "do_build"
diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index 6d9cd46bf30..f495e84c79b 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -649,7 +649,7 @@ class DevtoolModifyTests(DevtoolBase):
         self.track_for_cleanup(self.workspacedir)
         self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
 
-        testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split()
+        testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk'.split()
         # Find actual name of gcc-source since it now includes the version - crude, but good enough for this purpose
         result = runCmd('bitbake-layers show-recipes gcc-source*')
         for line in result.output.splitlines():
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index b74a60d0011..01fb5ad96ff 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -318,10 +318,6 @@ def _check_compatible_recipe(pn, d):
         raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
                            "not supported by this tool" % pn, 4)
 
-    if bb.data.inherits_class('meta', d):
-        raise DevtoolError("The %s recipe is a meta-recipe, and therefore is "
-                           "not supported by this tool" % pn, 4)
-
     if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
         # Not an incompatibility error per se, so we don't pass the error code
         raise DevtoolError("externalsrc is currently enabled for the %s "
-- 
2.32.0

[RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Richard Purdie]

There are several dependencies which recipes have which are not needed
at runtime, only at build time. Extend the list of these from bison to
include quilt, patch, meson, autoconf, automake and ninja which should
reduce the amount of data being included in native sysroots.

This speeds up and reduces the sstate needed for image builds for example.

If this change breaks recipes, it probably means they're missing an explicit
DEPENDS on something in this list which is a bug in the recipe.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/conf/layer.conf | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
index 307ad7e2237..b3cc8a249e9 100644
--- a/meta/conf/layer.conf
+++ b/meta/conf/layer.conf
@@ -48,6 +48,7 @@ SIGGEN_EXCLUDERECIPES_ABISAFE += " \
 "
 
 SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
+  *->patch-native \
   *->quilt-native \
   *->subversion-native \
   *->git-native \
@@ -90,7 +91,16 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
 # (e.g. X -> Y -> binutils-cross -> bison-native) no longer meet the
 # dependency incidentally. This improves determinism and avoids build
 # failures when people switch to external toolchains.
-SSTATE_EXCLUDEDEPS_SYSROOT += ".*->bison-native"
+SSTATE_EXCLUDEDEPS_SYSROOT += "\
+    .*->autoconf-native \
+    .*->automake-native \
+    .*->bison-native \
+    .*->meson-native \
+    .*->ninja-native \
+    .*->patch-native \
+    .*->pkgconfig-native \
+    .*->quilt-native \
+"
 # Nothing needs to depend on libc-initial
 # base-passwd/shadow-sysroot don't need their dependencies
 SSTATE_EXCLUDEDEPS_SYSROOT += "\
-- 
2.32.0

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Richard Purdie]

On Mon, 2021-09-20 at 13:46 +0100, Richard Purdie via lists.openembedded.org
wrote:
> There are several dependencies which recipes have which are not needed
> at runtime, only at build time. Extend the list of these from bison to
> include quilt, patch, meson, autoconf, automake and ninja which should
> reduce the amount of data being included in native sysroots.
> 
> This speeds up and reduces the sstate needed for image builds for example.
> 
> If this change breaks recipes, it probably means they're missing an explicit
> DEPENDS on something in this list which is a bug in the recipe.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/conf/layer.conf | 12 +++++++++++-
>  1 file changed, 11 insertions(+), 1 deletion(-)
> 
> diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
> index 307ad7e2237..b3cc8a249e9 100644
> --- a/meta/conf/layer.conf
> +++ b/meta/conf/layer.conf
> @@ -48,6 +48,7 @@ SIGGEN_EXCLUDERECIPES_ABISAFE += " \
>  "
>  
>  SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
> +  *->patch-native \
>    *->quilt-native \
>    *->subversion-native \
>    *->git-native \
> @@ -90,7 +91,16 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
>  # (e.g. X -> Y -> binutils-cross -> bison-native) no longer meet the
>  # dependency incidentally. This improves determinism and avoids build
>  # failures when people switch to external toolchains.
> -SSTATE_EXCLUDEDEPS_SYSROOT += ".*->bison-native"
> +SSTATE_EXCLUDEDEPS_SYSROOT += "\
> +    .*->autoconf-native \
> +    .*->automake-native \
> +    .*->bison-native \
> +    .*->meson-native \
> +    .*->ninja-native \
> +    .*->patch-native \
> +    .*->pkgconfig-native \
> +    .*->quilt-native \
> +"

I've marked this one as an RFC since it caused a fair bit of pain even in OE-
core due to recipes having pkgconfig class inherits missing.

We do really want to do this and correctly mark up the recipe dependencies, the
question is whether it is too late for 3.4 or not. We could go for a smaller
list for now too but I wanted to hear opinions.

Cheers,

Richard

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Joshua Watt]

[-- Attachment #1: Type: text/plain, Size: 2352 bytes --]


On 9/20/21 7:48 AM, Richard Purdie wrote:
> On Mon, 2021-09-20 at 13:46 +0100, Richard Purdie via lists.openembedded.org
> wrote:
>> There are several dependencies which recipes have which are not needed
>> at runtime, only at build time. Extend the list of these from bison to
>> include quilt, patch, meson, autoconf, automake and ninja which should
>> reduce the amount of data being included in native sysroots.
>>
>> This speeds up and reduces the sstate needed for image builds for example.
>>
>> If this change breaks recipes, it probably means they're missing an explicit
>> DEPENDS on something in this list which is a bug in the recipe.
>>
>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>> ---
>>   meta/conf/layer.conf | 12 +++++++++++-
>>   1 file changed, 11 insertions(+), 1 deletion(-)
>>
>> diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
>> index 307ad7e2237..b3cc8a249e9 100644
>> --- a/meta/conf/layer.conf
>> +++ b/meta/conf/layer.conf
>> @@ -48,6 +48,7 @@ SIGGEN_EXCLUDERECIPES_ABISAFE += " \
>>   "
>>   
>>   SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
>> +  *->patch-native \
>>     *->quilt-native \
>>     *->subversion-native \
>>     *->git-native \
>> @@ -90,7 +91,16 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
>>   # (e.g. X -> Y -> binutils-cross -> bison-native) no longer meet the
>>   # dependency incidentally. This improves determinism and avoids build
>>   # failures when people switch to external toolchains.
>> -SSTATE_EXCLUDEDEPS_SYSROOT += ".*->bison-native"
>> +SSTATE_EXCLUDEDEPS_SYSROOT += "\
>> +    .*->autoconf-native \
>> +    .*->automake-native \
>> +    .*->bison-native \
>> +    .*->meson-native \
>> +    .*->ninja-native \
>> +    .*->patch-native \
>> +    .*->pkgconfig-native \
>> +    .*->quilt-native \
>> +"
> I've marked this one as an RFC since it caused a fair bit of pain even in OE-
> core due to recipes having pkgconfig class inherits missing.
>
> We do really want to do this and correctly mark up the recipe dependencies, the
> question is whether it is too late for 3.4 or not. We could go for a smaller
> list for now too but I wanted to hear opinions.


It looks like a great change! I'd say it's too late for 3.4 and we 
should do it for 3.5 instead.

>
> Cheers,
>
> Richard
>
>
> 
>

[-- Attachment #2: Type: text/html, Size: 3213 bytes --]

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Khem Raj]

On Mon, Sep 20, 2021 at 6:34 AM Joshua Watt <JPEWhacker@gmail.com> wrote:
>
>
> On 9/20/21 7:48 AM, Richard Purdie wrote:
>
> On Mon, 2021-09-20 at 13:46 +0100, Richard Purdie via lists.openembedded.org
> wrote:
>
> There are several dependencies which recipes have which are not needed
> at runtime, only at build time. Extend the list of these from bison to
> include quilt, patch, meson, autoconf, automake and ninja which should
> reduce the amount of data being included in native sysroots.
>
> This speeds up and reduces the sstate needed for image builds for example.
>
> If this change breaks recipes, it probably means they're missing an explicit
> DEPENDS on something in this list which is a bug in the recipe.
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/conf/layer.conf | 12 +++++++++++-
>  1 file changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
> index 307ad7e2237..b3cc8a249e9 100644
> --- a/meta/conf/layer.conf
> +++ b/meta/conf/layer.conf
> @@ -48,6 +48,7 @@ SIGGEN_EXCLUDERECIPES_ABISAFE += " \
>  "
>
>  SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
> +  *->patch-native \
>    *->quilt-native \
>    *->subversion-native \
>    *->git-native \
> @@ -90,7 +91,16 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
>  # (e.g. X -> Y -> binutils-cross -> bison-native) no longer meet the
>  # dependency incidentally. This improves determinism and avoids build
>  # failures when people switch to external toolchains.
> -SSTATE_EXCLUDEDEPS_SYSROOT += ".*->bison-native"
> +SSTATE_EXCLUDEDEPS_SYSROOT += "\
> +    .*->autoconf-native \
> +    .*->automake-native \
> +    .*->bison-native \
> +    .*->meson-native \
> +    .*->ninja-native \
> +    .*->patch-native \
> +    .*->pkgconfig-native \
> +    .*->quilt-native \
> +"
>
> I've marked this one as an RFC since it caused a fair bit of pain even in OE-
> core due to recipes having pkgconfig class inherits missing.
>
> We do really want to do this and correctly mark up the recipe dependencies, the
> question is whether it is too late for 3.4 or not. We could go for a smaller
> list for now too but I wanted to hear opinions.
>
>
> It looks like a great change! I'd say it's too late for 3.4 and we should do it for 3.5 instead.
>

right see the failed recipes which would need fixing
https://errors.yoctoproject.org/Errors/Build/130779/

> Cheers,
>
> Richard
>
>
>
>
>
> 
>

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Martin Jansa]

[-- Attachment #1: Type: text/plain, Size: 4278 bytes --]

FWIW: I've fixed some bigger layers where pkgconfig was causing quite a few
build failures,

e.g. meta-webosose needed it in around 30 recipes (included in
https://github.com/shr-project/meta-webosose/commits/dunfell)
meta-ros layers I'm still working on, sofar over 50 recipes (
https://github.com/ros/meta-ros/pull/939)

One interesting case I wanted to mention here was collada-dom in meta-ros:
https://github.com/ros/meta-ros/pull/939/commits/4e790a7de5e740eae3084a925a8a23f575ddf410

including this layer.conf change caused collada-dom not to fail with
reasonable error about missing pkgconfig-native, but triggered textrel QA
issue and it was actually caused by pkgconfig not being available where
do_configure "silently" failed to find system bzip2 and minizip due to
missing pkgconfig and built own version instead which lead to this textrel
QA issue.

There might be many more recipes like this, which build something
differently without pkgconfig available, without triggering any QA check
like in this case. It might be useful to compare at least buildhistory
results with and without the layer.conf change to see what else was changed
unexpectedly.

Cheers,

On Tue, Sep 21, 2021 at 6:21 AM Khem Raj <raj.khem@gmail.com> wrote:

> On Mon, Sep 20, 2021 at 6:34 AM Joshua Watt <JPEWhacker@gmail.com> wrote:
> >
> >
> > On 9/20/21 7:48 AM, Richard Purdie wrote:
> >
> > On Mon, 2021-09-20 at 13:46 +0100, Richard Purdie via
> lists.openembedded.org
> > wrote:
> >
> > There are several dependencies which recipes have which are not needed
> > at runtime, only at build time. Extend the list of these from bison to
> > include quilt, patch, meson, autoconf, automake and ninja which should
> > reduce the amount of data being included in native sysroots.
> >
> > This speeds up and reduces the sstate needed for image builds for
> example.
> >
> > If this change breaks recipes, it probably means they're missing an
> explicit
> > DEPENDS on something in this list which is a bug in the recipe.
> >
> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > ---
> >  meta/conf/layer.conf | 12 +++++++++++-
> >  1 file changed, 11 insertions(+), 1 deletion(-)
> >
> > diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
> > index 307ad7e2237..b3cc8a249e9 100644
> > --- a/meta/conf/layer.conf
> > +++ b/meta/conf/layer.conf
> > @@ -48,6 +48,7 @@ SIGGEN_EXCLUDERECIPES_ABISAFE += " \
> >  "
> >
> >  SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
> > +  *->patch-native \
> >    *->quilt-native \
> >    *->subversion-native \
> >    *->git-native \
> > @@ -90,7 +91,16 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
> >  # (e.g. X -> Y -> binutils-cross -> bison-native) no longer meet the
> >  # dependency incidentally. This improves determinism and avoids build
> >  # failures when people switch to external toolchains.
> > -SSTATE_EXCLUDEDEPS_SYSROOT += ".*->bison-native"
> > +SSTATE_EXCLUDEDEPS_SYSROOT += "\
> > +    .*->autoconf-native \
> > +    .*->automake-native \
> > +    .*->bison-native \
> > +    .*->meson-native \
> > +    .*->ninja-native \
> > +    .*->patch-native \
> > +    .*->pkgconfig-native \
> > +    .*->quilt-native \
> > +"
> >
> > I've marked this one as an RFC since it caused a fair bit of pain even
> in OE-
> > core due to recipes having pkgconfig class inherits missing.
> >
> > We do really want to do this and correctly mark up the recipe
> dependencies, the
> > question is whether it is too late for 3.4 or not. We could go for a
> smaller
> > list for now too but I wanted to hear opinions.
> >
> >
> > It looks like a great change! I'd say it's too late for 3.4 and we
> should do it for 3.5 instead.
> >
>
> right see the failed recipes which would need fixing
> https://errors.yoctoproject.org/Errors/Build/130779/
>
> > Cheers,
> >
> > Richard
> >
> >
> >
> >
> >
> >
> >
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#156200):
> https://lists.openembedded.org/g/openembedded-core/message/156200
> Mute This Topic: https://lists.openembedded.org/mt/85739636/3617156
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [
> Martin.Jansa@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
>

[-- Attachment #2: Type: text/html, Size: 6311 bytes --]

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Andreas Müller]

On Fri, Oct 1, 2021 at 4:17 PM Martin Jansa <Martin.Jansa@gmail.com> wrote:
>
> FWIW: I've fixed some bigger layers where pkgconfig was causing quite a few build failures,
>
Had an off-oe time due to  heavy work load.
Think this one is the reason for the pkgconfig patch flood. You are
aware that there are packages not failing at build time for missing
pkgconfig but build output is different and bugs will pop up at
runtime.

To avoid responsible maintainers have to go through EVERY recipe (and
combination of PACKAGECONFIGs) and check if builds are still as
expected - Can't believe it!

Andreas

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Martin Jansa]

[-- Attachment #1: Type: text/plain, Size: 1993 bytes --]

On Mon, Oct 18, 2021 at 1:50 AM Andreas Müller <schnitzeltony@gmail.com>
wrote:

> On Fri, Oct 1, 2021 at 4:17 PM Martin Jansa <Martin.Jansa@gmail.com>
> wrote:
> >
> > FWIW: I've fixed some bigger layers where pkgconfig was causing quite a
> few build failures,
> >
> Had an off-oe time due to  heavy work load.
> Think this one is the reason for the pkgconfig patch flood. You are
> aware that there are packages not failing at build time for missing
> pkgconfig but build output is different and bugs will pop up at
> runtime.
>
> To avoid responsible maintainers have to go through EVERY recipe (and
> combination of PACKAGECONFIGs) and check if builds are still as
> expected - Can't believe it!
>

Not sure why this seems targeted to me.

Yes I've tried to fix whatever issues this change caused in my world builds
and I'm partially responsible for pkgconfig patch flood, because I wanted
layers like meta-ros/*, meta-qt5, meta-qt6, meta-webosose/* to stay
compatible with oe-core whenever this change from RP is merged to master.

I've also tried to compare buildhistory and image content (for
ros-image-world with ros1-melodic) to see if I can spot more cases of
non-fatal build output changes as in collada-dom and unfortunately there is
a lot of noise in buildhistory (due to many recipes included in this image
not really build-reproducible), so I've used
mostly installed-package-sizes.txt file to spot 2 more recipes where the
size was different (but both don't seem to be caused by pkgconfig-native in
the end).

I did both builds from scratch without sstate, so I would assume that it
will catch all possible cases, but surprisingly just yesterday I've caught
another missing pkgconfig-native in one of meta-ros1-melodic recipes
(caught in kirkstone based build - while before I was testing on top of a
bit older honister with this change cherry-picked just for test - to
prepare recipes to be future-proof for kirkstone).

Regards,

[-- Attachment #2: Type: text/html, Size: 2477 bytes --]

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Richard Purdie]

On Mon, 2021-10-18 at 16:12 +0200, Martin Jansa wrote:
> On Mon, Oct 18, 2021 at 1:50 AM Andreas Müller <schnitzeltony@gmail.com>
> wrote:
> > On Fri, Oct 1, 2021 at 4:17 PM Martin Jansa <Martin.Jansa@gmail.com> wrote:
> > > 
> > > FWIW: I've fixed some bigger layers where pkgconfig was causing quite a
> > > few
> > build failures,
> > > 
> > Had an off-oe time due to  heavy work load.
> > Think this one is the reason for the pkgconfig patch flood. You are
> > aware that there are packages not failing at build time for missing
> > pkgconfig but build output is different and bugs will pop up at
> > runtime.
> > 
> > To avoid responsible maintainers have to go through EVERY recipe (and
> > combination of PACKAGECONFIGs) and check if builds are still as
> > expected - Can't believe it!
> > 
> 
> 
> Not sure why this seems targeted to me.
> 
> Yes I've tried to fix whatever issues this change caused in my world builds
> and I'm partially responsible for pkgconfig patch flood, because I wanted
> layers like meta-ros/*, meta-qt5, meta-qt6, meta-webosose/* to stay compatible
> with oe-core whenever this change from RP is merged to master.
> 
> I've also tried to compare buildhistory and image content (for ros-image-world
> with ros1-melodic) to see if I can spot more cases of non-fatal build output
> changes as in collada-dom and unfortunately there is a lot of noise in
> buildhistory (due to many recipes included in this image not really build-
> reproducible), so I've used mostly installed-package-sizes.txt file to spot 2
> more recipes where the size was different (but both don't seem to be caused by
> pkgconfig-native in the end).
> 
> I did both builds from scratch without sstate, so I would assume that it will
> catch all possible cases, but surprisingly just yesterday I've caught another
> missing pkgconfig-native in one of meta-ros1-melodic recipes (caught in
> kirkstone based build - while before I was testing on top of a bit older
> honister with this change cherry-picked just for test - to prepare recipes to
> be future-proof for kirkstone).

I take responsibility for merging the pkgconfig-native change in behaviour. I
did wait until after honister and only merged to master.

It does highlight a real world problem with recipe dependencies since if some of
these recipes are only indirectly depending upon it, changes in those other
recipes could silently break the recipe in a similar way to the gperf breakage
we just saw (which thankfully shows an error). I therefore do think this is
worth fixing.

Cheers,

Richard

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Andreas Müller]

On Mon, Oct 18, 2021 at 4:12 PM Martin Jansa <martin.jansa@gmail.com> wrote:
>
> On Mon, Oct 18, 2021 at 1:50 AM Andreas Müller <schnitzeltony@gmail.com> wrote:
>>
>> On Fri, Oct 1, 2021 at 4:17 PM Martin Jansa <Martin.Jansa@gmail.com> wrote:
>> >
>> > FWIW: I've fixed some bigger layers where pkgconfig was causing quite a few build failures,
>> >
>> Had an off-oe time due to  heavy work load.
>> Think this one is the reason for the pkgconfig patch flood. You are
>> aware that there are packages not failing at build time for missing
>> pkgconfig but build output is different and bugs will pop up at
>> runtime.
>>
>> To avoid responsible maintainers have to go through EVERY recipe (and
>> combination of PACKAGECONFIGs) and check if builds are still as
>> expected - Can't believe it!
>
>
> Not sure why this seems targeted to me.
>
> Yes I've tried to fix whatever issues this change caused in my world builds and I'm partially responsible for pkgconfig patch flood, because I wanted layers like meta-ros/*, meta-qt5, meta-qt6, meta-webosose/* to stay compatible with oe-core whenever this change from RP is merged to master.
I am not against the flood of pkconfig patches I am against this patch.
>
> I've also tried to compare buildhistory and image content (for ros-image-world with ros1-melodic) to see if I can spot more cases of non-fatal build output changes as in collada-dom and unfortunately there is a lot of noise in buildhistory (due to many recipes included in this image not really build-reproducible), so I've used mostly installed-package-sizes.txt file to spot 2 more recipes where the size was different (but both don't seem to be caused by pkgconfig-native in the end).
>
> I did both builds from scratch without sstate, so I would assume that it will catch all possible cases, but surprisingly just yesterday I've caught another missing pkgconfig-native in one of meta-ros1-melodic recipes (caught in kirkstone based build - while before I was testing on top of a bit older honister with this change cherry-picked just for test - to prepare recipes to be future-proof for kirkstone).
>
This was huge effort - and it sounds there is still no 100% confidence
to get same you had before - right? Others have to do same.

And for what? A build acceleration of how much - this patch does not
mention numbers. Or is it: Only the perfect recipes are allowed to
survive.

To me oe-core has decided (again) to make changes without taking care:
Break dozens of layers and force maintainers to spend days to get what
they had before this change. From image perspective nothing has
changed. And the very best: Some layers were just made for fun and
since this is definitely no fun they are going to die.

After this I have to work on my motivation spending time on this
project - not an easy task.

My opinion

Andreas
> Regards,

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Andreas Müller]

On Mon, Oct 18, 2021 at 4:12 PM Martin Jansa <martin.jansa@gmail.com> wrote:
>
> On Mon, Oct 18, 2021 at 1:50 AM Andreas Müller <schnitzeltony@gmail.com> wrote:
>>
>> On Fri, Oct 1, 2021 at 4:17 PM Martin Jansa <Martin.Jansa@gmail.com> wrote:
>> >
>> > FWIW: I've fixed some bigger layers where pkgconfig was causing quite a few build failures,
>> >
>> Had an off-oe time due to  heavy work load.
>> Think this one is the reason for the pkgconfig patch flood. You are
>> aware that there are packages not failing at build time for missing
>> pkgconfig but build output is different and bugs will pop up at
>> runtime.
>>
>> To avoid responsible maintainers have to go through EVERY recipe (and
>> combination of PACKAGECONFIGs) and check if builds are still as
>> expected - Can't believe it!
>
>
> Not sure why this seems targeted to me.
It was not targeted to you - just responded all in the last post - sorry

Andreas
>
> Yes I've tried to fix whatever issues this change caused in my world builds and I'm partially responsible for pkgconfig patch flood, because I wanted layers like meta-ros/*, meta-qt5, meta-qt6, meta-webosose/* to stay compatible with oe-core whenever this change from RP is merged to master.
>
> I've also tried to compare buildhistory and image content (for ros-image-world with ros1-melodic) to see if I can spot more cases of non-fatal build output changes as in collada-dom and unfortunately there is a lot of noise in buildhistory (due to many recipes included in this image not really build-reproducible), so I've used mostly installed-package-sizes.txt file to spot 2 more recipes where the size was different (but both don't seem to be caused by pkgconfig-native in the end).
>
> I did both builds from scratch without sstate, so I would assume that it will catch all possible cases, but surprisingly just yesterday I've caught another missing pkgconfig-native in one of meta-ros1-melodic recipes (caught in kirkstone based build - while before I was testing on top of a bit older honister with this change cherry-picked just for test - to prepare recipes to be future-proof for kirkstone).
>
> Regards,

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Konrad Weihmann]

Just as an idea: can those knowingly breaking changes be announced 
upfront somehow? maybe in the architecture list?!? (and then applied on 
a flag day)

I know we all got a bit of a high workload, so it might be better to 
have breaking changes not hit "uncontrolled" into master.

I agree that master is a development branch, but the number of breaking 
changes that hit me by surprise this year is astonishingly high.

That doesn't mean, that I don't appreciate the actual change made, but 
that way it has been applied (and the breaking changes before) makes me 
rethink the way I do integrate my non-core layer in the future, while I 
personally would like to remain on this bleeding-edge model that worked 
well for a long time.

On 18.10.21 18:59, Andreas Müller wrote:
> On Mon, Oct 18, 2021 at 4:12 PM Martin Jansa <martin.jansa@gmail.com> wrote:
>>
>> On Mon, Oct 18, 2021 at 1:50 AM Andreas Müller <schnitzeltony@gmail.com> wrote:
>>>
>>> On Fri, Oct 1, 2021 at 4:17 PM Martin Jansa <Martin.Jansa@gmail.com> wrote:
>>>>
>>>> FWIW: I've fixed some bigger layers where pkgconfig was causing quite a few build failures,
>>>>
>>> Had an off-oe time due to  heavy work load.
>>> Think this one is the reason for the pkgconfig patch flood. You are
>>> aware that there are packages not failing at build time for missing
>>> pkgconfig but build output is different and bugs will pop up at
>>> runtime.
>>>
>>> To avoid responsible maintainers have to go through EVERY recipe (and
>>> combination of PACKAGECONFIGs) and check if builds are still as
>>> expected - Can't believe it!
>>
>>
>> Not sure why this seems targeted to me.
> It was not targeted to you - just responded all in the last post - sorry
> 
> Andreas
>>
>> Yes I've tried to fix whatever issues this change caused in my world builds and I'm partially responsible for pkgconfig patch flood, because I wanted layers like meta-ros/*, meta-qt5, meta-qt6, meta-webosose/* to stay compatible with oe-core whenever this change from RP is merged to master.
>>
>> I've also tried to compare buildhistory and image content (for ros-image-world with ros1-melodic) to see if I can spot more cases of non-fatal build output changes as in collada-dom and unfortunately there is a lot of noise in buildhistory (due to many recipes included in this image not really build-reproducible), so I've used mostly installed-package-sizes.txt file to spot 2 more recipes where the size was different (but both don't seem to be caused by pkgconfig-native in the end).
>>
>> I did both builds from scratch without sstate, so I would assume that it will catch all possible cases, but surprisingly just yesterday I've caught another missing pkgconfig-native in one of meta-ros1-melodic recipes (caught in kirkstone based build - while before I was testing on top of a bit older honister with this change cherry-picked just for test - to prepare recipes to be future-proof for kirkstone).
>>
>> Regards,
>>
>>
>> -=-=-=-=-=-=-=-=-=-=-=-
>> Links: You receive all messages sent to this group.
>> View/Reply Online (#157087): https://lists.openembedded.org/g/openembedded-core/message/157087
>> Mute This Topic: https://lists.openembedded.org/mt/85739636/3647476
>> Group Owner: openembedded-core+owner@lists.openembedded.org
>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [kweihmann@outlook.com]
>> -=-=-=-=-=-=-=-=-=-=-=-
>>

Re: [OE-core] [RFC PATCH 14/14] layer.conf: Extend recipes not to install without explict dependencies

[Richard Purdie]

On Mon, 2021-10-18 at 21:07 +0200, Konrad Weihmann wrote:
> Just as an idea: can those knowingly breaking changes be announced 
> upfront somehow? maybe in the architecture list?!? (and then applied on 
> a flag day)

This one was announced on the mailing list in patch form, in the weekly tech
calls and I think in the weekly status emails too. It was announced enough that
I had feedback saying "not in the release". I listened to the feedback and
deferred it until we started merging new development for the next release. So it
was actually pending for a while.

> I know we all got a bit of a high workload, so it might be better to 
> have breaking changes not hit "uncontrolled" into master.
> 
> I agree that master is a development branch, but the number of breaking 
> changes that hit me by surprise this year is astonishingly high.

I think the problem is that for a period, active development of these kinds of
changes stagnated. That is good and bad, it gave stability but it also meant we
didn't tackle and fix some of the hard issues we need to fix for the project to
move forward. 

For the record the bigger changes were discussed on the architecture list so
those shouldn't have surprised anyone yet they still did :(.

> That doesn't mean, that I don't appreciate the actual change made, but 
> that way it has been applied (and the breaking changes before) makes me 
> rethink the way I do integrate my non-core layer in the future, while I 
> personally would like to remain on this bleeding-edge model that worked 
> well for a long time.

I think the question is more how long these things are breaking for. If things
were broken and then spent weeks languishing in that state, there would be cause
for concern. I'm not aware of that happening and most things do seem to be
getting resolved quickly. Those that aren't are a warning sign as it means
they're not being actively maintained and you need to ask other questions in
that case.

There is a balance here between being able to move the project forward and
drowning under the shear weight of "before changing X you need to write a
proposal, wait X weeks, get TSC approval, ensure it was in X weekly status
reports, talked about in X weekly meetings" and so on. I actually keep asking
myself, "why do I bother trying to change anything as I generally just get
complaints about the work it causes people?". I get really depressed at the
complaints I see and if I feel like this, it must be a real barrier to others
too and it will stop things changing. I don't go out with the intent of making
things difficult, I actually want to see the project move forward technically
not stagnate which I think it is at serious risk of. Do we really want to put
high barriers in place to development?

I will raise this with the TSC as a concern so they can review things.

Cheers,

Richard

Re: [OE-core] [RFC PATCH 11/14] populate_sdk_base/images: Drop use of 'meta' class and hence do_build dependencies

[Richard Purdie]

On Mon, 2021-09-20 at 13:46 +0100, Richard Purdie via lists.openembedded.org
wrote:
> The 'meta' is old and not very useful. It empties PACKAGES and creates
> recursive do_build dependencies.
> 
> We shouldn't need such recursive build dependencies any more so simplify the code.
> 
> This does cause behaviour changes as some dependencies are no longer built.
> It did show up issues with qemu-helper-native handling for example but those
> issues look like real races and the underlying dependency chains were not correct.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/classes/populate_sdk_base.bbclass | 4 +++-
>  1 file changed, 3 insertions(+), 1 deletion(-)
> 
> diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
> index ccfe2232895..49e166e697d 100644
> --- a/meta/classes/populate_sdk_base.bbclass
> +++ b/meta/classes/populate_sdk_base.bbclass
> @@ -1,4 +1,6 @@
> -inherit meta image-postinst-intercepts image-artifact-names
> +PACKAGES = ""
> +
> +inherit image-postinst-intercepts image-artifact-names
>  
>  # Wildcards specifying complementary packages to install for every package that has been explicitly
>  # installed into the rootfs

This change is probably high impact in that it changes behaviour the most of any
of the changes. It does make image builds use a lot less sstate and in theory,
if there are issues with failures from this, there are probably existing races.

It does also have CI impact as it changes the things which are built as not all
build tasks now run. Open to opinions on whether we should do this now or not.

The meta class is pretty useless and should be removed regardless.

Cheers,

Richard

Re: [OE-core] [RFC PATCH 11/14] populate_sdk_base/images: Drop use of 'meta' class and hence do_build dependencies

[Khem Raj]

On 9/20/21 5:51 AM, Richard Purdie wrote:
> On Mon, 2021-09-20 at 13:46 +0100, Richard Purdie via lists.openembedded.org
> wrote:
>> The 'meta' is old and not very useful. It empties PACKAGES and creates
>> recursive do_build dependencies.
>>
>> We shouldn't need such recursive build dependencies any more so simplify the code.
>>
>> This does cause behaviour changes as some dependencies are no longer built.
>> It did show up issues with qemu-helper-native handling for example but those
>> issues look like real races and the underlying dependency chains were not correct.
>>
>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>> ---
>>   meta/classes/populate_sdk_base.bbclass | 4 +++-
>>   1 file changed, 3 insertions(+), 1 deletion(-)
>>
>> diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
>> index ccfe2232895..49e166e697d 100644
>> --- a/meta/classes/populate_sdk_base.bbclass
>> +++ b/meta/classes/populate_sdk_base.bbclass
>> @@ -1,4 +1,6 @@
>> -inherit meta image-postinst-intercepts image-artifact-names
>> +PACKAGES = ""
>> +
>> +inherit image-postinst-intercepts image-artifact-names
>>   
>>   # Wildcards specifying complementary packages to install for every package that has been explicitly
>>   # installed into the rootfs
> 
> This change is probably high impact in that it changes behaviour the most of any
> of the changes. It does make image builds use a lot less sstate and in theory,
> if there are issues with failures from this, there are probably existing races.
> 
> It does also have CI impact as it changes the things which are built as not all
> build tasks now run. Open to opinions on whether we should do this now or not.
> 


I definitely like this change and want it in upcoming LTS in 2022, not 
comfortable with rushing it into this release. It will cause fair bit of 
work for rest of layers in ecosystem.


> The meta class is pretty useless and should be removed regardless.
> 

yes

> Cheers,
> 
> Richard
> 
> 
> 
> 
> 

Re: [OE-core] [RFC PATCH 11/14] populate_sdk_base/images: Drop use of 'meta' class and hence do_build dependencies

[Richard Purdie]

On Mon, 2021-09-20 at 09:32 -0700, Khem Raj wrote:
> 
> On 9/20/21 5:51 AM, Richard Purdie wrote:
> > On Mon, 2021-09-20 at 13:46 +0100, Richard Purdie via lists.openembedded.org
> > wrote:
> > > The 'meta' is old and not very useful. It empties PACKAGES and creates
> > > recursive do_build dependencies.
> > > 
> > > We shouldn't need such recursive build dependencies any more so simplify the code.
> > > 
> > > This does cause behaviour changes as some dependencies are no longer built.
> > > It did show up issues with qemu-helper-native handling for example but those
> > > issues look like real races and the underlying dependency chains were not correct.
> > > 
> > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > > ---
> > >   meta/classes/populate_sdk_base.bbclass | 4 +++-
> > >   1 file changed, 3 insertions(+), 1 deletion(-)
> > > 
> > > diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
> > > index ccfe2232895..49e166e697d 100644
> > > --- a/meta/classes/populate_sdk_base.bbclass
> > > +++ b/meta/classes/populate_sdk_base.bbclass
> > > @@ -1,4 +1,6 @@
> > > -inherit meta image-postinst-intercepts image-artifact-names
> > > +PACKAGES = ""
> > > +
> > > +inherit image-postinst-intercepts image-artifact-names
> > >   
> > >   # Wildcards specifying complementary packages to install for every package that has been explicitly
> > >   # installed into the rootfs
> > 
> > This change is probably high impact in that it changes behaviour the most of any
> > of the changes. It does make image builds use a lot less sstate and in theory,
> > if there are issues with failures from this, there are probably existing races.
> > 
> > It does also have CI impact as it changes the things which are built as not all
> > build tasks now run. Open to opinions on whether we should do this now or not.
> > 
>
> I definitely like this change and want it in upcoming LTS in 2022, not 
> comfortable with rushing it into this release. It will cause fair bit of 
> work for rest of layers in ecosystem.

The question is whether this is a real world issue for anyone with CI out there.
I've been looking at the YP autobuilder and I'm not sure it is. Yes, it does
change things but not really in a way I'm too worried about.

There is some breakage in this series but it is mainly from the layer.conf
change.

So the question is whether anyone does see real issues in their CI from this?

There is an easy change if it does cause an issue which I've now documented in
the commit log as I replied to Joshua.

Cheers,

Richard