[Buildroot] [PATCH] package/systemd: needs glibc

[Buildroot] [PATCH] package/systemd: needs glibc

[Yann E. MORIN]

Since version v239, systemd-nspawn unconditioanlly uses prlimit(2),
which is not implemented in uClibc-ng. systemd-nspawn can not be
disabled.

This makes systemd glibc-only again.

After a bit of discussion with upstream (om IRC), it looks very
improbable that they accept a patch making systemd-nspawn optional.
They would probably consider a patch that provides that syscall wrapper
if it is missing, though, but that's less trivial...

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Waldemar Brodkorb <wbx@openadk.org>
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: Peter Korsgaard <peter@korsgaard.com>
---
 package/systemd/Config.in  |  3 ---
 package/systemd/systemd.mk | 11 +----------
 system/Config.in           | 11 +++--------
 3 files changed, 4 insertions(+), 21 deletions(-)

diff --git a/package/systemd/Config.in b/package/systemd/Config.in
index c858d89c05..25f322e8f3 100644
--- a/package/systemd/Config.in
+++ b/package/systemd/Config.in
@@ -192,7 +192,6 @@ config BR2_PACKAGE_SYSTEMD_MACHINED
 config BR2_PACKAGE_SYSTEMD_MYHOSTNAME
 	bool "enable myhostname NSS plugin"
 	default y
-	depends on !BR2_TOOLCHAIN_USES_UCLIBC # needs nss.h
 	help
 	  nss-myhostname is a plug-in module for the GNU Name Service
 	  Switch (NSS) functionality of the GNU C Library (glibc),
@@ -252,7 +251,6 @@ config BR2_PACKAGE_SYSTEMD_RANDOMSEED
 config BR2_PACKAGE_SYSTEMD_RESOLVED
 	bool "enable resolve daemon"
 	default y
-	depends on !BR2_TOOLCHAIN_USES_UCLIBC # needs nss.h
 	help
 	  systemd-resolved is a system service that provides network
 	  name resolution to local applications. It implements a
@@ -285,7 +283,6 @@ config BR2_PACKAGE_SYSTEMD_SMACK_SUPPORT
 
 config BR2_PACKAGE_SYSTEMD_SYSUSERS
 	bool "enable sysusers support"
-	depends on !BR2_TOOLCHAIN_USES_UCLIBC # needs gshadow.h
 	help
 	  systemd-sysusers creates system users and groups, based on
 	  the file format and location specified in sysusers.d(5).
diff --git a/package/systemd/systemd.mk b/package/systemd/systemd.mk
index ecbf7e97ff..e53f0b699a 100644
--- a/package/systemd/systemd.mk
+++ b/package/systemd/systemd.mk
@@ -40,18 +40,9 @@ SYSTEMD_CONF_OPTS += \
 	-Dsulogin-path=/usr/sbin/sulogin \
 	-Dmount-path=/usr/bin/mount \
 	-Dumount-path=/usr/bin/umount \
-	-Dnobody-group=nogroup
-
-# disable unsupported features for non-glibc toolchains
-ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),y)
-SYSTEMD_CONF_OPTS += \
+	-Dnobody-group=nogroup \
 	-Didn=true \
 	-Dnss-systemd=true
-else
-SYSTEMD_CONF_OPTS += \
-	-Didn=false \
-	-Dnss-systemd=false
-endif
 
 ifeq ($(BR2_PACKAGE_ACL),y)
 SYSTEMD_DEPENDENCIES += acl
diff --git a/system/Config.in b/system/Config.in
index f63f27b126..9e34f11b6c 100644
--- a/system/Config.in
+++ b/system/Config.in
@@ -127,9 +127,7 @@ config BR2_INIT_SYSV
 config BR2_INIT_SYSTEMD
 	bool "systemd"
 	depends on BR2_PACKAGE_SYSTEMD_ARCH_SUPPORTS
-	depends on BR2_TOOLCHAIN_USES_GLIBC || BR2_TOOLCHAIN_USES_UCLIBC
-	depends on BR2_USE_WCHAR
-	depends on BR2_TOOLCHAIN_HAS_THREADS
+	depends on BR2_TOOLCHAIN_USES_GLIBC
 	depends on BR2_TOOLCHAIN_HAS_SSP
 	depends on BR2_USE_MMU
 	depends on !BR2_STATIC_LIBS
@@ -137,14 +135,11 @@ config BR2_INIT_SYSTEMD
 	select BR2_ROOTFS_MERGED_USR
 	select BR2_PACKAGE_SYSTEMD
 
-comment "systemd needs a glibc or uClibc toolchain w/ wchar, threads, SSP, dynamic library, headers >= 3.10"
+comment "systemd needs a glibc toolchain w/ SSP, headers >= 3.10"
 	depends on BR2_PACKAGE_SYSTEMD_ARCH_SUPPORTS
 	depends on BR2_USE_MMU
-	depends on !(BR2_TOOLCHAIN_USES_GLIBC || BR2_TOOLCHAIN_USES_UCLIBC) || \
-		!BR2_USE_WCHAR || \
-		!BR2_TOOLCHAIN_HAS_THREADS || \
+	depends on !BR2_TOOLCHAIN_USES_GLIBC || \
 		!BR2_TOOLCHAIN_HAS_SSP || \
-		BR2_STATIC_LIBS || \
 		!BR2_TOOLCHAIN_HEADERS_AT_LEAST_3_10
 
 config BR2_INIT_NONE
-- 
2.14.1

[Buildroot] [PATCH] package/systemd: needs glibc

[Matthew Weber]

Yann,

On Wed, Nov 21, 2018 at 3:06 PM Yann E. MORIN <yann.morin.1998@free.fr> wrote:
>
> Since version v239, systemd-nspawn unconditioanlly uses prlimit(2),
> which is not implemented in uClibc-ng. systemd-nspawn can not be
> disabled.
>
> This makes systemd glibc-only again.
>
> After a bit of discussion with upstream (om IRC), it looks very
> improbable that they accept a patch making systemd-nspawn optional.
> They would probably consider a patch that provides that syscall wrapper
> if it is missing, though, but that's less trivial...
>
> Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
> Cc: Waldemar Brodkorb <wbx@openadk.org>
> Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
> Cc: Peter Korsgaard <peter@korsgaard.com>

Reviewed-by: Matt Weber <matthew.weber@rockwellcollins.com>
(Went back and reviewed Waldemar's original patch and this revert)

> ---
>  package/systemd/Config.in  |  3 ---
>  package/systemd/systemd.mk | 11 +----------
>  system/Config.in           | 11 +++--------
>  3 files changed, 4 insertions(+), 21 deletions(-)
>
> diff --git a/package/systemd/Config.in b/package/systemd/Config.in
> index c858d89c05..25f322e8f3 100644
> --- a/package/systemd/Config.in
> +++ b/package/systemd/Config.in
> @@ -192,7 +192,6 @@ config BR2_PACKAGE_SYSTEMD_MACHINED
>  config BR2_PACKAGE_SYSTEMD_MYHOSTNAME
>         bool "enable myhostname NSS plugin"
>         default y
> -       depends on !BR2_TOOLCHAIN_USES_UCLIBC # needs nss.h
>         help
>           nss-myhostname is a plug-in module for the GNU Name Service
>           Switch (NSS) functionality of the GNU C Library (glibc),
> @@ -252,7 +251,6 @@ config BR2_PACKAGE_SYSTEMD_RANDOMSEED
>  config BR2_PACKAGE_SYSTEMD_RESOLVED
>         bool "enable resolve daemon"
>         default y
> -       depends on !BR2_TOOLCHAIN_USES_UCLIBC # needs nss.h
>         help
>           systemd-resolved is a system service that provides network
>           name resolution to local applications. It implements a
> @@ -285,7 +283,6 @@ config BR2_PACKAGE_SYSTEMD_SMACK_SUPPORT
>
>  config BR2_PACKAGE_SYSTEMD_SYSUSERS
>         bool "enable sysusers support"
> -       depends on !BR2_TOOLCHAIN_USES_UCLIBC # needs gshadow.h
>         help
>           systemd-sysusers creates system users and groups, based on
>           the file format and location specified in sysusers.d(5).
> diff --git a/package/systemd/systemd.mk b/package/systemd/systemd.mk
> index ecbf7e97ff..e53f0b699a 100644
> --- a/package/systemd/systemd.mk
> +++ b/package/systemd/systemd.mk
> @@ -40,18 +40,9 @@ SYSTEMD_CONF_OPTS += \
>         -Dsulogin-path=/usr/sbin/sulogin \
>         -Dmount-path=/usr/bin/mount \
>         -Dumount-path=/usr/bin/umount \
> -       -Dnobody-group=nogroup
> -
> -# disable unsupported features for non-glibc toolchains
> -ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),y)
> -SYSTEMD_CONF_OPTS += \
> +       -Dnobody-group=nogroup \
>         -Didn=true \
>         -Dnss-systemd=true
> -else
> -SYSTEMD_CONF_OPTS += \
> -       -Didn=false \
> -       -Dnss-systemd=false
> -endif
>
>  ifeq ($(BR2_PACKAGE_ACL),y)
>  SYSTEMD_DEPENDENCIES += acl
> diff --git a/system/Config.in b/system/Config.in
> index f63f27b126..9e34f11b6c 100644
> --- a/system/Config.in
> +++ b/system/Config.in
> @@ -127,9 +127,7 @@ config BR2_INIT_SYSV
>  config BR2_INIT_SYSTEMD
>         bool "systemd"
>         depends on BR2_PACKAGE_SYSTEMD_ARCH_SUPPORTS
> -       depends on BR2_TOOLCHAIN_USES_GLIBC || BR2_TOOLCHAIN_USES_UCLIBC
> -       depends on BR2_USE_WCHAR
> -       depends on BR2_TOOLCHAIN_HAS_THREADS
> +       depends on BR2_TOOLCHAIN_USES_GLIBC
>         depends on BR2_TOOLCHAIN_HAS_SSP
>         depends on BR2_USE_MMU
>         depends on !BR2_STATIC_LIBS
> @@ -137,14 +135,11 @@ config BR2_INIT_SYSTEMD
>         select BR2_ROOTFS_MERGED_USR
>         select BR2_PACKAGE_SYSTEMD
>
> -comment "systemd needs a glibc or uClibc toolchain w/ wchar, threads, SSP, dynamic library, headers >= 3.10"
> +comment "systemd needs a glibc toolchain w/ SSP, headers >= 3.10"
>         depends on BR2_PACKAGE_SYSTEMD_ARCH_SUPPORTS
>         depends on BR2_USE_MMU
> -       depends on !(BR2_TOOLCHAIN_USES_GLIBC || BR2_TOOLCHAIN_USES_UCLIBC) || \
> -               !BR2_USE_WCHAR || \
> -               !BR2_TOOLCHAIN_HAS_THREADS || \
> +       depends on !BR2_TOOLCHAIN_USES_GLIBC || \
>                 !BR2_TOOLCHAIN_HAS_SSP || \
> -               BR2_STATIC_LIBS || \
>                 !BR2_TOOLCHAIN_HEADERS_AT_LEAST_3_10
>
>  config BR2_INIT_NONE
> --
> 2.14.1
>
> _______________________________________________
> buildroot mailing list
> buildroot at busybox.net
> http://lists.busybox.net/mailman/listinfo/buildroot



-- 
Matthew L Weber / Pr Software Engineer
Airborne Information Systems / RC Linux Secure Platforms
MS 131-100, C Ave NE, Cedar Rapids, IA, 52498, USA
www.rockwellcollins.com

Note: Any Export License Required Information and License Restricted
Third Party Intellectual Property (TPIP) content must be encrypted and
sent to matthew.weber at corp.rockwellcollins.com.

[Buildroot] [PATCH] package/systemd: needs glibc

[Peter Korsgaard]

>>>>> "Yann" == Yann E MORIN <yann.morin.1998@free.fr> writes:

 > Since version v239, systemd-nspawn unconditioanlly uses prlimit(2),
 > which is not implemented in uClibc-ng. systemd-nspawn can not be
 > disabled.

 > This makes systemd glibc-only again.

 > After a bit of discussion with upstream (om IRC), it looks very
 > improbable that they accept a patch making systemd-nspawn optional.
 > They would probably consider a patch that provides that syscall wrapper
 > if it is missing, though, but that's less trivial...

 > Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
 > Cc: Waldemar Brodkorb <wbx@openadk.org>
 > Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
 > Cc: Peter Korsgaard <peter@korsgaard.com>

So this has been broken since end of June? How come we don't have any
autobuilder failures?

Committed, thanks.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH] package/systemd: needs glibc

[Thomas Petazzoni]

Hello,

On Thu, 22 Nov 2018 17:19:43 +0100, Peter Korsgaard wrote:

>  > Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
>  > Cc: Waldemar Brodkorb <wbx@openadk.org>
>  > Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
>  > Cc: Peter Korsgaard <peter@korsgaard.com>  
> 
> So this has been broken since end of June? How come we don't have any
> autobuilder failures?

Yes, this is really weird. Even with the issue fixed
https://git.buildroot.org/buildroot/commit/utils/genrandconfig?id=3eb49f59d6804c452e784759d9b3869215a8098f,
we should have had some systemd/uclibc configurations. I'll try to do
some research in the DB, but my SQL is rusty and gives requests that
are very slow to execute :/

Best regards,

Thomas
-- 
Thomas Petazzoni, CTO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

[Buildroot] [PATCH] package/systemd: needs glibc

[Yann E. MORIN]

Peter, Thomas, All,

On 2018-11-22 17:44 +0100, Thomas Petazzoni spake thusly:
> On Thu, 22 Nov 2018 17:19:43 +0100, Peter Korsgaard wrote:
> 
> >  > Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
> >  > Cc: Waldemar Brodkorb <wbx@openadk.org>
> >  > Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
> >  > Cc: Peter Korsgaard <peter@korsgaard.com>  
> > 
> > So this has been broken since end of June? How come we don't have any
> > autobuilder failures?
> 
> Yes, this is really weird. Even with the issue fixed
> https://git.buildroot.org/buildroot/commit/utils/genrandconfig?id=3eb49f59d6804c452e784759d9b3869215a8098f,
> we should have had some systemd/uclibc configurations. I'll try to do
> some research in the DB, but my SQL is rusty and gives requests that
> are very slow to execute :/

Yeah, that's strange... But it sure as hell does not build at all.

Besides the prlimit() issue, it also unconditionally #include <gshadow.h>
which does not exist in uclibc-ng either. That one has been fixed upstream
now:
    https://github.com/systemd/systemd/commit/66a5b5ce9b991327d594f7b635de16999ca54093

(we don't need to backport it, now that we can't build on uClibc).

I may be looking into making a workaround by providing a missing.h
wrapper to prlimit() as they suggested, but that is definitely not
release material for us.

Regards,
Yann E. MORIN.

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 223 225 172 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'

[Buildroot] [PATCH] package/systemd: needs glibc

[Matthew Weber]

All,

On Thu, Nov 22, 2018 at 11:13 AM Yann E. MORIN <yann.morin.1998@free.fr> wrote:
>
> Peter, Thomas, All,
>
> On 2018-11-22 17:44 +0100, Thomas Petazzoni spake thusly:
> > On Thu, 22 Nov 2018 17:19:43 +0100, Peter Korsgaard wrote:
> >
> > >  > Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
> > >  > Cc: Waldemar Brodkorb <wbx@openadk.org>
> > >  > Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
> > >  > Cc: Peter Korsgaard <peter@korsgaard.com>
> > >
> > > So this has been broken since end of June? How come we don't have any
> > > autobuilder failures?
> >
> > Yes, this is really weird. Even with the issue fixed
> > https://git.buildroot.org/buildroot/commit/utils/genrandconfig?id=3eb49f59d6804c452e784759d9b3869215a8098f,
> > we should have had some systemd/uclibc configurations. I'll try to do
> > some research in the DB, but my SQL is rusty and gives requests that
> > are very slow to execute :/
>
> Yeah, that's strange... But it sure as hell does not build at all.
>
> Besides the prlimit() issue, it also unconditionally #include <gshadow.h>
> which does not exist in uclibc-ng either. That one has been fixed upstream
> now:
>     https://github.com/systemd/systemd/commit/66a5b5ce9b991327d594f7b635de16999ca54093
>
> (we don't need to backport it, now that we can't build on uClibc).
>
> I may be looking into making a workaround by providing a missing.h
> wrapper to prlimit() as they suggested, but that is definitely not
> release material for us.
>

It looks like the utils/genrandconfig is generating the right cfg
combinations for uclibc and systemd.  It did however look like the way
the random selection of init options +"BR2_STATIC_LIBS" selection
(systemd depends on !BR2_STATIC_LIBS),  may lower the number of builds
considerably.  (guessing ~0.15% chance of a build if my math is
correct --> 100% / 2 / 16 / 20)

    if randint(0, 1) == 0:
        configlines.append("BR2_INIT_BUSYBOX=y\n")
    elif randint(0, 15) == 0:
        configlines.append("BR2_INIT_SYSTEMD=y\n")
    elif randint(0, 10) == 0:
        configlines.append("BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y\n")
    if randint(0, 20) == 0:
        configlines.append("BR2_STATIC_LIBS=y\n")

Additionally, it looks like at least one (maybe more) of our prebuilt
uclibc toolchains don't include  BR2_USE_WCHAR which is required for
systemd.

Maybe we should move the conditionals for BR2_STATIC_LIBS and
BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV inside the busybox condition.
Plus toolchain updates for BR2_USE_WCHAR (if that matters now that
uclibc is gone).....

Matt

[Buildroot] [PATCH] package/systemd: needs glibc

[Thomas Petazzoni]

Hello,

On Thu, 22 Nov 2018 20:30:21 -0600, Matthew Weber wrote:

> It looks like the utils/genrandconfig is generating the right cfg
> combinations for uclibc and systemd.  It did however look like the way
> the random selection of init options +"BR2_STATIC_LIBS" selection
> (systemd depends on !BR2_STATIC_LIBS),  may lower the number of builds
> considerably.  (guessing ~0.15% chance of a build if my math is
> correct --> 100% / 2 / 16 / 20)

BR2_STATIC_LIBS is added only on one out of 20 configurations, so the
math here is not correct I believe: the last "divide by 20" would mean
that BR2_STATIC_LIBS disabled is one every 20 builds, but it's the
opposite.

Or is my math failing ?

> Additionally, it looks like at least one (maybe more) of our prebuilt
> uclibc toolchains don't include  BR2_USE_WCHAR which is required for
> systemd.

Most of them have BR2_USE_WCHAR.

> Maybe we should move the conditionals for BR2_STATIC_LIBS and
> BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV inside the busybox condition.
> Plus toolchain updates for BR2_USE_WCHAR (if that matters now that
> uclibc is gone).....

The BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV condition is already
mutually exclusive with BR2_INIT_SYSTEMD, due to the "elif". However,
it is also mutually exclusive with Busybox, which is a bit silly.

Best regards,

Thomas
-- 
Thomas Petazzoni, CTO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

[Buildroot] [PATCH] package/systemd: needs glibc

[Thomas Petazzoni]

Hello,

On Thu, 22 Nov 2018 17:44:17 +0100, Thomas Petazzoni wrote:

> Yes, this is really weird. Even with the issue fixed
> https://git.buildroot.org/buildroot/commit/utils/genrandconfig?id=3eb49f59d6804c452e784759d9b3869215a8098f,
> we should have had some systemd/uclibc configurations. I'll try to do
> some research in the DB, but my SQL is rusty and gives requests that
> are very slow to execute :/

No uclibc/systemd configuration tested apparently, at least since
September 1st:

mysql> select * from (select * from results where builddate > '2018-09-01') as foo inner join symbol_per_result on symbol_per_result.result_id=foo.id where symbol_per_result.symbol_id=(select id from config_symbol where name="BR2_INIT_SYSTEMD" and value="y");
+--------+--------+---------------------+---------------------------------------------+------------------------------------------+------------------------------------------+-------------+-------------------------------+-------+--------+------------+----------+-----------+----------+-----------+-----------+
| id     | status | builddate           | submitter                                   | commitid                                 | identifier                               | arch        | reason                        | libc  | static | subarch    | duration | branch    | id       | result_id | symbol_id |
+--------+--------+---------------------+---------------------------------------------+------------------------------------------+------------------------------------------+-------------+-------------------------------+-------+--------+------------+----------+-----------+----------+-----------+-----------+
|  87855 |      1 | 2018-09-19 01:03:03 | Romain Naour (Smile)                        | 0760e6ba8cae4fb9c646bd1be54857f48319df76 | 5b4a7a9ae7d4918ba7caab42ae9a3699e2509246 | mipsel      | Makefile:708: target-finalize | glibc |      0 | mips32r6   |    13255 | master    | 45730830 |     87855 |     26766 |
|  89347 |      0 | 2018-09-23 08:05:20 | Giulio Benetti (Micronova srl Server)       | f8f8ff33472641f9192a5e1ca65ebc614821665a | 1222d15f3fd623948cb464a50b38b4eee253f8bd | arm         | none                          | glibc |      0 | cortex-a8  |     2230 | master    | 46553162 |     89347 |     26766 |
|  92804 |      0 | 2018-10-03 22:57:02 | Romain Naour (Smile)                        | 5daeca2d5379a1c4a3d87d7f20bea743ad8388f9 | eb0928e364d426f84db10fa3d1672e8165ae7813 | arm         | none                          | glibc |      0 | cortex-a8  |     7766 | 2018.05.x | 48442941 |     92804 |     26766 |
|  93003 |      1 | 2018-10-04 18:33:10 | Romain Naour (Smile)                        | 51e17496cc403c49bc20d3a81f9c71a8a20edd0f | 93b28a2397e30ea4c2f7b4b29419f9e71eaa25a3 | arm         | dt-utils-v2018.05.0           | glibc |      0 | arm926ej-s |     2630 | master    | 48557905 |     93003 |     26766 |
|  94915 |      1 | 2018-10-11 10:38:44 | Peter Korsgaard (gcc20)                     | 45e5d2ce68650b075468c02f5c9c43b40c1ab81d | c71a16b33e42fef2995c3b6fab69497b9d8fa953 | x86_64      | samba4-4.8.4                  | glibc |      0 | bdver3     |     8714 | 2018.02.x | 49609560 |     94915 |     26766 |
|  95539 |      0 | 2018-10-13 17:33:49 | Peter Korsgaard (gcc112/ppc64le)            | 0d1a6f152f3185e128fd122f05b81f8d0fb9e6fe | c2b820940f2cdbf6572c638834e9c4d7bc7b07ef | arm         | none                          | glibc |      0 | cortex-a8  |     6439 | master    | 49951359 |     95539 |     26766 |
|  97631 |      0 | 2018-10-21 23:37:27 | Giulio Benetti (Micronova srl House Server) | bef30a9f4b58194e02ff5b2b6b78a3095c72c72c | 593899dc629e1e4b23fbc07ff2c6f1a36959698d | powerpc64   | none                          | glibc |      0 | power7     |    25716 | master    | 51128306 |     97631 |     26766 |
| 103550 |      0 | 2018-11-14 13:25:34 | Giulio Benetti (Micronova srl Server)       | ebffca8ba483610478f33cdc52a24a165e4da87e | d1901f74833c9dfa025e9f85043002cafc1e8341 | powerpc64   | none                          | glibc |      0 | power7     |     6060 | master    | 54498910 |    103550 |     26766 |
| 105139 |      0 | 2018-11-20 06:09:07 | Julien Boibessot (Armadeus systems server)  | 00d63a153decb668d67547600ca9032de7205c8b | ae5284383ded4e264991f5c3d6f724c0ca4688c0 | powerpc64   | none                          | glibc |      0 | power7     |    21588 | master    | 55402960 |    105139 |     26766 |
| 105151 |      0 | 2018-11-20 06:48:21 | Matt Weber (U14.04 Sandboxed)               | 00d63a153decb668d67547600ca9032de7205c8b | 1cf7e10ced759d02f052cc697e1b8773e3f10630 | arm         | none                          | glibc |      0 | arm926ej-s |     4507 | master    | 55409615 |    105151 |     26766 |
| 105239 |      1 | 2018-11-20 09:34:25 | Matt Weber (U16.04 THOR)                    | 5682ba9363d262a67567ed9b0531a414e6aae1bb | fc6435282630e60927f88bf5954805ebbaa60cb3 | aarch64     | libostree-2018.9              | glibc |      0 | cortex-a53 |     3040 | master    | 55464556 |    105239 |     26766 |
| 105268 |      0 | 2018-11-20 16:52:42 | Giulio Benetti (Micronova srl Server)       | 5682ba9363d262a67567ed9b0531a414e6aae1bb | 85b585eb9c8b9c4aeb9fe0346ce2d7aed6abfb6b | powerpc64le | none                          | glibc |      0 | power8     |     5651 | master    | 55480443 |    105268 |     26766 |
| 105325 |      0 | 2018-11-20 22:41:52 | Peter Korsgaard (gcc68/Debian 9/gcc 6.3.0)  | 5682ba9363d262a67567ed9b0531a414e6aae1bb | 278fc24b91305cbc7d219c027f625087a67b7e63 | powerpc64   | none                          | glibc |      0 | power7     |     4463 | master    | 55511692 |    105325 |     26766 |
| 105593 |      0 | 2018-11-21 20:38:34 | Julien Boibessot (Armadeus systems server)  | bc89c1a834cd8a1a74e71522029ec827bf0aec92 | 2d0cfff8a3632bca096f933552d126b923d2a3a1 | aarch64     | none                          | glibc |      0 | cortex-a53 |     3665 | master    | 55663453 |    105593 |     26766 |
| 105696 |      0 | 2018-11-22 05:50:26 | Julien Boibessot (Armadeus systems server)  | bc89c1a834cd8a1a74e71522029ec827bf0aec92 | 22afb0278b8443cfbce3425e93d0fa0bfc55dfc3 | arm         | none                          | glibc |      0 | arm926ej-s |    35777 | master    | 55722233 |    105696 |     26766 |
| 106023 |      0 | 2018-11-23 07:52:17 | Julien Boibessot (Armadeus systems server)  | 2ffdcbe0a5304a865e4486e7de0a7e6d983e2ce2 | ef4bf75f1429790f49a1b632faa8b00ffafcd459 | aarch64     | none                          | glibc |      0 | cortex-a53 |    31948 | next      | 55909269 |    106023 |     26766 |
+--------+--------+---------------------+---------------------------------------------+------------------------------------------+------------------------------------------+-------------+-------------------------------+-------+--------+------------+----------+-----------+----------+-----------+-----------+
16 rows in set (1 min 51.23 sec)

If anybody knows how to speed up this SQL query and/or adjust the SQL
database schema to make it faster, it would be nice. I can provide a
dump of the SQL database.

Best regards,

Thomas
-- 
Thomas Petazzoni, CTO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

[Buildroot] [PATCH] package/systemd: needs glibc

[Peter Korsgaard]

>>>>> "Yann" == Yann E MORIN <yann.morin.1998@free.fr> writes:

 > Since version v239, systemd-nspawn unconditioanlly uses prlimit(2),
 > which is not implemented in uClibc-ng. systemd-nspawn can not be
 > disabled.

 > This makes systemd glibc-only again.

 > After a bit of discussion with upstream (om IRC), it looks very
 > improbable that they accept a patch making systemd-nspawn optional.
 > They would probably consider a patch that provides that syscall wrapper
 > if it is missing, though, but that's less trivial...

 > Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
 > Cc: Waldemar Brodkorb <wbx@openadk.org>
 > Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
 > Cc: Peter Korsgaard <peter@korsgaard.com>

Committed to 2018.08.x, thanks.

-- 
Bye, Peter Korsgaard

[Buildroot] [PATCH] package/systemd: needs glibc

[Arnout Vandecappelle]

On 23/11/2018 09:27, Thomas Petazzoni wrote:
[snip]
> mysql> select * from (select * from results where builddate > '2018-09-01') as foo inner join symbol_per_result on symbol_per_result.result_id=foo.id where symbol_per_result.symbol_id=(select id from config_symbol where name="BR2_INIT_SYSTEMD" and value="y");
[snip]
> If anybody knows how to speed up this SQL query and/or adjust the SQL
> database schema to make it faster, it would be nice. I can provide a
> dump of the SQL database.

 It's hard to predict the efficiency of SQL statements, but I would expect
something like this to be more efficient:

select * from results
  inner join (select result_id from symbol_per_result A, symbol_per_result B
              where A.result_id = B.result_id
                and A.symbol_id = (select id from config_symbol
                                   where name = "BR2_INIT_SYSTEMD")
                and A.value = "y"
                and B.symbol_id = (select id from config_symbol
                                   where name = "BR2_TOOLCHAIN_USES_UCLIBC")
                and B.value = "y")
    as foo
  on foo.result_id = results.id
where builddate > '2018-09-01';

 I do assume that symbol_per_result is indexed or keyed on symbol_id so lookups
are fast, right?

 The filtering on builddate could of course be made a lot faster if the
builddate was also added to the symbol_per_result table. But the again, you
could add all of results to the symbol_per_result table...

 Regards,
 Arnout

[Buildroot] [PATCH] package/systemd: needs glibc

[Thomas Petazzoni]

Hello,

On Tue, 27 Nov 2018 22:20:37 +0100, Arnout Vandecappelle wrote:

>  It's hard to predict the efficiency of SQL statements, but I would expect
> something like this to be more efficient:
> 
> select * from results
>   inner join (select result_id from symbol_per_result A, symbol_per_result B
>               where A.result_id = B.result_id
>                 and A.symbol_id = (select id from config_symbol
>                                    where name = "BR2_INIT_SYSTEMD")
>                 and A.value = "y"
>                 and B.symbol_id = (select id from config_symbol
>                                    where name = "BR2_TOOLCHAIN_USES_UCLIBC")
>                 and B.value = "y")
>     as foo
>   on foo.result_id = results.id
> where builddate > '2018-09-01';

This query doesn't work as-is. First:

ERROR 1052 (23000): Column 'result_id' in field list is ambiguous

Because the "select result_id" in the inner query is ambiguous, using
A.result_id gets passed that.

Then, the "value" field does not exist in the symbol_per_result table.
It's the config_symbol table that associates a symbol name and its
value, so I changed the inner query to:

select A.result_id from symbol_per_result A, symbol_per_result B
	where A.result_id = B.result_id and
	A.symbol_id = (select id from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and
	B.symbol_id = (select id from config_symbol where name = "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");

but that query itself already takes a significant amount of time to
complete:

mysql> select A.result_id from symbol_per_result A, symbol_per_result B where A.result_id = B.result_id and A.symbol_id = (select id from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and B.symbol_id = (select id from config_symbol where name = "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
Empty set (3 min 52.15 sec)

I think the issue is that this query is operating on the full set of
results, and that the filtering on the build date to reduce the number
of build results to consider happens afterwards in the outermost query.
I'm really no SQL guru, but since we're interested only in querying the
results for the last few weeks or months, I would assume it should be
faster to first filter the results we're interested in by build date,
and then only do this massive symbol/value research.

If you (or anyone else) wants to play around with this, I've put online
the dump of the database as of today at
http://autobuild.buildroot.net/brautobuild.sql.gz.

Thanks,

Thomas
-- 
Thomas Petazzoni, CTO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

[Buildroot] [PATCH] package/systemd: needs glibc

[Nicolas Cavallari]

On 04/12/2018 09:28, Thomas Petazzoni wrote:
> Hello,
> 
> On Tue, 27 Nov 2018 22:20:37 +0100, Arnout Vandecappelle wrote:
> 
>>  It's hard to predict the efficiency of SQL statements, but I would expect
>> something like this to be more efficient:
>>
>> select * from results
>>   inner join (select result_id from symbol_per_result A, symbol_per_result B
>>               where A.result_id = B.result_id
>>                 and A.symbol_id = (select id from config_symbol
>>                                    where name = "BR2_INIT_SYSTEMD")
>>                 and A.value = "y"
>>                 and B.symbol_id = (select id from config_symbol
>>                                    where name = "BR2_TOOLCHAIN_USES_UCLIBC")
>>                 and B.value = "y")
>>     as foo
>>   on foo.result_id = results.id
>> where builddate > '2018-09-01';
> 
> This query doesn't work as-is. First:
> 
> ERROR 1052 (23000): Column 'result_id' in field list is ambiguous
> 
> Because the "select result_id" in the inner query is ambiguous, using
> A.result_id gets passed that.
> 
> Then, the "value" field does not exist in the symbol_per_result table.
> It's the config_symbol table that associates a symbol name and its
> value, so I changed the inner query to:
> 
> select A.result_id from symbol_per_result A, symbol_per_result B
> 	where A.result_id = B.result_id and
> 	A.symbol_id = (select id from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and
> 	B.symbol_id = (select id from config_symbol where name = "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
> 
> but that query itself already takes a significant amount of time to
> complete:
> 
> mysql> select A.result_id from symbol_per_result A, symbol_per_result B where A.result_id = B.result_id and A.symbol_id = (select id from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and B.symbol_id = (select id from config_symbol where name = "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
> Empty set (3 min 52.15 sec)

an "explain extended select [...]" on this seems to says? that mysql browses
every row of the table twice, because there is no index to use.

Adding an index on result_id helps:

MariaDB [brautobuild]> create index resid on symbol_per_result (result_id);

MariaDB [brautobuild]> select A.result_id from symbol_per_result A,
symbol_per_result B where A.result_id = B.result_id and A.symbol_id = (select id
from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and
B.symbol_id = (select id from config_symbol where name =
"BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
Empty set (9.55 sec)

> I think the issue is that this query is operating on the full set of
> results, and that the filtering on the build date to reduce the number
> of build results to consider happens afterwards in the outermost query.
> I'm really no SQL guru, but since we're interested only in querying the
> results for the last few weeks or months, I would assume it should be
> faster to first filter the results we're interested in by build date,
> and then only do this massive symbol/value research.

There is no index on builddate, so there is no way around a full scan of the
table.  But there, adding an index on date does not seems to do much, probably
because results after 2019-09-01 still represent 1/4 of all results (and mysql
guesses that it only filters 50% of the results), so using an index maybe does
not do enough to cull the results.

But yeah, most read performance problems can be resolved by adding indexes, at
the expense of disk space and write performance.

(There are also ways to reduce the space taken by the database by splitting
duplicate fields into tables, but i don't think it would improve performance.)

[?] The output of mysql explain is pretty horrible. There was
https://mariadb.org/explain_analyzer/analyze/ but it seems to be broken now :(

[Buildroot] [PATCH] package/systemd: needs glibc

[Arnout Vandecappelle]

On 04/12/2018 11:32, Nicolas Cavallari wrote:
>> select A.result_id from symbol_per_result A, symbol_per_result B
>> 	where A.result_id = B.result_id and
>> 	A.symbol_id = (select id from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and
>> 	B.symbol_id = (select id from config_symbol where name = "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
>>
>> but that query itself already takes a significant amount of time to
>> complete:
>>
>> mysql> select A.result_id from symbol_per_result A, symbol_per_result B where A.result_id = B.result_id and A.symbol_id = (select id from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and B.symbol_id = (select id from config_symbol where name = "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
>> Empty set (3 min 52.15 sec)
> an "explain extended select [...]" on this seems to says? that mysql browses
> every row of the table twice, because there is no index to use.
> 
> Adding an index on result_id helps:
> 
> MariaDB [brautobuild]> create index resid on symbol_per_result (result_id);
> 
> MariaDB [brautobuild]> select A.result_id from symbol_per_result A,
> symbol_per_result B where A.result_id = B.result_id and A.symbol_id = (select id
> from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and
> B.symbol_id = (select id from config_symbol where name =
> "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
> Empty set (9.55 sec)

 Maybe it would be even better to make (result_id, symbol_id) the primary key of
symbol_per_result? Or do I misunderstand the implications of primary key?

 Regards,
 Arnout

[Buildroot] [PATCH] package/systemd: needs glibc

[Nicolas Cavallari]

On 04/12/2018 12:02, Arnout Vandecappelle wrote:
> On 04/12/2018 11:32, Nicolas Cavallari wrote:
>> Adding an index on result_id helps:
>>
>> MariaDB [brautobuild]> create index resid on symbol_per_result (result_id);
>>
>> MariaDB [brautobuild]> select A.result_id from symbol_per_result A,
>> symbol_per_result B where A.result_id = B.result_id and A.symbol_id = (select id
>> from config_symbol where name = "BR2_INIT_SYSTEMD" and value = "y") and
>> B.symbol_id = (select id from config_symbol where name =
>> "BR2_TOOLCHAIN_USES_UCLIBC" and value = "y");
>> Empty set (9.55 sec)
> 
>  Maybe it would be even better to make (result_id, symbol_id) the primary key of
> symbol_per_result? Or do I misunderstand the implications of primary key?

i was a bit confused about the various id in symbol_per_result.  If
symbol_per_result.id is unused, then yes, remove it and make (result_id,
symbol_id) the primary key.

(i'm assuming that mysql can use an (a,b) index to search for a, it should be
the case now in 2018)