[Buildroot] Build reproducibility

[Buildroot] Build reproducibility

[Jérôme Pouiller]

Hello,

I noticed a problem with reproducibility of build. If I list targets 
(using make show-targets) on Fedora and on Ubuntu, targets are on 
sorted on same orders. Indeed, result of $(wildcard *.mk) are not 
the same on these systems. 

It seems Make does not sort anymore result of wildcard since 3.82: 
http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260.

Following patch should fix problem.

[Buildroot] [PATCH] Fix build reproducibility in Make 3.82

[Jérôme Pouiller]

Make 3.82 do not sort anymore result of wildcards (see
http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break
build reproducibility.

This patch sort results of wildcards to ensure reproducibility.

Signed-off-by: J?r?me Pouiller <jezz@sysmic.org>
---
 Makefile                               |    2 +-
 boot/common.mk                         |    2 +-
 docs/manual/manual.mk                  |    2 +-
 fs/common.mk                           |    2 +-
 linux/linux.mk                         |    2 +-
 package/efl/efl.mk                     |    2 +-
 package/freescale-imx/freescale-imx.mk |    2 +-
 package/gcc/gcc.mk                     |    2 +-
 package/gtk2-themes/gtk2-themes.mk     |    2 +-
 package/matchbox/matchbox.mk           |    2 +-
 package/opengl/opengl.mk               |    2 +-
 package/qt5/qt5.mk                     |    2 +-
 package/x11r7/x11r7.mk                 |    2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/Makefile b/Makefile
index 84d160a..5d6315a 100644
--- a/Makefile
+++ b/Makefile
@@ -337,7 +337,7 @@ ifneq ($(PACKAGE_OVERRIDE_FILE),)
 -include $(PACKAGE_OVERRIDE_FILE)
 endif
 
-include package/*/*.mk
+include $(sort $(wildcard package/*/*.mk))
 
 include boot/common.mk
 include linux/linux.mk
diff --git a/boot/common.mk b/boot/common.mk
index b315fe8..3021e51 100644
--- a/boot/common.mk
+++ b/boot/common.mk
@@ -1 +1 @@
-include boot/*/*.mk
+include $(sort $(wildcard boot/*/*.mk))
diff --git a/docs/manual/manual.mk b/docs/manual/manual.mk
index 4906bc8..bcbedad 100644
--- a/docs/manual/manual.mk
+++ b/docs/manual/manual.mk
@@ -53,5 +53,5 @@ $(1)-clean:
 .PHONY: $(1) $(1)-clean manual-update-lists
 endef
 
-MANUAL_SOURCES = $(wildcard docs/manual/*.txt) $(wildcard docs/images/*)
+MANUAL_SOURCES = $(sort $(wildcard docs/manual/*.txt) $(wildcard docs/images/*))
 $(eval $(call GENDOC,manual))
diff --git a/fs/common.mk b/fs/common.mk
index 36da2dd..4dab7ea 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -94,4 +94,4 @@ define ROOTFS_TARGET
 $(call ROOTFS_TARGET_INTERNAL,$(1),$(call UPPERCASE,$(1)))
 endef
 
-include fs/*/*.mk
+include $(sort $(wildcard fs/*/*.mk))
diff --git a/linux/linux.mk b/linux/linux.mk
index 578d535..b9d7028 100644
--- a/linux/linux.mk
+++ b/linux/linux.mk
@@ -281,7 +281,7 @@ define LINUX_INSTALL_TARGET_CMDS
 	$(LINUX_INSTALL_HOST_TOOLS)
 endef
 
-include linux/linux-ext-*.mk
+include $(sort $(wildcard linux/linux-ext-*.mk))
 
 $(eval $(generic-package))
 
diff --git a/package/efl/efl.mk b/package/efl/efl.mk
index f284c93..1de68cf 100644
--- a/package/efl/efl.mk
+++ b/package/efl/efl.mk
@@ -1,3 +1,3 @@
 EFL_VERSION = 1.7.7
 
-include package/efl/*/*.mk
+include $(sort $(wildcard package/efl/*/*.mk))
diff --git a/package/freescale-imx/freescale-imx.mk b/package/freescale-imx/freescale-imx.mk
index 9f63ce3..c8f99b6 100644
--- a/package/freescale-imx/freescale-imx.mk
+++ b/package/freescale-imx/freescale-imx.mk
@@ -9,5 +9,5 @@ FREESCALE_IMX_VERSION = 1.1.0
 # No official download site from freescale, just this mirror
 FREESCALE_IMX_MIRROR_SITE   = http://download.ossystems.com.br/bsp/freescale/source
 
-include package/freescale-imx/*/*.mk
+include $(sort $(wildcard package/freescale-imx/*/*.mk))
 
diff --git a/package/gcc/gcc.mk b/package/gcc/gcc.mk
index cd6efed..66af425 100644
--- a/package/gcc/gcc.mk
+++ b/package/gcc/gcc.mk
@@ -215,4 +215,4 @@ HOST_GCC_COMMON_CONF_OPT += \
 	--with-long-double-128
 endif
 
-include package/gcc/*/*.mk
+include $(sort $(wildcard package/gcc/*/*.mk))
diff --git a/package/gtk2-themes/gtk2-themes.mk b/package/gtk2-themes/gtk2-themes.mk
index 6f31c6d..fc66698 100644
--- a/package/gtk2-themes/gtk2-themes.mk
+++ b/package/gtk2-themes/gtk2-themes.mk
@@ -1 +1 @@
-include package/gtk2-themes/*/*.mk
+include $(sort $(wildcard package/gtk2-themes/*/*.mk))
diff --git a/package/matchbox/matchbox.mk b/package/matchbox/matchbox.mk
index abda915..fff4add 100644
--- a/package/matchbox/matchbox.mk
+++ b/package/matchbox/matchbox.mk
@@ -1,4 +1,4 @@
 ifeq ($(BR2_PACKAGE_MATCHBOX),y)
-include package/matchbox/*/*.mk
+include $(sort $(wildcard package/matchbox/*/*.mk))
 TARGETS+=matchbox-lib matchbox-wm
 endif
diff --git a/package/opengl/opengl.mk b/package/opengl/opengl.mk
index 68df3b1..abf96d5 100644
--- a/package/opengl/opengl.mk
+++ b/package/opengl/opengl.mk
@@ -1 +1 @@
-include package/opengl/*/*.mk
+include $(sort $(wildcard package/opengl/*/*.mk))
diff --git a/package/qt5/qt5.mk b/package/qt5/qt5.mk
index 9e71c83..fce0ca3 100644
--- a/package/qt5/qt5.mk
+++ b/package/qt5/qt5.mk
@@ -1,6 +1,6 @@
 QT5_VERSION = 5.0.2
 QT5_SITE = http://download.qt-project.org/archive/qt/5.0/$(QT5_VERSION)/submodules/
-include package/qt5/*/*.mk
+include $(sort $(wildcard package/qt5/*/*.mk))
 
 define QT5_LA_PRL_FILES_FIXUP
 	for i in $$(find $(STAGING_DIR)/usr/lib* -name "libQt5*.la"); do \
diff --git a/package/x11r7/x11r7.mk b/package/x11r7/x11r7.mk
index ea0363f..85ecbc5 100644
--- a/package/x11r7/x11r7.mk
+++ b/package/x11r7/x11r7.mk
@@ -1 +1 @@
-include package/x11r7/*/*.mk
+include $(sort $(wildcard package/x11r7/*/*.mk))
-- 
1.7.9.5

[Buildroot] Build reproducibility

[Thomas De Schampheleire]

Op 30-aug.-2013 10:32 schreef "J?r?me Pouiller" <jezz@sysmic.org> het
volgende:
>
> Hello,
>
> I noticed a problem with reproducibility of build. If I list targets
> (using make show-targets) on Fedora and on Ubuntu, targets are on
> sorted on same orders. Indeed, result of $(wildcard *.mk) are not
> the same on these systems.
>
> It seems Make does not sort anymore result of wildcard since 3.82:
> http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260.
>
> Following patch should fix problem.

In one of the previous buildroot developer days we discussed this. IIRC,
the conclusion was that this change in order shouldn't matter, because all
dependencies should be expressed in make. Hence, the end result should be
the same, even though build order was not.

Have you come across a scenario where there actually was a problem?

Best regards,
Thomas

>
> _______________________________________________
> buildroot mailing list
> buildroot at busybox.net
> http://lists.busybox.net/mailman/listinfo/buildroot
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20130830/1296e9e7/attachment.html>

[Buildroot] Build reproducibility

[Jérôme Pouiller]

On 2013-08-30 13:59, Thomas De Schampheleire wrote:
> Op 30-aug.-2013 10:32 schreef "J?r?me Pouiller" <jezz@sysmic.org> het 
> volgende:
>  >
>  > Hello,
>  >
>  > I noticed a problem with reproducibility of build. If I list 
> targets
>  > (using make show-targets) on Fedora and on Ubuntu, targets are on
>  > sorted on same orders. Indeed, result of $(wildcard *.mk) are not
>  > the same on these systems.
>  >
>  > It seems Make does not sort anymore result of wildcard since 3.82:
>  > http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260 [1].
>  >
>  > Following patch should fix problem.
>
> In one of the previous buildroot developer days we discussed this.
> IIRC, the conclusion was that this change in order shouldn't matter,
> because all dependencies should be expressed in make. Hence, the end
> result should be the same, even though build order was not.
>
> Have you come across a scenario where there actually was a problem?

Some packages may detect installed library during ./configure run. 
Buildroot
should be aware of this and fix dependencies with that library. 
Although, it
is not always the case and Buildroot should at guarantee 
reproducibility of
build.

Without that, we cannot guarantee to reproduce a build done by 
Autobuilder.



-- 
J?r?me Pouiller, Sysmic
Embedded Linux specialist
http://www.sysmic.fr

[Buildroot] Build reproducibility

[Thomas Petazzoni]

Dear J?r?me Pouiller,

On Fri, 30 Aug 2013 14:44:01 +0200, J?r?me Pouiller wrote:

> > In one of the previous buildroot developer days we discussed this.
> > IIRC, the conclusion was that this change in order shouldn't matter,
> > because all dependencies should be expressed in make. Hence, the end
> > result should be the same, even though build order was not.
> >
> > Have you come across a scenario where there actually was a problem?
> 
> Some packages may detect installed library during ./configure run. 
> Buildroot
> should be aware of this and fix dependencies with that library. 
> Although, it
> is not always the case and Buildroot should at guarantee 
> reproducibility of
> build.
> 
> Without that, we cannot guarantee to reproduce a build done by 
> Autobuilder.

I agree with this. Thomas, I'm not sure that what you say what a
conclusion of a developer day. I believe we always said that it is
hardly possible to guarantee that a package .mk file will contain *all*
the possible dependencies of the package. Whenever someone bumps a
package, we rarely look in detail at whether the software has gained
some optional dependencies, and make sure they are handled in the
corresponding package .mk file.

Having the packages always built in the same order guarantees that, in
the absence of expressed dependencies, the build result will be
reproducible.

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] Build reproducibility

[Thomas De Schampheleire]

Hi Thomas,

On Fri, Aug 30, 2013 at 2:52 PM, Thomas Petazzoni
<thomas.petazzoni@free-electrons.com> wrote:
> Dear J?r?me Pouiller,
>
> On Fri, 30 Aug 2013 14:44:01 +0200, J?r?me Pouiller wrote:
>
>> > In one of the previous buildroot developer days we discussed this.
>> > IIRC, the conclusion was that this change in order shouldn't matter,
>> > because all dependencies should be expressed in make. Hence, the end
>> > result should be the same, even though build order was not.
>> >
>> > Have you come across a scenario where there actually was a problem?
>>
>> Some packages may detect installed library during ./configure run.
>> Buildroot
>> should be aware of this and fix dependencies with that library.
>> Although, it
>> is not always the case and Buildroot should at guarantee
>> reproducibility of
>> build.
>>
>> Without that, we cannot guarantee to reproduce a build done by
>> Autobuilder.
>
> I agree with this. Thomas, I'm not sure that what you say what a
> conclusion of a developer day. I believe we always said that it is
> hardly possible to guarantee that a package .mk file will contain *all*
> the possible dependencies of the package. Whenever someone bumps a
> package, we rarely look in detail at whether the software has gained
> some optional dependencies, and make sure they are handled in the
> corresponding package .mk file.
>
> Having the packages always built in the same order guarantees that, in
> the absence of expressed dependencies, the build result will be
> reproducible.

I may be wrong about the conclusion.
Regardless: it's true that it's hard to guarantee that all
dependencies are expressed properly in the .mk files. However, by
'fixing' the wildcard behavior into a sorted one such as with previous
versions of make, we just hide the problem. We will never notice that
some dependencies are missing, as long as the dependency comes before
the dependant (or whichever word) in alphabetical order.
With the random behavior of wildcards in newer versions of make, we
would still see issues in autobuilds, and get the opportunity to fix
them. It may take time, and may be a never-ending story as packages
are bumped and new packages are added, but at least there can be
improvement.
So, my viewpoint is to keep the current behavior and instead focus on
fixing any missing dependency when we spot it.

Best regards,
Thomas

[Buildroot] Build reproducibility

[Thomas Petazzoni]

Dear Thomas De Schampheleire,

On Mon, 2 Sep 2013 10:44:17 +0200, Thomas De Schampheleire wrote:

> > I agree with this. Thomas, I'm not sure that what you say what a
> > conclusion of a developer day. I believe we always said that it is
> > hardly possible to guarantee that a package .mk file will contain *all*
> > the possible dependencies of the package. Whenever someone bumps a
> > package, we rarely look in detail at whether the software has gained
> > some optional dependencies, and make sure they are handled in the
> > corresponding package .mk file.
> >
> > Having the packages always built in the same order guarantees that, in
> > the absence of expressed dependencies, the build result will be
> > reproducible.
> 
> I may be wrong about the conclusion.
> Regardless: it's true that it's hard to guarantee that all
> dependencies are expressed properly in the .mk files. However, by
> 'fixing' the wildcard behavior into a sorted one such as with previous
> versions of make, we just hide the problem. We will never notice that
> some dependencies are missing, as long as the dependency comes before
> the dependant (or whichever word) in alphabetical order.
> With the random behavior of wildcards in newer versions of make, we
> would still see issues in autobuilds, and get the opportunity to fix
> them. It may take time, and may be a never-ending story as packages
> are bumped and new packages are added, but at least there can be
> improvement.
> So, my viewpoint is to keep the current behavior and instead focus on
> fixing any missing dependency when we spot it.

I obviously disagree, because in the mean time, our users are having
non-reproducible builds. An user within a company uses Buildroot to
create a system, adds some packages, creates a configuration for his
project. Then he passes this Buildroot to another colleague: the
date/times of the various Buildroot files will be different, maybe
affecting the order in which the wildcards are resolved by make 3.82.
This colleague will attempt the build, and maybe get a failure, or a
different build result than the first colleague who has created the 
Buildroot configuration. This is really damaging for Buildroot's
reputation and the user experience. We clearly to not want this to
happen.

Of course, if within the Buildroot project we are interested in fixing
such missing dependencies, then we can find a way of adding randomness
into the build order in our autobuilders. But clearly, we do want to
expose this randomness to our users.

In fact, I had already thought about adding such randomness in the
autobuilders. But I've refrained from doing so because it also means
that the builds that the autobuilders are doing cannot be reproduced.
So when you'll get an autobuilder failure that you can't reproduce
locally, you'll never know if it's due to the random order of package
building, or due to some difference in the build environment.

Best regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] Build reproducibility

[Thomas De Schampheleire]

Hi Thomas,

On Mon, Sep 2, 2013 at 10:53 AM, Thomas Petazzoni
<thomas.petazzoni@free-electrons.com> wrote:
> Dear Thomas De Schampheleire,
>
> On Mon, 2 Sep 2013 10:44:17 +0200, Thomas De Schampheleire wrote:
>
>> > I agree with this. Thomas, I'm not sure that what you say what a
>> > conclusion of a developer day. I believe we always said that it is
>> > hardly possible to guarantee that a package .mk file will contain *all*
>> > the possible dependencies of the package. Whenever someone bumps a
>> > package, we rarely look in detail at whether the software has gained
>> > some optional dependencies, and make sure they are handled in the
>> > corresponding package .mk file.
>> >
>> > Having the packages always built in the same order guarantees that, in
>> > the absence of expressed dependencies, the build result will be
>> > reproducible.
>>
>> I may be wrong about the conclusion.
>> Regardless: it's true that it's hard to guarantee that all
>> dependencies are expressed properly in the .mk files. However, by
>> 'fixing' the wildcard behavior into a sorted one such as with previous
>> versions of make, we just hide the problem. We will never notice that
>> some dependencies are missing, as long as the dependency comes before
>> the dependant (or whichever word) in alphabetical order.
>> With the random behavior of wildcards in newer versions of make, we
>> would still see issues in autobuilds, and get the opportunity to fix
>> them. It may take time, and may be a never-ending story as packages
>> are bumped and new packages are added, but at least there can be
>> improvement.
>> So, my viewpoint is to keep the current behavior and instead focus on
>> fixing any missing dependency when we spot it.
>
> I obviously disagree, because in the mean time, our users are having
> non-reproducible builds. An user within a company uses Buildroot to
> create a system, adds some packages, creates a configuration for his
> project. Then he passes this Buildroot to another colleague: the
> date/times of the various Buildroot files will be different, maybe
> affecting the order in which the wildcards are resolved by make 3.82.
> This colleague will attempt the build, and maybe get a failure, or a
> different build result than the first colleague who has created the
> Buildroot configuration. This is really damaging for Buildroot's
> reputation and the user experience. We clearly to not want this to
> happen.

Ok, fair enough, I can follow this argument.

>
> Of course, if within the Buildroot project we are interested in fixing
> such missing dependencies, then we can find a way of adding randomness
> into the build order in our autobuilders. But clearly, we do want to
> expose this randomness to our users.

I think indeed we should try to set the dependencies right some way or another.

If we assume that a package does not have any configurable options
that would change its dependencies, a simple way to check if all
dependencies are properly expressed is through:
make clean toolchain foo

Correct? Although it would take many more builds, each of them would
be relatively short. If we even can find a way to
'clean-all-but-the-toolchain', the cleaning time will be much shorter.
For this type of simple package test, only one toolchain needs to be
used because the dependencies should not depend on the toolchain.
Also, it's not necessary that each package gets build every night:
once the dependencies are correct, they will stay correct until a
version bump. This means we can spread out the execution of this type
of tests over time, interleaving them with the already existing
autobuilds with random configurations.

Of course, when a package is more complex and the its Config.in file
allows to enable/disable certain options which are based on another
library that would need to be added to the dependencies, then we'd
still need a way to test the different configurations. This could be
achieved with the regular autobuilds, but disabling the wildcard
sorting, which brings us to...

>
> In fact, I had already thought about adding such randomness in the
> autobuilders. But I've refrained from doing so because it also means
> that the builds that the autobuilders are doing cannot be reproduced.
> So when you'll get an autobuilder failure that you can't reproduce
> locally, you'll never know if it's due to the random order of package
> building, or due to some difference in the build environment.

The make targets of buildroot itself are executed sequentially.
Suppose that we keep a list of all targets executed, something like:
python-source
python-extract
python-patch
python-configure
python-build
python-install-target
pyfoo-source
pyfoo-extract
pyfoo-build
...

To reproduce a build, we can explicitly pass this list on the make
command-line, roughly like:
cat <target-list> | xargs make clean toolchain


Regardless of this possibility (or not, I may be overlooking
something), not all autobuilds need to be randomized. We could
randomize some, and clearly indicate on the autobuild page that this
was the case. A full build log would be useful in this case.

Best regards,
Thomas

[Buildroot] Build reproducibility

[Arnout Vandecappelle]

On 09/02/13 10:44, Thomas De Schampheleire wrote:
> Hi Thomas,
>
> On Fri, Aug 30, 2013 at 2:52 PM, Thomas Petazzoni
> <thomas.petazzoni@free-electrons.com> wrote:
>> Dear J?r?me Pouiller,
>>
>> On Fri, 30 Aug 2013 14:44:01 +0200, J?r?me Pouiller wrote:
>>
>>>> In one of the previous buildroot developer days we discussed this.
>>>> IIRC, the conclusion was that this change in order shouldn't matter,
>>>> because all dependencies should be expressed in make. Hence, the end
>>>> result should be the same, even though build order was not.
>>>>
>>>> Have you come across a scenario where there actually was a problem?
>>>
>>> Some packages may detect installed library during ./configure run.
>>> Buildroot
>>> should be aware of this and fix dependencies with that library.
>>> Although, it
>>> is not always the case and Buildroot should at guarantee
>>> reproducibility of
>>> build.
>>>
>>> Without that, we cannot guarantee to reproduce a build done by
>>> Autobuilder.
>>
>> I agree with this. Thomas, I'm not sure that what you say what a
>> conclusion of a developer day. I believe we always said that it is
>> hardly possible to guarantee that a package .mk file will contain *all*
>> the possible dependencies of the package. Whenever someone bumps a
>> package, we rarely look in detail at whether the software has gained
>> some optional dependencies, and make sure they are handled in the
>> corresponding package .mk file.
>>
>> Having the packages always built in the same order guarantees that, in
>> the absence of expressed dependencies, the build result will be
>> reproducible.
>
> I may be wrong about the conclusion.
> Regardless: it's true that it's hard to guarantee that all
> dependencies are expressed properly in the .mk files. However, by
> 'fixing' the wildcard behavior into a sorted one such as with previous
> versions of make, we just hide the problem. We will never notice that
> some dependencies are missing, as long as the dependency comes before
> the dependant (or whichever word) in alphabetical order.
 > With the random behavior of wildcards in newer versions of make, we
 > would still see issues in autobuilds, and get the opportunity to fix
 > them. It may take time, and may be a never-ending story as packages
 > are bumped and new packages are added, but at least there can be
 > improvement.
 > So, my viewpoint is to keep the current behavior and instead focus on
 > fixing any missing dependency when we spot it.

  I think this scenario is rather unlikely in reality. If it's a 
dependency which is not mentioned at all in .mk or Config.in, then at 
some point there will be a random config that has the package enabled but 
not the dependency. So this scenario will only occur if the dependency is 
mentioned in Config.in, but not in the .mk. With the amount of review we 
do, that is not very likely to happen. And even if it does, it is not 
correct but buildroot will still function correctly in the normal use 
case ('make' instead of 'make pkg').

  What is much more likely to happen is that there is some optional 
dependency in the package's configure or build system that is not 
expressed in Config.in or pkg.mk. Most reviewers do not run a 'configure 
--help', and even then it is easy to miss something. Since the dependency 
is optional, the build will not fail either way. Only, when user A builds 
it, TLS support is included, but when user B builds it, it is not... 
That's the kind of lack of reproducability we really need to avoid.


  Note that doing more randomized build order in the autobuilder also 
will not capture the latter scenario. You would have to compare the build 
result - but binary differences are likely because of changing timestamps 
or changing optimizations depending on memory randomness.


  Regards,
  Arnout

>
> Best regards,
> Thomas
> _______________________________________________
> buildroot mailing list
> buildroot at busybox.net
> http://lists.busybox.net/mailman/listinfo/buildroot
>
>


-- 
Arnout Vandecappelle                          arnout at mind be
Senior Embedded Software Architect            +32-16-286500
Essensium/Mind                                http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium           BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F

[Buildroot] [PATCH] Fix build reproducibility in Make 3.82

[Arnout Vandecappelle]

On 08/30/13 10:31, J?r?me Pouiller wrote:
> Make 3.82 do not sort anymore result of wildcards (see
> http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break
> build reproducibility.
>
> This patch sort results of wildcards to ensure reproducibility.
>
> Signed-off-by: J?r?me Pouiller <jezz@sysmic.org>

Acked-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>

  You missed one in support/dependencies/dependencies.mk, but that one 
doesn't make a difference for the build result.

  Also the one of the manual isn't really necessary; it is only used to 
check for rebuilding the manual and doesn't affect the way the manual is 
built.

  But both of these are not really important differences, hence my ack.


  Regards,
  Arnout

> ---
>   Makefile                               |    2 +-
>   boot/common.mk                         |    2 +-
>   docs/manual/manual.mk                  |    2 +-
>   fs/common.mk                           |    2 +-
>   linux/linux.mk                         |    2 +-
>   package/efl/efl.mk                     |    2 +-
>   package/freescale-imx/freescale-imx.mk |    2 +-
>   package/gcc/gcc.mk                     |    2 +-
>   package/gtk2-themes/gtk2-themes.mk     |    2 +-
>   package/matchbox/matchbox.mk           |    2 +-
>   package/opengl/opengl.mk               |    2 +-
>   package/qt5/qt5.mk                     |    2 +-
>   package/x11r7/x11r7.mk                 |    2 +-
>   13 files changed, 13 insertions(+), 13 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index 84d160a..5d6315a 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -337,7 +337,7 @@ ifneq ($(PACKAGE_OVERRIDE_FILE),)
>   -include $(PACKAGE_OVERRIDE_FILE)
>   endif
>
> -include package/*/*.mk
> +include $(sort $(wildcard package/*/*.mk))
>
>   include boot/common.mk
>   include linux/linux.mk
> diff --git a/boot/common.mk b/boot/common.mk
> index b315fe8..3021e51 100644
> --- a/boot/common.mk
> +++ b/boot/common.mk
> @@ -1 +1 @@
> -include boot/*/*.mk
> +include $(sort $(wildcard boot/*/*.mk))
> diff --git a/docs/manual/manual.mk b/docs/manual/manual.mk
> index 4906bc8..bcbedad 100644
> --- a/docs/manual/manual.mk
> +++ b/docs/manual/manual.mk
> @@ -53,5 +53,5 @@ $(1)-clean:
>   .PHONY: $(1) $(1)-clean manual-update-lists
>   endef
>
> -MANUAL_SOURCES = $(wildcard docs/manual/*.txt) $(wildcard docs/images/*)
> +MANUAL_SOURCES = $(sort $(wildcard docs/manual/*.txt) $(wildcard docs/images/*))
>   $(eval $(call GENDOC,manual))
> diff --git a/fs/common.mk b/fs/common.mk
> index 36da2dd..4dab7ea 100644
> --- a/fs/common.mk
> +++ b/fs/common.mk
> @@ -94,4 +94,4 @@ define ROOTFS_TARGET
>   $(call ROOTFS_TARGET_INTERNAL,$(1),$(call UPPERCASE,$(1)))
>   endef
>
> -include fs/*/*.mk
> +include $(sort $(wildcard fs/*/*.mk))
> diff --git a/linux/linux.mk b/linux/linux.mk
> index 578d535..b9d7028 100644
> --- a/linux/linux.mk
> +++ b/linux/linux.mk
> @@ -281,7 +281,7 @@ define LINUX_INSTALL_TARGET_CMDS
>   	$(LINUX_INSTALL_HOST_TOOLS)
>   endef
>
> -include linux/linux-ext-*.mk
> +include $(sort $(wildcard linux/linux-ext-*.mk))
>
>   $(eval $(generic-package))
>
> diff --git a/package/efl/efl.mk b/package/efl/efl.mk
> index f284c93..1de68cf 100644
> --- a/package/efl/efl.mk
> +++ b/package/efl/efl.mk
> @@ -1,3 +1,3 @@
>   EFL_VERSION = 1.7.7
>
> -include package/efl/*/*.mk
> +include $(sort $(wildcard package/efl/*/*.mk))
> diff --git a/package/freescale-imx/freescale-imx.mk b/package/freescale-imx/freescale-imx.mk
> index 9f63ce3..c8f99b6 100644
> --- a/package/freescale-imx/freescale-imx.mk
> +++ b/package/freescale-imx/freescale-imx.mk
> @@ -9,5 +9,5 @@ FREESCALE_IMX_VERSION = 1.1.0
>   # No official download site from freescale, just this mirror
>   FREESCALE_IMX_MIRROR_SITE   = http://download.ossystems.com.br/bsp/freescale/source
>
> -include package/freescale-imx/*/*.mk
> +include $(sort $(wildcard package/freescale-imx/*/*.mk))
>
> diff --git a/package/gcc/gcc.mk b/package/gcc/gcc.mk
> index cd6efed..66af425 100644
> --- a/package/gcc/gcc.mk
> +++ b/package/gcc/gcc.mk
> @@ -215,4 +215,4 @@ HOST_GCC_COMMON_CONF_OPT += \
>   	--with-long-double-128
>   endif
>
> -include package/gcc/*/*.mk
> +include $(sort $(wildcard package/gcc/*/*.mk))
> diff --git a/package/gtk2-themes/gtk2-themes.mk b/package/gtk2-themes/gtk2-themes.mk
> index 6f31c6d..fc66698 100644
> --- a/package/gtk2-themes/gtk2-themes.mk
> +++ b/package/gtk2-themes/gtk2-themes.mk
> @@ -1 +1 @@
> -include package/gtk2-themes/*/*.mk
> +include $(sort $(wildcard package/gtk2-themes/*/*.mk))
> diff --git a/package/matchbox/matchbox.mk b/package/matchbox/matchbox.mk
> index abda915..fff4add 100644
> --- a/package/matchbox/matchbox.mk
> +++ b/package/matchbox/matchbox.mk
> @@ -1,4 +1,4 @@
>   ifeq ($(BR2_PACKAGE_MATCHBOX),y)
> -include package/matchbox/*/*.mk
> +include $(sort $(wildcard package/matchbox/*/*.mk))
>   TARGETS+=matchbox-lib matchbox-wm
>   endif
> diff --git a/package/opengl/opengl.mk b/package/opengl/opengl.mk
> index 68df3b1..abf96d5 100644
> --- a/package/opengl/opengl.mk
> +++ b/package/opengl/opengl.mk
> @@ -1 +1 @@
> -include package/opengl/*/*.mk
> +include $(sort $(wildcard package/opengl/*/*.mk))
> diff --git a/package/qt5/qt5.mk b/package/qt5/qt5.mk
> index 9e71c83..fce0ca3 100644
> --- a/package/qt5/qt5.mk
> +++ b/package/qt5/qt5.mk
> @@ -1,6 +1,6 @@
>   QT5_VERSION = 5.0.2
>   QT5_SITE = http://download.qt-project.org/archive/qt/5.0/$(QT5_VERSION)/submodules/
> -include package/qt5/*/*.mk
> +include $(sort $(wildcard package/qt5/*/*.mk))
>
>   define QT5_LA_PRL_FILES_FIXUP
>   	for i in $$(find $(STAGING_DIR)/usr/lib* -name "libQt5*.la"); do \
> diff --git a/package/x11r7/x11r7.mk b/package/x11r7/x11r7.mk
> index ea0363f..85ecbc5 100644
> --- a/package/x11r7/x11r7.mk
> +++ b/package/x11r7/x11r7.mk
> @@ -1 +1 @@
> -include package/x11r7/*/*.mk
> +include $(sort $(wildcard package/x11r7/*/*.mk))
>


-- 
Arnout Vandecappelle                          arnout at mind be
Senior Embedded Software Architect            +32-16-286500
Essensium/Mind                                http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium           BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F

[Buildroot] Build reproducibility

[Peter Korsgaard]

>>>>> "Arnout" == Arnout Vandecappelle <arnout@mind.be> writes:

Hi,

 Arnout>  What is much more likely to happen is that there is some optional
 Arnout> dependency in the package's configure or build system that is not
 Arnout> expressed in Config.in or pkg.mk. Most reviewers do not run a
 Arnout> configure --help', and even then it is easy to miss something. Since
 Arnout> the dependency is optional, the build will not fail either way. Only,
 Arnout> when user A builds it, TLS support is included, but when user B builds
 Arnout> it, it is not... That's the kind of lack of reproducability we really
 Arnout> need to avoid.

Indeed.

 Arnout>  Note that doing more randomized build order in the autobuilder also
 Arnout> will not capture the latter scenario. You would have to compare the
 Arnout> build result - but binary differences are likely because of changing
 Arnout> timestamps or changing optimizations depending on memory randomness.

Exactly. I don't have any good ideas about how to detect this (besides
building all packages in clean staging dirs, E.G. only populated with
its explicit dependencies like afaik OE lite can do, but that would
require quite some work), anyone?

-- 
Bye, Peter Korsgaard

[Buildroot] Build reproducibility

[Thomas Petazzoni]

Dear Peter Korsgaard,

On Tue, 03 Sep 2013 08:26:07 +0200, Peter Korsgaard wrote:

>  Arnout>  Note that doing more randomized build order in the autobuilder also
>  Arnout> will not capture the latter scenario. You would have to compare the
>  Arnout> build result - but binary differences are likely because of changing
>  Arnout> timestamps or changing optimizations depending on memory randomness.
> 
> Exactly. I don't have any good ideas about how to detect this (besides
> building all packages in clean staging dirs, E.G. only populated with
> its explicit dependencies like afaik OE lite can do, but that would
> require quite some work), anyone?

Doing a per-package sysroot that is generated for each package before
it gets built, with only the explicitly listed dependencies, is indeed
the only way to ensure that a package is not seeing/using something
that isn't declared as a dependency.

This would certainly be nice to have (as it also helps top-level
parallel build, as was discussed with Fabio Porcedda some time ago),
but:

 1) I'm worried about the additional complexity inside Buildroot.

 2) I'm worried about the additional build time required to generate a
    per-package sysroot for each package. When building large stacks
    like X.org that has many small packages, but each have a lot of
    dependencies, the cost of creating a sysroot before building each
    package could be huge.

 3) We still need to provide the user a global sysroot with all
    libraries installed, so that he can use the toolchain generated by
    Buildroot to build his own libraries/applications. This would mean
    we would need to have two sysroots: the global sysroot, that gets
    incrementally populated with what all packages are installing, but
    that isn't used for building packages inside Buildroot, and a
    separate temporary sysroot, used when building the current package.
    Since the compiler would default to the 'global sysroot', we would
    have to pass --sysroot $(TMPSYSROOT) all the time, or have a
    separate wrapper, or something. Not impossible, but see (1).

Best regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] Build reproducibility

[Peter Korsgaard]

>>>>> "Thomas" == Thomas Petazzoni <thomas.petazzoni@free-electrons.com> writes:

Hi,

 Thomas> This would certainly be nice to have (as it also helps top-level
 Thomas> parallel build, as was discussed with Fabio Porcedda some time ago),
 Thomas> but:

[snip]

Indeed. For the record, I also don't think we should do this, I was just
stating what was needed if we wanted to do so.

As long as the dependencies are correct, I don't think seperate staging
dirs are needed for toplevel parallel builds.

-- 
Bye, Peter Korsgaard

[Buildroot] Build reproducibility

[Jérôme Pouiller]

On 2013-09-03 08:26, Peter Korsgaard wrote:
>>>>>> "Arnout" == Arnout Vandecappelle <arnout@mind.be> writes:
>
> Hi,
>
>  Arnout>  What is much more likely to happen is that there is some 
> optional
>  Arnout> dependency in the package's configure or build system that 
> is not
>  Arnout> expressed in Config.in or pkg.mk. Most reviewers do not run 
> a
>  Arnout> configure --help', and even then it is easy to miss 
> something. Since
>  Arnout> the dependency is optional, the build will not fail either 
> way. Only,
>  Arnout> when user A builds it, TLS support is included, but when 
> user B builds
>  Arnout> it, it is not... That's the kind of lack of reproducability 
> we really
>  Arnout> need to avoid.
>
> Indeed.
>
>  Arnout>  Note that doing more randomized build order in the 
> autobuilder also
>  Arnout> will not capture the latter scenario. You would have to 
> compare the
>  Arnout> build result - but binary differences are likely because of 
> changing
>  Arnout> timestamps or changing optimizations depending on memory 
> randomness.
>
> Exactly. I don't have any good ideas about how to detect this 
> (besides
> building all packages in clean staging dirs, E.G. only populated with
> its explicit dependencies like afaik OE lite can do, but that would
> require quite some work), anyone?

I may have an idea to detect cases where a package has a non declared 
dependency.

Inotify may help us to know which files are accessed during build of 
one package.
Especially, we can know which files in staging/ are accessed. If we 
knew which
package own which file under staging and we may find what are 
dependencies
of a package

I already made a system to monitor which files are installed by each
package 
(http://comments.gmane.org/gmane.comp.lib.uclibc.buildroot/53094). It
may be adapted to detect owner of staging/ files

So, process would be:

   make allyespackageconfig
   make        # And monitor which package create which file in staging/
   for PKG in $PACKAGES[@]; do
      make $PKG-dirclean
      make $PKG # And monitor which files are accessed in staging/
      # process result to get $PKG dependencies
   done

It would not fix automatically the problem, but an autobuilder may send
notifications. Inotify instrumentation may stay in a separate patch or 
branch
to not be intrusive in Buildroot.

Do you think it would be useful?

(I will not have enough time to implement this until end of October, at
least)

-- 
J?r?me Pouiller, Sysmic
Embedded Linux specialist
http://www.sysmic.fr

[Buildroot] [PATCH v2] Fix build reproducibility in Make 3.82

[Jérôme Pouiller]

Make 3.82 do not sort anymore result of wildcards (see
http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break
build reproducibility.

This patch sort results of wildcards to ensure reproducibility.

Signed-off-by: J?r?me Pouiller <jezz@sysmic.org>
---
 V2:
    Add forgotten file support/dependencies/dependencies.mk. Even if it
    does not make difference for the build result, I prefer to be 
    consistent.

 Makefile                               |    2 +-
 boot/common.mk                         |    2 +-
 docs/manual/manual.mk                  |    2 +-
 fs/common.mk                           |    2 +-
 linux/linux.mk                         |    2 +-
 package/efl/efl.mk                     |    2 +-
 package/freescale-imx/freescale-imx.mk |    2 +-
 package/gcc/gcc.mk                     |    2 +-
 package/gtk2-themes/gtk2-themes.mk     |    2 +-
 package/matchbox/matchbox.mk           |    2 +-
 package/opengl/opengl.mk               |    2 +-
 package/qt5/qt5.mk                     |    2 +-
 package/x11r7/x11r7.mk                 |    2 +-
 support/dependencies/dependencies.mk   |    2 +-
 14 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/Makefile b/Makefile
index 09faeba..f547af5 100644
--- a/Makefile
+++ b/Makefile
@@ -333,7 +333,7 @@ ifneq ($(PACKAGE_OVERRIDE_FILE),)
 -include $(PACKAGE_OVERRIDE_FILE)
 endif
 
-include package/*/*.mk
+include $(sort $(wildcard package/*/*.mk))
 
 include boot/common.mk
 include linux/linux.mk
diff --git a/boot/common.mk b/boot/common.mk
index b315fe8..3021e51 100644
--- a/boot/common.mk
+++ b/boot/common.mk
@@ -1 +1 @@
-include boot/*/*.mk
+include $(sort $(wildcard boot/*/*.mk))
diff --git a/docs/manual/manual.mk b/docs/manual/manual.mk
index 4906bc8..bcbedad 100644
--- a/docs/manual/manual.mk
+++ b/docs/manual/manual.mk
@@ -53,5 +53,5 @@ $(1)-clean:
 .PHONY: $(1) $(1)-clean manual-update-lists
 endef
 
-MANUAL_SOURCES = $(wildcard docs/manual/*.txt) $(wildcard docs/images/*)
+MANUAL_SOURCES = $(sort $(wildcard docs/manual/*.txt) $(wildcard docs/images/*))
 $(eval $(call GENDOC,manual))
diff --git a/fs/common.mk b/fs/common.mk
index 36da2dd..4dab7ea 100644
--- a/fs/common.mk
+++ b/fs/common.mk
@@ -94,4 +94,4 @@ define ROOTFS_TARGET
 $(call ROOTFS_TARGET_INTERNAL,$(1),$(call UPPERCASE,$(1)))
 endef
 
-include fs/*/*.mk
+include $(sort $(wildcard fs/*/*.mk))
diff --git a/linux/linux.mk b/linux/linux.mk
index 025fd51..8508a54 100644
--- a/linux/linux.mk
+++ b/linux/linux.mk
@@ -281,7 +281,7 @@ define LINUX_INSTALL_TARGET_CMDS
 	$(LINUX_INSTALL_HOST_TOOLS)
 endef
 
-include linux/linux-ext-*.mk
+include $(sort $(wildcard linux/linux-ext-*.mk))
 
 $(eval $(generic-package))
 
diff --git a/package/efl/efl.mk b/package/efl/efl.mk
index f284c93..1de68cf 100644
--- a/package/efl/efl.mk
+++ b/package/efl/efl.mk
@@ -1,3 +1,3 @@
 EFL_VERSION = 1.7.7
 
-include package/efl/*/*.mk
+include $(sort $(wildcard package/efl/*/*.mk))
diff --git a/package/freescale-imx/freescale-imx.mk b/package/freescale-imx/freescale-imx.mk
index 9f63ce3..c8f99b6 100644
--- a/package/freescale-imx/freescale-imx.mk
+++ b/package/freescale-imx/freescale-imx.mk
@@ -9,5 +9,5 @@ FREESCALE_IMX_VERSION = 1.1.0
 # No official download site from freescale, just this mirror
 FREESCALE_IMX_MIRROR_SITE   = http://download.ossystems.com.br/bsp/freescale/source
 
-include package/freescale-imx/*/*.mk
+include $(sort $(wildcard package/freescale-imx/*/*.mk))
 
diff --git a/package/gcc/gcc.mk b/package/gcc/gcc.mk
index cd6efed..66af425 100644
--- a/package/gcc/gcc.mk
+++ b/package/gcc/gcc.mk
@@ -215,4 +215,4 @@ HOST_GCC_COMMON_CONF_OPT += \
 	--with-long-double-128
 endif
 
-include package/gcc/*/*.mk
+include $(sort $(wildcard package/gcc/*/*.mk))
diff --git a/package/gtk2-themes/gtk2-themes.mk b/package/gtk2-themes/gtk2-themes.mk
index 6f31c6d..fc66698 100644
--- a/package/gtk2-themes/gtk2-themes.mk
+++ b/package/gtk2-themes/gtk2-themes.mk
@@ -1 +1 @@
-include package/gtk2-themes/*/*.mk
+include $(sort $(wildcard package/gtk2-themes/*/*.mk))
diff --git a/package/matchbox/matchbox.mk b/package/matchbox/matchbox.mk
index abda915..fff4add 100644
--- a/package/matchbox/matchbox.mk
+++ b/package/matchbox/matchbox.mk
@@ -1,4 +1,4 @@
 ifeq ($(BR2_PACKAGE_MATCHBOX),y)
-include package/matchbox/*/*.mk
+include $(sort $(wildcard package/matchbox/*/*.mk))
 TARGETS+=matchbox-lib matchbox-wm
 endif
diff --git a/package/opengl/opengl.mk b/package/opengl/opengl.mk
index 68df3b1..abf96d5 100644
--- a/package/opengl/opengl.mk
+++ b/package/opengl/opengl.mk
@@ -1 +1 @@
-include package/opengl/*/*.mk
+include $(sort $(wildcard package/opengl/*/*.mk))
diff --git a/package/qt5/qt5.mk b/package/qt5/qt5.mk
index 9e71c83..fce0ca3 100644
--- a/package/qt5/qt5.mk
+++ b/package/qt5/qt5.mk
@@ -1,6 +1,6 @@
 QT5_VERSION = 5.0.2
 QT5_SITE = http://download.qt-project.org/archive/qt/5.0/$(QT5_VERSION)/submodules/
-include package/qt5/*/*.mk
+include $(sort $(wildcard package/qt5/*/*.mk))
 
 define QT5_LA_PRL_FILES_FIXUP
 	for i in $$(find $(STAGING_DIR)/usr/lib* -name "libQt5*.la"); do \
diff --git a/package/x11r7/x11r7.mk b/package/x11r7/x11r7.mk
index ea0363f..85ecbc5 100644
--- a/package/x11r7/x11r7.mk
+++ b/package/x11r7/x11r7.mk
@@ -1 +1 @@
-include package/x11r7/*/*.mk
+include $(sort $(wildcard package/x11r7/*/*.mk))
diff --git a/support/dependencies/dependencies.mk b/support/dependencies/dependencies.mk
index 4a220e0..149b8e5 100644
--- a/support/dependencies/dependencies.mk
+++ b/support/dependencies/dependencies.mk
@@ -14,7 +14,7 @@ DEPENDENCIES_HOST_PREREQ :=
 define suitable-host-package
 $(shell support/dependencies/check-host-$(1).sh $(2))
 endef
--include support/dependencies/check-host-*.mk
+-include $(sort $(wildcard support/dependencies/check-host-*.mk))
 
 ifeq ($(BR2_STRIP_sstrip),y)
 DEPENDENCIES_HOST_PREREQ+=host-sstrip
-- 
1.7.9.5

[Buildroot] [PATCH v2] Fix build reproducibility in Make 3.82

[Arnout Vandecappelle]

On 09/03/13 10:45, J?r?me Pouiller wrote:
> Make 3.82 do not sort anymore result of wildcards (see
> http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break
> build reproducibility.
>
> This patch sort results of wildcards to ensure reproducibility.
>
> Signed-off-by: J?r?me Pouiller<jezz@sysmic.org>

  You forgot

Acked-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>

:-)

> ---
>   V2:
>      Add forgotten file support/dependencies/dependencies.mk. Even if it
>      does not make difference for the build result, I prefer to be
>      consistent.

  I agree!

  Regards,
  Arnout

-- 
Arnout Vandecappelle                          arnout at mind be
Senior Embedded Software Architect            +32-16-286500
Essensium/Mind                                http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium           BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F

[Buildroot] Build reproducibility

[Thomas Petazzoni]

Dear Peter Korsgaard,

(Adding Fabio to the CC list)

On Tue, 03 Sep 2013 09:47:14 +0200, Peter Korsgaard wrote:

>  Thomas> This would certainly be nice to have (as it also helps
>  Thomas> top-level parallel build, as was discussed with Fabio
>  Thomas> Porcedda some time ago), but:
> 
> [snip]
> 
> Indeed. For the record, I also don't think we should do this, I was
> just stating what was needed if we wanted to do so.
> 
> As long as the dependencies are correct, I don't think seperate
> staging dirs are needed for toplevel parallel builds.

I disagree on this one. Since there is no way to be sure that all the
possible optional dependencies have been taken into account, I believe
separate staging dirs are a requirement to ensure that toplevel
parallel builds are reproducible.

Whenever we bump a package, some additional optional dependencies may
have been added by the upstream developers, and we rarely go deep into
the configure.ac script or other build system files to verify that the
addition or removal of optional dependencies. Having to do such review
work would be very time-consuming and terribly annoying. So the only
way to be sure that the optional dependencies are all properly taken
into account when doing top-level parallel build is by ensuring that
the staging dir used when building a package only contains the
dependencies that have been explicitly listed by the package .mk file.

Best regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] Build reproducibility

[Thomas Petazzoni]

Dear J?r?me Pouiller,

(Your e-mail client has wrapped the e-mail in a somewhat weird fashion,
leaving only one word one every two lines. Ah, yes a Roundcube Webmail!)

On Tue, 03 Sep 2013 10:15:55 +0200, J?r?me Pouiller wrote:
> I may have an idea to detect cases where a package has a non declared 
> dependency.
> 
> Inotify may help us to know which files are accessed during build of 
> one package.
> Especially, we can know which files in staging/ are accessed. If we 
> knew which
> package own which file under staging and we may find what are 
> dependencies
> of a package
> 
> I already made a system to monitor which files are installed by each
> package 
> (http://comments.gmane.org/gmane.comp.lib.uclibc.buildroot/53094). It
> may be adapted to detect owner of staging/ files

This indeed looks interesting. I'm not sure it's really easy to do (on
one side, we need to get the list of files that are installed/owned by
each package, and then whenever a file is accessed, check whether this
file is owned by a package which is part of the dependencies of the
currently built package), but if it's possible, it would quite
certainly raise a number of interesting warnings/issues.

>    make allyespackageconfig

Note that an allyespackageconfig doesn't build all packages: they are
some choices in the Buildroot Config.in options, and some packages
depend on the value of those choices. Hence, in a given configuration,
it is not possible to really enable and test all packages.

>    make        # And monitor which package create which file in
> staging/ for PKG in $PACKAGES[@]; do
>       make $PKG-dirclean
>       make $PKG # And monitor which files are accessed in staging/
>       # process result to get $PKG dependencies

I don't think we should try to infer automatically the dependencies,
but at least show a report of files that were accessed even though they
don't belong to packages that are part of the dependencies.

>    done
> 
> It would not fix automatically the problem, but an autobuilder may
> send notifications. Inotify instrumentation may stay in a separate
> patch or branch to not be intrusive in Buildroot.
> 
> Do you think it would be useful?

It might certainly be useful to do some experiments around this idea
and see if it brings a reasonable result (both in terms of code to
merge, and in terms of warnings/issues that we are able to discover and
fix).

Thanks,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] Build reproducibility

[Thomas Petazzoni]

Dear Thomas De Schampheleire,

On Mon, 2 Sep 2013 15:18:09 +0200, Thomas De Schampheleire wrote:
> > Of course, if within the Buildroot project we are interested in
> > fixing such missing dependencies, then we can find a way of adding
> > randomness into the build order in our autobuilders. But clearly,
> > we do want to expose this randomness to our users.
> 
> I think indeed we should try to set the dependencies right some way
> or another.
> 
> If we assume that a package does not have any configurable options
> that would change its dependencies, a simple way to check if all
> dependencies are properly expressed is through:
> make clean toolchain foo

This is already done by the autobuilders. Thanks to the randomness of
the configuration, if a package fails to express a mandatory
dependency, sooner or later the autobuilders will generate a
configuration that has the package enabled but not one of its unknown
dependencies. The autobuilders have triggered such cases very quickly
in the past when a new package was added, so I'm pretty confident we
have good coverage on this one.

However, I think this suggestion misses the point of the current
discussion: we are talking about *optional* dependencies, i.e
functionality of a given package that are enabled if OpenSSL is
available, or disabled if OpenSSL is not available. Those optional
dependencies cannot be checked/discovered by a build such as "make
clean toolchain foo".

> Also, it's not necessary that each package gets build every night:
> once the dependencies are correct, they will stay correct until a
> version bump. This means we can spread out the execution of this type
> of tests over time, interleaving them with the already existing
> autobuilds with random configurations.

As stated above, for the mandatory dependencies, the autobuilders are
already finding them very quickly. Introduce a new package that lacks a
mandatory dependency expressed in its .mk file, and you'll see it very
quickly in the autobuilders.

> The make targets of buildroot itself are executed sequentially.
> Suppose that we keep a list of all targets executed, something like:
> python-source
> python-extract
> python-patch
> python-configure
> python-build
> python-install-target
> pyfoo-source
> pyfoo-extract
> pyfoo-build
> ...
> 
> To reproduce a build, we can explicitly pass this list on the make
> command-line, roughly like:
> cat <target-list> | xargs make clean toolchain

Could be doable.

I've lost track. Are we talking about all of this to use "include
package/*/*.mk" in make 3.82 which doesn't guarantee sorting, or to be
able to do top-level parallel build?

Seeing how simple it is to get make 3.82 to behave like make 3.81 by
sorting the "include package/*/*.mk" inclusions, I don't think it's
worth doing anything but the fix that J?r?me has suggested.

Only the top-level parallel build would be a good enough to worry about
reproducibility of more complicated builds (and therefore the need to
list in which order the targets were built).

Best regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] Build reproducibility

[Thomas De Schampheleire]

Hi Thomas,

On Tue, Sep 3, 2013 at 7:13 PM, Thomas Petazzoni
<thomas.petazzoni@free-electrons.com> wrote:
> Dear Thomas De Schampheleire,
>
> On Mon, 2 Sep 2013 15:18:09 +0200, Thomas De Schampheleire wrote:
>> > Of course, if within the Buildroot project we are interested in
>> > fixing such missing dependencies, then we can find a way of adding
>> > randomness into the build order in our autobuilders. But clearly,
>> > we do want to expose this randomness to our users.
>>
>> I think indeed we should try to set the dependencies right some way
>> or another.
>>
>> If we assume that a package does not have any configurable options
>> that would change its dependencies, a simple way to check if all
>> dependencies are properly expressed is through:
>> make clean toolchain foo
>
> This is already done by the autobuilders. Thanks to the randomness of
> the configuration, if a package fails to express a mandatory
> dependency, sooner or later the autobuilders will generate a
> configuration that has the package enabled but not one of its unknown
> dependencies. The autobuilders have triggered such cases very quickly
> in the past when a new package was added, so I'm pretty confident we
> have good coverage on this one.

I'm currently running 'make clean toolchain <pkg>' for each package,
and preliminary results have already found several dependency
problems. I will analyze the results when it's done and submit patches
for them.
The advantage of such a non-random test is that it will find problems
faster than the autobuilders.

>
> However, I think this suggestion misses the point of the current
> discussion: we are talking about *optional* dependencies, i.e
> functionality of a given package that are enabled if OpenSSL is
> available, or disabled if OpenSSL is not available. Those optional
> dependencies cannot be checked/discovered by a build such as "make
> clean toolchain foo".

Correct. When I wrote my mail, I hadn't though about such optional dependencies.

>
>> Also, it's not necessary that each package gets build every night:
>> once the dependencies are correct, they will stay correct until a
>> version bump. This means we can spread out the execution of this type
>> of tests over time, interleaving them with the already existing
>> autobuilds with random configurations.
>
> As stated above, for the mandatory dependencies, the autobuilders are
> already finding them very quickly. Introduce a new package that lacks a
> mandatory dependency expressed in its .mk file, and you'll see it very
> quickly in the autobuilders.
>
>> The make targets of buildroot itself are executed sequentially.
>> Suppose that we keep a list of all targets executed, something like:
>> python-source
>> python-extract
>> python-patch
>> python-configure
>> python-build
>> python-install-target
>> pyfoo-source
>> pyfoo-extract
>> pyfoo-build
>> ...
>>
>> To reproduce a build, we can explicitly pass this list on the make
>> command-line, roughly like:
>> cat <target-list> | xargs make clean toolchain
>
> Could be doable.
>
> I've lost track. Are we talking about all of this to use "include
> package/*/*.mk" in make 3.82 which doesn't guarantee sorting, or to be
> able to do top-level parallel build?

The original mail was about the wildcard-make-3.82-thingy. You were
the one that dragged in the parallel build situation :-)

>
> Seeing how simple it is to get make 3.82 to behave like make 3.81 by
> sorting the "include package/*/*.mk" inclusions, I don't think it's
> worth doing anything but the fix that J?r?me has suggested.

Agreed.

>
> Only the top-level parallel build would be a good enough to worry about
> reproducibility of more complicated builds (and therefore the need to
> list in which order the targets were built).

Although such a list could help in reproducing a parallel build, in
the context of making buildroot work successfully with top-level
parallel build, it is just a side-problem I think. Do you think we
should be doing something about it right now?

Best regards,
Thomas

[Buildroot] Build reproducibility

[Jérôme Pouiller]

Hello Thomas,

On Thursday 05 September 2013 21:56:01 Thomas De Schampheleire wrote:
> On Tue, Sep 3, 2013 at 7:13 PM, Thomas Petazzoni
> <thomas.petazzoni@free-electrons.com> wrote:
> > On Mon, 2 Sep 2013 15:18:09 +0200, Thomas De Schampheleire wrote:
> >> > Of course, if within the Buildroot project we are interested in
> >> > fixing such missing dependencies, then we can find a way of adding
> >> > randomness into the build order in our autobuilders. But clearly,
> >> > we do want to expose this randomness to our users.
> >> 
> >> I think indeed we should try to set the dependencies right some way
> >> or another.
> >> 
> >> If we assume that a package does not have any configurable options
> >> that would change its dependencies, a simple way to check if all
> >> dependencies are properly expressed is through:
> >> make clean toolchain foo
> > 
> > This is already done by the autobuilders. Thanks to the randomness of
> > the configuration, if a package fails to express a mandatory
> > dependency, sooner or later the autobuilders will generate a
> > configuration that has the package enabled but not one of its unknown
> > dependencies. The autobuilders have triggered such cases very quickly
> > in the past when a new package was added, so I'm pretty confident we
> > have good coverage on this one.
> 
> I'm currently running 'make clean toolchain <pkg>' for each package,
> and preliminary results have already found several dependency
> problems. I will analyze the results when it's done and submit patches
> for them.
> The advantage of such a non-random test is that it will find problems
> faster than the autobuilders.

Just to avoid duplicate work, I am currently testing an automated system to 
detect dependencies of packages (based on inotify as I explain in my previous 
mail). First results can be found there:
http://www.sysmic.org/~jezz/dependencies.cooked

-- 
J?r?me Pouiller, Sysmic
Embedded Linux specialist
http://www.sysmic.fr

[Buildroot] [PATCH v2] Fix build reproducibility in Make 3.82

[Peter Korsgaard]

>>>>> "J?r?me" == J?r?me Pouiller <jezz@sysmic.org> writes:

 J?r?me> Make 3.82 do not sort anymore result of wildcards (see
 J?r?me> http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break
 J?r?me> build reproducibility.

 J?r?me> This patch sort results of wildcards to ensure reproducibility.

 J?r?me> Signed-off-by: J?r?me Pouiller <jezz@sysmic.org>
 J?r?me> ---
 J?r?me>  V2:
 J?r?me>     Add forgotten file support/dependencies/dependencies.mk. Even if it
 J?r?me>     does not make difference for the build result, I prefer to be 
 J?r?me>     consistent.

Committed, thanks.

-- 
Bye, Peter Korsgaard