[PATCH 1/4] buildman: Add a note about the out-env file

[PATCH 1/4] buildman: Add a note about the out-env file

[Simon Glass]

This file holds the environment used when doing a build. Add a note
about it.

Signed-off-by: Simon Glass <sjg@chromium.org>
---

 tools/buildman/buildman.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst
index 2a83cb7e4f8..9a2d913c785 100644
--- a/tools/buildman/buildman.rst
+++ b/tools/buildman/buildman.rst
@@ -1108,6 +1108,8 @@ and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a
 value for 'altbootcmd', but lost one for ' altbootcmd'.
 
 The -U option uses the u-boot.env files which are produced by a build.
+Internally, buildman writes out an out-env file into the build directory for
+later comparison.
 
 
 Building with clang
-- 
2.39.2.637.g21b0678d19-goog

[PATCH 2/4] buildman: Write out the build command used

[Simon Glass]

It is sometimes useful to see the exact 'make' command used by buildman
for a commit. Add an output file for this.

Signed-off-by: Simon Glass <sjg@chromium.org>
---

 tools/buildman/builderthread.py | 13 +++++++++++++
 tools/buildman/buildman.rst     |  8 ++++++++
 tools/buildman/func_test.py     | 13 +++++++++++++
 3 files changed, 34 insertions(+)

diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py
index 680efae02d7..7ba9a856dd5 100644
--- a/tools/buildman/builderthread.py
+++ b/tools/buildman/builderthread.py
@@ -273,14 +273,19 @@ class BuilderThread(threading.Thread):
 
                 # If we need to reconfigure, do that now
                 cfg_file = os.path.join(out_dir, '.config')
+                cmd_list = []
                 if do_config or adjust_cfg:
                     config_out = ''
                     if self.mrproper:
                         result = self.Make(commit, brd, 'mrproper', cwd,
                                 'mrproper', *args, env=env)
                         config_out += result.combined
+                        cmd_list.append([self.builder.gnu_make, 'mrproper',
+                                         *args])
                     result = self.Make(commit, brd, 'config', cwd,
                             *(args + config_args), env=env)
+                    cmd_list.append([self.builder.gnu_make] + args +
+                                    config_args)
                     config_out += result.combined
                     do_config = False   # No need to configure next time
                     if adjust_cfg:
@@ -290,6 +295,7 @@ class BuilderThread(threading.Thread):
                         args.append('cfg')
                     result = self.Make(commit, brd, 'build', cwd, *args,
                             env=env)
+                    cmd_list.append([self.builder.gnu_make] + args)
                     if (result.return_code == 2 and
                         ('Some images are invalid' in result.stderr)):
                         # This is handled later by the check for output in
@@ -303,6 +309,7 @@ class BuilderThread(threading.Thread):
                 result.stderr = result.stderr.replace(src_dir + '/', '')
                 if self.builder.verbose_build:
                     result.stdout = config_out + result.stdout
+                result.cmd_list = cmd_list
             else:
                 result.return_code = 1
                 result.stderr = 'No tool chain for %s\n' % brd.arch
@@ -378,6 +385,12 @@ class BuilderThread(threading.Thread):
             with open(os.path.join(build_dir, 'out-env'), 'wb') as fd:
                 for var in sorted(env.keys()):
                     fd.write(b'%s="%s"' % (var, env[var]))
+
+            with open(os.path.join(build_dir, 'out-cmd'), 'w',
+                      encoding='utf-8') as fd:
+                for cmd in result.cmd_list:
+                    print(' '.join(cmd), file=fd)
+
             lines = []
             for fname in BASE_ELF_FILENAMES:
                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst
index 9a2d913c785..11c72141791 100644
--- a/tools/buildman/buildman.rst
+++ b/tools/buildman/buildman.rst
@@ -1300,6 +1300,14 @@ You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file,
 since it may be dropped altogether in future.
 
 
+Checking the command
+--------------------
+
+Buildman writes out the toolchain information to a `toolchain` file within the
+output directory. It also writes the commands used to build U-Boot in an
+`out-cmd` file. You can check these if you suspect something strange is
+happening.
+
 TODO
 ----
 
diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
index 559e4edf74b..799c609446e 100644
--- a/tools/buildman/func_test.py
+++ b/tools/buildman/func_test.py
@@ -723,3 +723,16 @@ Some images are invalid'''
                          control.get_allow_missing(False, False, 2, True))
         self.assertEqual(False,
                          control.get_allow_missing(False, True, 2, True))
+
+    def testCmdFile(self):
+        """Test that the -cmd-out file is produced"""
+        self._RunControl('-o', self._output_dir)
+        board0_dir = os.path.join(self._output_dir, 'current', 'board0')
+        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done')))
+        cmd_fname = os.path.join(board0_dir, 'out-cmd')
+        self.assertTrue(os.path.exists(cmd_fname))
+        data = tools.read_file(cmd_fname)
+        lines = data.splitlines()
+        self.assertEqual(2, len(lines))
+        self.assertRegex(lines[0], b'make O=/.*board0_defconfig')
+        self.assertRegex(lines[0], b'make O=/.*-s.*')
-- 
2.39.2.637.g21b0678d19-goog

[PATCH 3/4] buildman: Support disabling LTO

[Simon Glass]

This cuts down build performance considerably and is not always needed,
when checking for build errors, etc.

Add a flag to disable it.

Signed-off-by: Simon Glass <sjg@chromium.org>
---

 tools/buildman/builder.py       |  5 ++++-
 tools/buildman/builderthread.py |  2 ++
 tools/buildman/buildman.rst     | 14 ++++++++++++++
 tools/buildman/cmdline.py       |  2 ++
 tools/buildman/control.py       |  2 +-
 tools/buildman/func_test.py     | 25 +++++++++++++++++++++----
 6 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py
index c2a69027f88..107086cc0e5 100644
--- a/tools/buildman/builder.py
+++ b/tools/buildman/builder.py
@@ -194,6 +194,7 @@ class Builder:
         work_in_output: Use the output directory as the work directory and
             don't write to a separate output directory.
         thread_exceptions: List of exceptions raised by thread jobs
+        no_lto (bool): True to set the NO_LTO flag when building
 
     Private members:
         _base_board_dict: Last-summarised Dict of boards
@@ -253,7 +254,7 @@ class Builder:
                  config_only=False, squash_config_y=False,
                  warnings_as_errors=False, work_in_output=False,
                  test_thread_exceptions=False, adjust_cfg=None,
-                 allow_missing=False):
+                 allow_missing=False, no_lto=False):
         """Create a new Builder object
 
         Args:
@@ -292,6 +293,7 @@ class Builder:
                     C=val to set the value of C (val must have quotes if C is
                         a string Kconfig
             allow_missing: Run build with BINMAN_ALLOW_MISSING=1
+            no_lto (bool): True to set the NO_LTO flag when building
 
         """
         self.toolchains = toolchains
@@ -331,6 +333,7 @@ class Builder:
         self.adjust_cfg = adjust_cfg
         self.allow_missing = allow_missing
         self._ide = False
+        self.no_lto = no_lto
 
         if not self.squash_config_y:
             self.config_filenames += EXTRA_CONFIG_FILENAMES
diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py
index 7ba9a856dd5..dae3d4ab9ff 100644
--- a/tools/buildman/builderthread.py
+++ b/tools/buildman/builderthread.py
@@ -255,6 +255,8 @@ class BuilderThread(threading.Thread):
                     args.append('KCFLAGS=-Werror')
                 if self.builder.allow_missing:
                     args.append('BINMAN_ALLOW_MISSING=1')
+                if self.builder.no_lto:
+                    args.append('NO_LTO=1')
                 config_args = ['%s_defconfig' % brd.target]
                 config_out = ''
                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst
index 11c72141791..800b83a89de 100644
--- a/tools/buildman/buildman.rst
+++ b/tools/buildman/buildman.rst
@@ -1123,6 +1123,20 @@ toolchain. For example:
    buildman -O clang-7 --board sandbox
 
 
+Building without LTO
+--------------------
+
+Link-time optimisation (LTO) is designed to reduce code size by globally
+optimising the U-Boot build. Unfortunately this can dramatically slow down
+builds. This is particularly noticeable when running a lot of builds.
+
+Use the -L (--no-lto) flag to disable LTO.
+
+.. code-block:: bash
+
+   buildman -L --board sandbox
+
+
 Doing a simple build
 --------------------
 
diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py
index c485994e9fe..409013671be 100644
--- a/tools/buildman/cmdline.py
+++ b/tools/buildman/cmdline.py
@@ -71,6 +71,8 @@ def ParseArgs():
           default=False, help="Don't convert y to 1 in configs")
     parser.add_option('-l', '--list-error-boards', action='store_true',
           default=False, help='Show a list of boards next to each error/warning')
+    parser.add_option('-L', '--no-lto', action='store_true',
+          default=False, help='Disable Link-time Optimisation (LTO) for builds')
     parser.add_option('--list-tool-chains', action='store_true', default=False,
           help='List available tool chains (use -v to see probing detail)')
     parser.add_option('-m', '--mrproper', action='store_true',
diff --git a/tools/buildman/control.py b/tools/buildman/control.py
index 87e7d0e2012..13a462ae595 100644
--- a/tools/buildman/control.py
+++ b/tools/buildman/control.py
@@ -351,7 +351,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
             work_in_output=options.work_in_output,
             test_thread_exceptions=test_thread_exceptions,
             adjust_cfg=adjust_cfg,
-            allow_missing=allow_missing)
+            allow_missing=allow_missing, no_lto=options.no_lto)
     builder.force_config_on_failure = not options.quick
     if make_func:
         builder.do_make = make_func
diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
index 799c609446e..6d1557293f0 100644
--- a/tools/buildman/func_test.py
+++ b/tools/buildman/func_test.py
@@ -724,15 +724,32 @@ Some images are invalid'''
         self.assertEqual(False,
                          control.get_allow_missing(False, True, 2, True))
 
-    def testCmdFile(self):
-        """Test that the -cmd-out file is produced"""
-        self._RunControl('-o', self._output_dir)
+    def check_command(self, *extra_args):
+        """Run a command with the extra arguments and return the commands used
+
+        Args:
+            extra_args (list of str): List of extra arguments
+
+        Returns:
+            list of str: Lines returned in the out-cmd file
+        """
+        self._RunControl('-o', self._output_dir, *extra_args)
         board0_dir = os.path.join(self._output_dir, 'current', 'board0')
         self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done')))
         cmd_fname = os.path.join(board0_dir, 'out-cmd')
         self.assertTrue(os.path.exists(cmd_fname))
         data = tools.read_file(cmd_fname)
-        lines = data.splitlines()
+        return data.splitlines()
+
+    def testCmdFile(self):
+        """Test that the -cmd-out file is produced"""
+        lines = self.check_command()
         self.assertEqual(2, len(lines))
         self.assertRegex(lines[0], b'make O=/.*board0_defconfig')
         self.assertRegex(lines[0], b'make O=/.*-s.*')
+
+    def testNoLto(self):
+        """Test that the --no-lto flag works"""
+        lines = self.check_command('-L')
+        self.assertIn(b'NO_LTO=1', lines[0])
+
-- 
2.39.2.637.g21b0678d19-goog

[PATCH 4/4] buildman: Add a flag for reproducible builds

[Simon Glass]

This is quite a useful thing to use when building since it avoids small
size changes between commits. Add a -r flag for it.

Signed-off-by: Simon Glass <sjg@chromium.org>
---

 doc/build/reproducible.rst      | 2 ++
 tools/buildman/builder.py       | 4 +++-
 tools/buildman/builderthread.py | 2 ++
 tools/buildman/buildman.rst     | 7 ++++---
 tools/buildman/cmdline.py       | 2 ++
 tools/buildman/control.py       | 3 ++-
 tools/buildman/func_test.py     | 4 ++++
 7 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/doc/build/reproducible.rst b/doc/build/reproducible.rst
index 5423080633e..8b030f469d7 100644
--- a/doc/build/reproducible.rst
+++ b/doc/build/reproducible.rst
@@ -23,3 +23,5 @@ This date is shown when we launch U-Boot:
 
     ./u-boot -T
     U-Boot 2023.01 (Jan 01 2023 - 00:00:00 +0000)
+
+The same effect can be obtained with buildman using the `-r` flag.
diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py
index 107086cc0e5..7b9be887e55 100644
--- a/tools/buildman/builder.py
+++ b/tools/buildman/builder.py
@@ -195,6 +195,7 @@ class Builder:
             don't write to a separate output directory.
         thread_exceptions: List of exceptions raised by thread jobs
         no_lto (bool): True to set the NO_LTO flag when building
+        reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds
 
     Private members:
         _base_board_dict: Last-summarised Dict of boards
@@ -254,7 +255,7 @@ class Builder:
                  config_only=False, squash_config_y=False,
                  warnings_as_errors=False, work_in_output=False,
                  test_thread_exceptions=False, adjust_cfg=None,
-                 allow_missing=False, no_lto=False):
+                 allow_missing=False, no_lto=False, reproducible_builds=False):
         """Create a new Builder object
 
         Args:
@@ -334,6 +335,7 @@ class Builder:
         self.allow_missing = allow_missing
         self._ide = False
         self.no_lto = no_lto
+        self.reproducible_builds = reproducible_builds
 
         if not self.squash_config_y:
             self.config_filenames += EXTRA_CONFIG_FILENAMES
diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py
index dae3d4ab9ff..8b88c68e5d2 100644
--- a/tools/buildman/builderthread.py
+++ b/tools/buildman/builderthread.py
@@ -257,6 +257,8 @@ class BuilderThread(threading.Thread):
                     args.append('BINMAN_ALLOW_MISSING=1')
                 if self.builder.no_lto:
                     args.append('NO_LTO=1')
+                if self.builder.reproducible_builds:
+                    args.append('SOURCE_DATE_EPOCH=0')
                 config_args = ['%s_defconfig' % brd.target]
                 config_out = ''
                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst
index 800b83a89de..c8b0db3d8b9 100644
--- a/tools/buildman/buildman.rst
+++ b/tools/buildman/buildman.rst
@@ -1023,14 +1023,15 @@ U-Boot's build system embeds information such as a build timestamp into the
 final binary. This information varies each time U-Boot is built. This causes
 various files to be rebuilt even if no source changes are made, which in turn
 requires that the final U-Boot binary be re-linked. This unnecessary work can
-be avoided by turning off the timestamp feature. This can be achieved by
-setting the SOURCE_DATE_EPOCH environment variable to 0.
+be avoided by turning off the timestamp feature. This can be achieved using
+the `-r` flag, which enables reproducible builds by setting
+`SOURCE_DATE_EPOCH=0` when building.
 
 Combining all of these options together yields the command-line shown below.
 This will provide the quickest possible feedback regarding the current content
 of the source tree, thus allowing rapid tested evolution of the code::
 
-    SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra
+    ./tools/buildman/buildman -Pr tegra
 
 
 Checking configuration
diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py
index 409013671be..da7f1a99f6b 100644
--- a/tools/buildman/cmdline.py
+++ b/tools/buildman/cmdline.py
@@ -97,6 +97,8 @@ def ParseArgs():
           default=False, help="Use full toolchain path in CROSS_COMPILE")
     parser.add_option('-P', '--per-board-out-dir', action='store_true',
           default=False, help="Use an O= (output) directory per board rather than per thread")
+    parser.add_option('-r', '--reproducible-builds', action='store_true',
+          help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build')
     parser.add_option('-R', '--regen-board-list', action='store_true',
           help='Force regeneration of the list of boards, like the old boards.cfg file')
     parser.add_option('-s', '--summary', action='store_true',
diff --git a/tools/buildman/control.py b/tools/buildman/control.py
index 13a462ae595..0d7c5f5269f 100644
--- a/tools/buildman/control.py
+++ b/tools/buildman/control.py
@@ -351,7 +351,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
             work_in_output=options.work_in_output,
             test_thread_exceptions=test_thread_exceptions,
             adjust_cfg=adjust_cfg,
-            allow_missing=allow_missing, no_lto=options.no_lto)
+            allow_missing=allow_missing, no_lto=options.no_lto,
+            reproducible_builds=options.reproducible_builds)
     builder.force_config_on_failure = not options.quick
     if make_func:
         builder.do_make = make_func
diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
index 6d1557293f0..47fca33002d 100644
--- a/tools/buildman/func_test.py
+++ b/tools/buildman/func_test.py
@@ -753,3 +753,7 @@ Some images are invalid'''
         lines = self.check_command('-L')
         self.assertIn(b'NO_LTO=1', lines[0])
 
+    def testReproducible(self):
+        """Test that the -r flag works"""
+        lines = self.check_command('-r')
+        self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0])
-- 
2.39.2.637.g21b0678d19-goog