[PATCH 1/3] runqueue: Fix unihash cache mismatch issues

[PATCH 1/3] runqueue: Fix unihash cache mismatch issues

[Richard Purdie]

Very occasionally we see errors in eSDK testing on the autobuilder where the task
hashes in the eSDK don't match what was just built. I was able to inspect one of
these build directories and noticed that the bb_unihashes.dat file in the eSDK
was zero sized. Whilst inspecting the code to understand the cause, I noticed that
updated hashes are not saved out in subsequent updates of the values in the rehash
process.

Add a missing sync call to ensure this happens.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index f34f1568e2..1e47fe70ef 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -2299,6 +2299,9 @@ class RunQueueExecute:
                     self.rqdata.runtaskentries[hashtid].unihash = unihash
                     bb.parse.siggen.set_unihash(hashtid, unihash)
                     toprocess.add(hashtid)
+                if torehash:
+                    # Need to save after set_unihash above
+                    bb.parse.siggen.save_unitaskhashes()
 
         # Work out all tasks which depend upon these
         total = set()
-- 
2.34.1

[PATCH 2/3] cache/siggen: Add unihash cache copy function

[Richard Purdie]

We see rare failures in eSDK generation with zero sized unihash cache files. This is
almost certainly due to races in the cache file being updated. Add a copy function
where the cache file can be copied with the lock held to avoid this.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cache.py  | 9 +++++++++
 lib/bb/siggen.py | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index 92e9a3ced7..988c596c39 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -24,6 +24,7 @@ from collections.abc import Mapping
 import bb.utils
 from bb import PrefixLoggerAdapter
 import re
+import shutil
 
 logger = logging.getLogger("BitBake.Cache")
 
@@ -998,3 +999,11 @@ class SimpleCache(object):
             p.dump([data, self.cacheversion])
 
         bb.utils.unlockfile(glf)
+
+    def copyfile(self, target):
+        if not self.cachefile:
+            return
+
+        glf = bb.utils.lockfile(self.cachefile + ".lock")
+        shutil.copy(self.cachefile, target)
+        bb.utils.unlockfile(glf)
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 08eca7860e..3f3d6df54d 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -120,6 +120,9 @@ class SignatureGenerator(object):
     def save_unitaskhashes(self):
         return
 
+    def copy_unitaskhashes(self, targetdir):
+        return
+
     def set_setscene_tasks(self, setscene_tasks):
         return
 
@@ -358,6 +361,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def save_unitaskhashes(self):
         self.unihash_cache.save(self.unitaskhashes)
 
+    def copy_unitaskhashes(self, targetdir):
+        self.unihash_cache.copyfile(targetdir)
+
     def dump_sigtask(self, fn, task, stampbase, runtime):
 
         tid = fn + ":" + task
-- 
2.34.1

[PATCH 3/3] bitbake: Bump to version 2.0.1

[Richard Purdie]

This allows OE to depend on the unihash copy functionality.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 bin/bitbake        | 2 +-
 lib/bb/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/bitbake b/bin/bitbake
index 042c91807d..b56f6207c6 100755
--- a/bin/bitbake
+++ b/bin/bitbake
@@ -28,7 +28,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "2.0.0"
+__version__ = "2.0.1"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:
diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py
index b8333bdb81..6b470aa195 100644
--- a/lib/bb/__init__.py
+++ b/lib/bb/__init__.py
@@ -9,7 +9,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "2.0.0"
+__version__ = "2.0.1"
 
 import sys
 if sys.version_info < (3, 6, 0):
-- 
2.34.1