[PATCH 1/9] hashserv: Don't daemonize server process

[PATCH 1/9] hashserv: Don't daemonize server process

[Richard Purdie]

From: Joshua Watt <jpewhacker@gmail.com>

The hash server process is terminated and waited on with join(), so it
should not be a daemon. Daemonizing it cause races with the server
cleanup, especially in the selftest because the process may not have
terminated and cleanup up its socket before the test cleanup runs and
tries to do it.

[YOCTO #13542]

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cooker.py      | 1 -
 lib/hashserv/tests.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 0c540028ae..20ef04d3ff 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -399,7 +399,6 @@ class BBCooker:
                 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
                 self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False)
                 self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever)
-                self.hashserv.process.daemon = True
                 self.hashserv.process.start()
             self.data.setVar("BB_HASHSERVE", self.hashservaddr)
             self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr)
diff --git a/lib/hashserv/tests.py b/lib/hashserv/tests.py
index 6584ff57b4..a5472a996d 100644
--- a/lib/hashserv/tests.py
+++ b/lib/hashserv/tests.py
@@ -32,7 +32,6 @@ class TestHashEquivalenceServer(object):
 
         self.server = create_server(self.get_server_addr(), self.dbfile)
         self.server_thread = multiprocessing.Process(target=self._run_server)
-        self.server_thread.daemon = True
         self.server_thread.start()
         self.client = create_client(self.server.address)
 
-- 
2.20.1

[PATCH 2/9] runqueue: Fix task migration problems

[Richard Purdie]

Tasks were not migrating consistently, particularly:

* if a task was rehashed which had already run
* if a task which was valid became invalid due to a rehash

We need to always run the migration code for rehashed tasks and then
reprocess them for hash validity. This means rearranging the code.

It also means several tests are no longer correct and can't be written
correctly to work on all possible workflows so those are removed.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py       | 99 ++++++++++++++++++++++------------------
 lib/bb/tests/runqueue.py | 89 ------------------------------------
 2 files changed, 55 insertions(+), 133 deletions(-)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index d9a67a3167..02160ef4d7 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -73,7 +73,7 @@ def build_tid(mc, fn, taskname):
 def pending_hash_index(tid, rqdata):
     (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
     pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
-    h = rqdata.runtaskentries[tid].hash
+    h = rqdata.runtaskentries[tid].unihash
     return pn + ":" + "taskname" + h
 
 class RunQueueStats:
@@ -2299,9 +2299,6 @@ class RunQueueExecute:
         for tid in changed:
             if tid not in self.rqdata.runq_setscene_tids:
                 continue
-            valid = self.rq.validate_hashes(set([tid]), self.cooker.data, None, False)
-            if not valid:
-                continue
             if tid in self.runq_running:
                 continue
             if tid not in self.pending_migrations:
@@ -2358,6 +2355,7 @@ class RunQueueExecute:
 
             logger.info("Setscene task %s now valid and being rerun" % tid)
             self.sqdone = False
+            update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
 
         if changed:
             self.holdoff_need_update = True
@@ -2674,64 +2672,77 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
 
     rqdata.init_progress_reporter.next_stage()
 
-    multiconfigs = set()
+    sqdata.multiconfigs = set()
     for tid in sqdata.sq_revdeps:
-        multiconfigs.add(mc_from_tid(tid))
+        sqdata.multiconfigs.add(mc_from_tid(tid))
         if len(sqdata.sq_revdeps[tid]) == 0:
             sqrq.sq_buildable.add(tid)
 
     rqdata.init_progress_reporter.finish()
 
-    if rq.hashvalidate:
-        noexec = []
-        stamppresent = []
-        tocheck = set()
+    sqdata.noexec = set()
+    sqdata.stamppresent = set()
+    sqdata.valid = set()
 
-        for tid in sorted(sqdata.sq_revdeps):
-            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+    update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq)
 
-            taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
 
-            if 'noexec' in taskdep and taskname in taskdep['noexec']:
-                noexec.append(tid)
-                sqrq.sq_task_skip(tid)
-                bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
-                continue
+    tocheck = set()
 
-            if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
-                logger.debug(2, 'Setscene stamp current for task %s', tid)
-                stamppresent.append(tid)
-                sqrq.sq_task_skip(tid)
-                continue
+    for tid in sorted(tids):
+        if tid in sqdata.stamppresent:
+            sqdata.stamppresent.remove(tid)
+        if tid in sqdata.valid:
+            sqdata.valid.remove(tid)
 
-            if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
-                logger.debug(2, 'Normal stamp current for task %s', tid)
-                stamppresent.append(tid)
-                sqrq.sq_task_skip(tid)
-                continue
+        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
 
-            tocheck.add(tid)
+        taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
 
-        valid = rq.validate_hashes(tocheck, cooker.data, len(stamppresent), False)
+        if 'noexec' in taskdep and taskname in taskdep['noexec']:
+            sqdata.noexec.add(tid)
+            sqrq.sq_task_skip(tid)
+            bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
+            continue
+
+        if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
+            logger.debug(2, 'Setscene stamp current for task %s', tid)
+            sqdata.stamppresent.add(tid)
+            sqrq.sq_task_skip(tid)
+            continue
 
-        valid_new = stamppresent
-        for v in valid:
-            valid_new.append(v)
+        if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
+            logger.debug(2, 'Normal stamp current for task %s', tid)
+            sqdata.stamppresent.add(tid)
+            sqrq.sq_task_skip(tid)
+            continue
 
-        hashes = {}
-        for mc in sorted(multiconfigs):
-          for tid in sorted(sqdata.sq_revdeps):
+        tocheck.add(tid)
+
+    sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False)
+
+    sqdata.hashes = {}
+    for mc in sorted(sqdata.multiconfigs):
+        for tid in sorted(sqdata.sq_revdeps):
             if mc_from_tid(tid) != mc:
                 continue
-            if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered:
-                sqdata.outrightfail.add(tid)
+            if tid in sqdata.stamppresent:
+                continue
+            if tid in sqdata.valid:
+                continue
+            if tid in sqdata.noexec:
+                continue
+            if tid in sqrq.scenequeue_notcovered:
+                continue
+            sqdata.outrightfail.add(tid)
 
-                h = pending_hash_index(tid, rqdata)
-                if h not in hashes:
-                    hashes[h] = tid
-                else:
-                    sqrq.sq_deferred[tid] = hashes[h]
-                    bb.warn("Deferring %s after %s" % (tid, hashes[h]))
+            h = pending_hash_index(tid, rqdata)
+            if h not in sqdata.hashes:
+                sqdata.hashes[h] = tid
+            else:
+                sqrq.sq_deferred[tid] = sqdata.hashes[h]
+                bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h]))
 
 
 class TaskFailure(Exception):
diff --git a/lib/bb/tests/runqueue.py b/lib/bb/tests/runqueue.py
index cb4d526f13..01b992c47c 100644
--- a/lib/bb/tests/runqueue.py
+++ b/lib/bb/tests/runqueue.py
@@ -312,92 +312,3 @@ class RunQueueTests(unittest.TestCase):
                 else:
                     self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i)
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
-    def test_hashserv_partial_match(self):
-        # e1:do_package matches initial built but not second hash value
-        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
-            extraenv = {
-                "BB_HASHSERVE" : "auto",
-                "BB_SIGNATURE_HANDLER" : "TestEquivHash"
-            }
-            cmd = ["bitbake", "a1", "b1"]
-            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
-                             'populate_sysroot_setscene', 'package_qa_setscene']
-            sstatevalid = ""
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
-            self.assertEqual(set(tasks), set(expected))
-            with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
-               f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
-            with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
-               f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
-            cmd = ["bitbake", "e1"]
-            sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d"
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
-                        'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
-                        'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
-                        'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
-            expected.remove('e1:package')
-            self.assertEqual(set(tasks), set(expected))
-
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
-    def test_hashserv_partial_match2(self):
-        # e1:do_package + e1:do_populate_sysroot matches initial built but not second hash value
-        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
-            extraenv = {
-                "BB_HASHSERVE" : "auto",
-                "BB_SIGNATURE_HANDLER" : "TestEquivHash"
-            }
-            cmd = ["bitbake", "a1", "b1"]
-            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
-                             'populate_sysroot_setscene', 'package_qa_setscene']
-            sstatevalid = ""
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
-            self.assertEqual(set(tasks), set(expected))
-            with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
-               f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
-            with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
-               f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
-            cmd = ["bitbake", "e1"]
-            sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d e1:do_populate_sysroot:ef7dc0e2dd55d0534e75cba50731ff42f949818b6f29a65d72bc05856e56711d"
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
-                        'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
-                        'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
-                        'e1:package_setscene', 'e1:populate_sysroot_setscene', 'e1:build', 'e1:package_qa', 'e1:package_write_rpm', 'e1:package_write_ipk', 'e1:packagedata']
-            self.assertEqual(set(tasks), set(expected))
-
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
-    def test_hashserv_partial_match3(self):
-        # e1:do_package is valid for a1 but not after b1
-        # In former buggy code, this triggered e1:do_fetch, then e1:do_populate_sysroot to run
-        # with none of the intermediate tasks which is a serious bug
-        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
-            extraenv = {
-                "BB_HASHSERVE" : "auto",
-                "BB_SIGNATURE_HANDLER" : "TestEquivHash"
-            }
-            cmd = ["bitbake", "a1", "b1"]
-            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
-                             'populate_sysroot_setscene', 'package_qa_setscene']
-            sstatevalid = ""
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
-            self.assertEqual(set(tasks), set(expected))
-            with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
-               f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
-            with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
-               f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
-            cmd = ["bitbake", "e1", "-DD"]
-            sstatevalid = "e1:do_package:af056eae12a733a6a8c4f4da8c6757e588e13565852c94e2aad4d953a3989c13 e1:do_package:a3677703db82b22d28d57c1820a47851dd780104580863f5bd32e66e003a779d"
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="e1:fetch b1:install")
-            expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
-                        'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
-                        'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
-                        'e1:package_setscene']  + ['e1:' + x for x in self.alltasks]
-            expected.remove('e1:package')
-            self.assertEqual(set(tasks), set(expected))
-
-
-- 
2.20.1

[PATCH 3/9] siggen: Ensure setscenetasks list is available to worker context

[Richard Purdie]

The setscenetasks list needs to be available in the worker contexts
else the signature behaviour there mismatches what the server does.

Add the data to get/set_taskdata to ensure this happens.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/siggen.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 2b51967c02..71fa018227 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -44,6 +44,7 @@ class SignatureGenerator(object):
         self.file_checksum_values = {}
         self.taints = {}
         self.unitaskhashes = {}
+        self.setscenetasks = {}
 
     def finalise(self, fn, d, varient):
         return
@@ -75,10 +76,10 @@ class SignatureGenerator(object):
         return
 
     def get_taskdata(self):
-        return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes)
+        return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.setscenetasks)
 
     def set_taskdata(self, data):
-        self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes = data
+        self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.setscenetasks = data
 
     def reset(self, data):
         self.__init__(data)
-- 
2.20.1

[PATCH 4/9] runqueue: Change task migration behaviour for rerunning setscene tasks

[Richard Purdie]

Currently runqueue will rerun setscene tasks multiple times as hashes
change. This has caused numerous problems since a setscene task may
become "unavailable" for some future signature combination and the code
then can't easily "unskip" tasks its already passed into the execution
queue.

At least for now, only run setscene once and assume they're equivalent
at that point. In practise that has been much more stable in testing.

Tweak the test to match the change in behaviour.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py       | 4 ++++
 lib/bb/tests/runqueue.py | 5 +----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 02160ef4d7..65169931f1 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -2301,6 +2301,10 @@ class RunQueueExecute:
                 continue
             if tid in self.runq_running:
                 continue
+            if tid in self.scenequeue_covered:
+                # Potentially risky, should we report this hash as a match?
+                logger.info("Already covered setscene for %s so ignoring rehash" % (tid))
+                continue
             if tid not in self.pending_migrations:
                 self.pending_migrations.add(tid)
 
diff --git a/lib/bb/tests/runqueue.py b/lib/bb/tests/runqueue.py
index 01b992c47c..50b3392bc1 100644
--- a/lib/bb/tests/runqueue.py
+++ b/lib/bb/tests/runqueue.py
@@ -307,8 +307,5 @@ class RunQueueTests(unittest.TestCase):
                         'e1:package_setscene']
             self.assertEqual(set(tasks), set(expected))
             for i in expected:
-                if i in ["e1:package_setscene"]:
-                    self.assertEqual(tasks.count(i), 4, "%s not in task list four times" % i)
-                else:
-                    self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i)
+                self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i)
 
-- 
2.20.1

[PATCH 5/9] siggen/runqueue: Fix signature mismatch issues

[Richard Purdie]

We need to set the setscene tasklist before we call into the
taskhash/unihash code else the behaviour is inconsistent.

Avoid reporting hashes for non setscene tasks since we'd never
query that.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py | 3 ++-
 lib/bb/siggen.py   | 6 +++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 65169931f1..29bfd65e0b 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1162,6 +1162,8 @@ class RunQueueData:
 
         self.init_progress_reporter.next_stage()
 
+        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
+
         # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(self.runtaskentries)
@@ -1173,7 +1175,6 @@ class RunQueueData:
                     self.prepare_task_hash(tid)
 
         bb.parse.siggen.writeout_file_checksum_cache()
-        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
 
         #self.dump_data()
         return len(self.runtaskentries)
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 71fa018227..6207cbca30 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -455,7 +455,11 @@ class SignatureGeneratorUniHashMixIn(object):
         report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
         tempdir = d.getVar('T')
         fn = d.getVar('BB_FILENAME')
-        key = fn + ':do_' + task + ':' + taskhash
+        tid = fn + ':do_' + task
+        key = tid + ':' + taskhash
+
+        if self.setscenetasks and tid not in self.setscenetasks:
+            return
 
         # Sanity checks
         cache_unihash = self.unitaskhashes.get(key, None)
-- 
2.20.1

[PATCH 6/9] siggen: Avoid writing misleading sigdata files

[Richard Purdie]

Use the unihash in the output filename of sigdata files else the contents
of stamp directories is misleading. Write the unihash into the singature to
make it clear what happened.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/siggen.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 6207cbca30..90f0926f25 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -268,7 +268,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
             sigfile = stampbase
             referencestamp = runtime[11:]
         elif runtime and tid in self.taskhash:
-            sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[tid]
+            sigfile = stampbase + "." + task + ".sigdata" + "." + self.get_unihash(tid)
         else:
             sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid]
 
@@ -296,6 +296,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
             for dep in data['runtaskdeps']:
                 data['runtaskhashes'][dep] = self.get_unihash(dep)
             data['taskhash'] = self.taskhash[tid]
+            data['unihash'] = self.get_unihash(tid)
 
         taint = self.read_taint(fn, task, referencestamp)
         if taint:
-- 
2.20.1

[PATCH 7/9] runqueue: Save unihashes more frequently

[Richard Purdie]

There are some runqueue code paths where the unihash cache would not be
saved where for example only parsing or an occurred. Save the cache at the
end of runqueue generation to ensure entries are cached.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 29bfd65e0b..31de3ed1cf 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1443,6 +1443,7 @@ class RunQueue:
                 self.state = runQueueComplete
             else:
                 self.state = runQueueSceneInit
+                bb.parse.siggen.save_unitaskhashes()
 
         if self.state is runQueueSceneInit:
             self.rqdata.init_progress_reporter.next_stage()
-- 
2.20.1

[PATCH 8/9] runqueue: Small performance optimisation

[Richard Purdie]

A minor performance optmisation to keep lists smaller when running large
builds. We can do this since once a task has been built, we don't need
to worry about it. This improves a major bottleneck that shows up on
performance profile charts in dryruns.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 31de3ed1cf..18049436fd 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -207,6 +207,8 @@ class RunQueueScheduler(object):
 
     def newbuildable(self, task):
         self.buildable.add(task)
+        # Once tasks are running we don't need to worry about them again
+        self.buildable.difference_update(self.rq.runq_running)
 
     def removebuildable(self, task):
         self.buildable.remove(task)
-- 
2.20.1

[PATCH 9/9] siggen: Remove full path from unitaskhashes keys

[Richard Purdie]

The full paths make the cache useless in the sdk. They also bloat the
cache size. They're for human debugging benefit only so compromise and
reduce this to the filename.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/siggen.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 90f0926f25..a4bb1ff7fb 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -386,7 +386,7 @@ class SignatureGeneratorUniHashMixIn(object):
     def __get_task_unihash_key(self, tid):
         # TODO: The key only *needs* to be the taskhash, the tid is just
         # convenient
-        return '%s:%s' % (tid, self.taskhash[tid])
+        return '%s:%s' % (tid.rsplit("/", 1)[1], self.taskhash[tid])
 
     def get_stampfile_hash(self, tid):
         if tid in self.taskhash:
@@ -457,7 +457,7 @@ class SignatureGeneratorUniHashMixIn(object):
         tempdir = d.getVar('T')
         fn = d.getVar('BB_FILENAME')
         tid = fn + ':do_' + task
-        key = tid + ':' + taskhash
+        key = tid.rsplit("/", 1)[1] + ':' + taskhash
 
         if self.setscenetasks and tid not in self.setscenetasks:
             return
-- 
2.20.1

Re: [PATCH 5/9] siggen/runqueue: Fix signature mismatch issues

[Joshua Watt]

On 9/27/19 7:33 AM, Richard Purdie wrote:
> We need to set the setscene tasklist before we call into the
> taskhash/unihash code else the behaviour is inconsistent.
>
> Avoid reporting hashes for non setscene tasks since we'd never
> query that.
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>   lib/bb/runqueue.py | 3 ++-
>   lib/bb/siggen.py   | 6 +++++-
>   2 files changed, 7 insertions(+), 2 deletions(-)
>
> diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
> index 65169931f1..29bfd65e0b 100644
> --- a/lib/bb/runqueue.py
> +++ b/lib/bb/runqueue.py
> @@ -1162,6 +1162,8 @@ class RunQueueData:
>   
>           self.init_progress_reporter.next_stage()
>   
> +        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
> +
>           # Iterate over the task list and call into the siggen code
>           dealtwith = set()
>           todeal = set(self.runtaskentries)
> @@ -1173,7 +1175,6 @@ class RunQueueData:
>                       self.prepare_task_hash(tid)
>   
>           bb.parse.siggen.writeout_file_checksum_cache()
> -        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
>   
>           #self.dump_data()
>           return len(self.runtaskentries)
> diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
> index 71fa018227..6207cbca30 100644
> --- a/lib/bb/siggen.py
> +++ b/lib/bb/siggen.py
> @@ -455,7 +455,11 @@ class SignatureGeneratorUniHashMixIn(object):
>           report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
>           tempdir = d.getVar('T')
>           fn = d.getVar('BB_FILENAME')
> -        key = fn + ':do_' + task + ':' + taskhash
> +        tid = fn + ':do_' + task
> +        key = tid + ':' + taskhash
> +
> +        if self.setscenetasks and tid not in self.setscenetasks:
> +            return

Would this be worth of a bb.fatal() (or perhaps a bb.warn()) to try and 
discover issues sooner?


>   
>           # Sanity checks
>           cache_unihash = self.unitaskhashes.get(key, None)

Re: [PATCH 5/9] siggen/runqueue: Fix signature mismatch issues

[Richard Purdie]

On Fri, 2019-09-27 at 08:35 -0500, Joshua Watt wrote:
> On 9/27/19 7:33 AM, Richard Purdie wrote:
> > We need to set the setscene tasklist before we call into the
> > taskhash/unihash code else the behaviour is inconsistent.
> > 
> > Avoid reporting hashes for non setscene tasks since we'd never
> > query that.
> > 
> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > ---
> >   lib/bb/runqueue.py | 3 ++-
> >   lib/bb/siggen.py   | 6 +++++-
> >   2 files changed, 7 insertions(+), 2 deletions(-)
> > 
> > diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
> > index 65169931f1..29bfd65e0b 100644
> > --- a/lib/bb/runqueue.py
> > +++ b/lib/bb/runqueue.py
> > @@ -1162,6 +1162,8 @@ class RunQueueData:
> >   
> >           self.init_progress_reporter.next_stage()
> >   
> > +        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids
> > )
> > +
> >           # Iterate over the task list and call into the siggen
> > code
> >           dealtwith = set()
> >           todeal = set(self.runtaskentries)
> > @@ -1173,7 +1175,6 @@ class RunQueueData:
> >                       self.prepare_task_hash(tid)
> >   
> >           bb.parse.siggen.writeout_file_checksum_cache()
> > -        bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids
> > )
> >   
> >           #self.dump_data()
> >           return len(self.runtaskentries)
> > diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
> > index 71fa018227..6207cbca30 100644
> > --- a/lib/bb/siggen.py
> > +++ b/lib/bb/siggen.py
> > @@ -455,7 +455,11 @@ class SignatureGeneratorUniHashMixIn(object):
> >           report_taskdata =
> > d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
> >           tempdir = d.getVar('T')
> >           fn = d.getVar('BB_FILENAME')
> > -        key = fn + ':do_' + task + ':' + taskhash
> > +        tid = fn + ':do_' + task
> > +        key = tid + ':' + taskhash
> > +
> > +        if self.setscenetasks and tid not in self.setscenetasks:
> > +            return
> 
> Would this be worth of a bb.fatal() (or perhaps a bb.warn()) to try
> and discover issues sooner?

There are some circumstances where such a message is confusing and I
couldn't make it work reliably yet. Its worth looking at further
though...

Cheers,

Richard