[PATCH 1/5] runqueue: Simplify some convoluted logic

[PATCH 1/5] runqueue: Simplify some convoluted logic

[Richard Purdie]

This was left from when task IDs complicated the code, simplify.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index db0d852c54..48b668abc1 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1908,12 +1908,7 @@ class RunQueueExecute:
                     if nexttask in self.sqdata.unskippable:
                         logger.debug(2, "Setscene task %s is unskippable" % nexttask)
                     if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
-                        fn = fn_from_tid(nexttask)
-                        foundtarget = False
-
-                        if nexttask in self.rqdata.target_tids:
-                            foundtarget = True
-                        if not foundtarget:
+                        if nexttask not in self.rqdata.target_tids:
                             logger.debug(2, "Skipping setscene for task %s" % nexttask)
                             self.sq_task_skip(nexttask)
                             self.scenequeue_notneeded.add(nexttask)
-- 
2.20.1

[PATCH 2/5] runqueue: Streamline outright setscene failure handling

[Richard Purdie]

With the recent codebase changes we can just call the outrightfail
function immediately, removing some intermediate data structures.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 48b668abc1..2d52ef3190 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1913,9 +1913,6 @@ class RunQueueExecute:
                             self.sq_task_skip(nexttask)
                             self.scenequeue_notneeded.add(nexttask)
                             return True
-                    if nexttask in self.sqdata.outrightfail:
-                        self.sq_task_failoutright(nexttask)
-                        return True
                     task = nexttask
                     break
         if task is not None:
@@ -2330,8 +2327,6 @@ class SQData(object):
         self.stamps = {}
         # Setscene tasks directly depended upon by the build
         self.unskippable = set()
-        # List of setscene tasks which aren't present
-        self.outrightfail = []
         # A list of normal tasks a setscene task covers
         self.sq_covered_tasks = {}
 
@@ -2553,7 +2548,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
         for tid in sqdata.sq_revdeps:
             if tid not in valid_new and tid not in noexec:
                 logger.debug(2, 'No package found, so skipping setscene task %s', tid)
-                sqdata.outrightfail.append(tid)
+                sqrq.sq_task_failoutright(tid)
 
 class TaskFailure(Exception):
     """
-- 
2.20.1

[PATCH 3/5] runqueue: Whitespace fix

[Richard Purdie]

Fix some unwanted extra indentation.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 2d52ef3190..624cc756fd 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -2438,29 +2438,29 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
     # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies
     for tid in rqdata.runq_setscene_tids:
-            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-            realtid = tid + "_setscene"
-            idepends = rqdata.taskData[mc].taskentries[realtid].idepends
-            sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", rqdata.dataCaches[mc], taskfn, noextra=True)
-            for (depname, idependtask) in idepends:
+        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+        realtid = tid + "_setscene"
+        idepends = rqdata.taskData[mc].taskentries[realtid].idepends
+        sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", rqdata.dataCaches[mc], taskfn, noextra=True)
+        for (depname, idependtask) in idepends:
 
-                if depname not in rqdata.taskData[mc].build_targets:
-                    continue
+            if depname not in rqdata.taskData[mc].build_targets:
+                continue
 
-                depfn = rqdata.taskData[mc].build_targets[depname][0]
-                if depfn is None:
-                     continue
-                deptid = depfn + ":" + idependtask.replace("_setscene", "")
-                if deptid not in rqdata.runtaskentries:
-                    bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask))
+            depfn = rqdata.taskData[mc].build_targets[depname][0]
+            if depfn is None:
+                continue
+            deptid = depfn + ":" + idependtask.replace("_setscene", "")
+            if deptid not in rqdata.runtaskentries:
+                bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask))
 
-                if not deptid in sqdata.sq_harddeps:
-                    sqdata.sq_harddeps[deptid] = set()
-                sqdata.sq_harddeps[deptid].add(tid)
+            if not deptid in sqdata.sq_harddeps:
+                sqdata.sq_harddeps[deptid] = set()
+            sqdata.sq_harddeps[deptid].add(tid)
 
-                sq_revdeps_squash[tid].add(deptid)
-                # Have to zero this to avoid circular dependencies
-                sq_revdeps_squash[deptid] = set()
+            sq_revdeps_squash[tid].add(deptid)
+            # Have to zero this to avoid circular dependencies
+            sq_revdeps_squash[deptid] = set()
 
     rqdata.init_progress_reporter.next_stage()
 
-- 
2.20.1

[PATCH 4/5] siggen: Use unique hashes for tasks

[Richard Purdie]

Now that runqueue optimises based on task hash, we need to ensure
tasks have unique hashes even in the simplest siggen mode. Use
the task name to calculate a unique hash.

This fixes runqueue tests when hash optimisations are added.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/siggen.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 3b017219ed..f9c4798e3d 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -49,8 +49,9 @@ class SignatureGenerator(object):
         return self.taskhash[task]
 
     def get_taskhash(self, fn, task, deps, dataCache):
-        self.taskhash[fn + "." + task] = "0"
-        return self.taskhash[fn + "." + task]
+        k = fn + "." + task
+        self.taskhash[k] = hashlib.sha256(k.encode("utf-8")).hexdigest()
+        return self.taskhash[k]
 
     def writeout_file_checksum_cache(self):
         """Write/update the file checksum cache onto disk"""
-- 
2.20.1

[PATCH 5/5] runqueue: Optimise multiconfig with overlapping setscene

[Richard Purdie]

Currently if a multiconfig build has setscene tasks which share the same
hash but it wasn't present in an sstate cache at the start of the build,
the tasks would all rebuild the same artefact.

This changes the code to spot this and start building the first hash whilst
holding the other tasks in the scenequeue. When the first task finishes,
the others in the scenequeue are allowed to proceed and should then be able
to take advantage of a shared sstate directory.

This change adds in a unit test for this which complicates the testing
metadata slight more. In particular comments are added to the tasks to
ensure they do have differing signatures to make the test results more
obvious and as expected.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/runqueue.py                            | 22 +++++++++++++++----
 .../tests/runqueue-tests/classes/base.bbclass | 19 +++++++++++++++-
 lib/bb/tests/runqueue-tests/conf/bitbake.conf |  9 ++++++--
 .../runqueue-tests/conf/multiconfig/mc1.conf  |  1 +
 .../runqueue-tests/conf/multiconfig/mc2.conf  |  1 +
 lib/bb/tests/runqueue.py                      | 19 ++++++++++++++++
 6 files changed, 64 insertions(+), 7 deletions(-)
 create mode 100644 lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
 create mode 100644 lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 624cc756fd..6f9ac5e46c 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1701,6 +1701,7 @@ class RunQueueExecute:
         self.build_stamps = {}
         self.build_stamps2 = []
         self.failed_tids = []
+        self.pending_hashes = {}
 
         self.stampcache = {}
 
@@ -1868,6 +1869,10 @@ class RunQueueExecute:
             if alldeps:
                 self.setbuildable(revdep)
                 logger.debug(1, "Marking task %s as buildable", revdep)
+        if task in self.rqdata.runq_setscene_tids:
+           h = self.rqdata.runtaskentries[task].hash
+           if h in self.pending_hashes:
+               del self.pending_hashes[h]
 
     def task_complete(self, task):
         self.stats.taskCompleted()
@@ -1899,20 +1904,25 @@ class RunQueueExecute:
         """
 
         self.rq.read_workers()
+        pending = 0
 
         task = None
         if not self.sqdone and self.can_start_task():
             # Find the next setscene to run
             for nexttask in self.rqdata.runq_setscene_tids:
                 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
-                    if nexttask in self.sqdata.unskippable:
-                        logger.debug(2, "Setscene task %s is unskippable" % nexttask)
                     if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
                         if nexttask not in self.rqdata.target_tids:
                             logger.debug(2, "Skipping setscene for task %s" % nexttask)
                             self.sq_task_skip(nexttask)
                             self.scenequeue_notneeded.add(nexttask)
                             return True
+                    if self.rqdata.runtaskentries[nexttask].hash in self.pending_hashes:
+                        pending += 1
+                        # This setscene task has a hash matching something we're currently building, so wait
+                        continue
+                    if nexttask in self.sqdata.unskippable:
+                        logger.debug(2, "Setscene task %s is unskippable" % nexttask)
                     task = nexttask
                     break
         if task is not None:
@@ -1963,7 +1973,7 @@ class RunQueueExecute:
             if self.can_start_task():
                 return True
 
-        if not self.sq_live and not self.sqdone:
+        if not self.sq_live and not self.sqdone and not pending:
             logger.info("Setscene tasks completed")
             logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
 
@@ -2232,6 +2242,8 @@ class RunQueueExecute:
         bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.tasks_notcovered.add(task)
+        h = self.rqdata.runtaskentries[task].hash
+        self.pending_hashes[h] = task
         self.scenequeue_updatecounters(task, True)
         self.sq_check_taskfail(task)
 
@@ -2242,6 +2254,8 @@ class RunQueueExecute:
         self.sq_stats.taskCompleted()
         self.scenequeue_notcovered.add(task)
         self.tasks_notcovered.add(task)
+        h = self.rqdata.runtaskentries[task].hash
+        self.pending_hashes[h] = task
         self.scenequeue_updatecounters(task, True)
 
     def sq_task_skip(self, task):
@@ -2546,7 +2560,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
             valid_new.append(sq_task[v])
 
         for tid in sqdata.sq_revdeps:
-            if tid not in valid_new and tid not in noexec:
+            if tid not in valid_new and tid not in noexec and rqdata.runtaskentries[tid].hash not in sqrq.pending_hashes:
                 logger.debug(2, 'No package found, so skipping setscene task %s', tid)
                 sqrq.sq_task_failoutright(tid)
 
diff --git a/lib/bb/tests/runqueue-tests/classes/base.bbclass b/lib/bb/tests/runqueue-tests/classes/base.bbclass
index e81df7ac42..b966568dc7 100644
--- a/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -4,7 +4,9 @@ SSTATEVALID ??= ""
 def stamptask(d):
     import time
 
-    thistask = d.expand("${PN}:${BB_CURRENTTASK}") 
+    thistask = d.expand("${PN}:${BB_CURRENTTASK}")
+    if d.getVar("BB_CURRENT_MC") != "default":
+        thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
     if thistask in d.getVar("SLOWTASKS").split():
         bb.note("Slowing task %s" % thistask)
         time.sleep(0.5)
@@ -13,48 +15,63 @@ def stamptask(d):
         f.write(thistask + "\n")
 
 python do_fetch() {
+    # fetch
     stamptask(d)
 }
 python do_unpack() {
+    # unpack
     stamptask(d)
 }
 python do_patch() {
+    # patch
     stamptask(d)
 }
 python do_populate_lic() {
+    # populate_lic
     stamptask(d)
 }
 python do_prepare_recipe_sysroot() {
+    # prepare_recipe_sysroot
     stamptask(d)
 }
 python do_configure() {
+    # configure
     stamptask(d)
 }
 python do_compile() {
+    # compile
     stamptask(d)
 }
 python do_install() {
+    # install
     stamptask(d)
 }
 python do_populate_sysroot() {
+    # populate_sysroot
     stamptask(d)
 }
 python do_package() {
+    # package
     stamptask(d)
 }
 python do_package_write_ipk() {
+    # package_write_ipk
     stamptask(d)
 }
 python do_package_write_rpm() {
+    # package_write_rpm
     stamptask(d)
 }
 python do_packagedata() {
+    # packagedata
     stamptask(d)
 }
 python do_package_qa() {
+    # package_qa
     stamptask(d)
 }
 python do_build() {
+    # build
     stamptask(d)
 }
 do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
diff --git a/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index cccd677966..d87ded29a7 100644
--- a/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -5,6 +5,11 @@ BBFILES = "${COREBASE}/recipes/*.bb"
 PROVIDES = "${PN}"
 PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
 export PATH
-STAMP = "${TOPDIR}/stamps/${PN}"
-T = "${TOPDIR}/workdir/${PN}/temp"
+TMPDIR ??= "${TOPDIR}"
+STAMP = "${TMPDIR}/stamps/${PN}"
+T = "${TMPDIR}/workdir/${PN}/temp"
 BB_NUMBER_THREADS = "4"
+
+BB_HASHBASE_WHITELIST = "BB_CURRENT_MC"
+
+include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
new file mode 100644
index 0000000000..ecf23e1c73
--- /dev/null
+++ b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc1/"
diff --git a/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
new file mode 100644
index 0000000000..eef338e4cc
--- /dev/null
+++ b/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc2/"
diff --git a/lib/bb/tests/runqueue.py b/lib/bb/tests/runqueue.py
index b1a23bc5d4..8b29fde864 100644
--- a/lib/bb/tests/runqueue.py
+++ b/lib/bb/tests/runqueue.py
@@ -189,3 +189,22 @@ class RunQueueTests(unittest.TestCase):
                         'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+    def test_multiconfig_setscene_optimise(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            extraenv = {
+                "BBMULTICONFIG" : "mc1 mc2",
+                "BB_SIGNATURE_HANDLER" : "basic"
+            }
+            cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
+            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+                             'populate_sysroot_setscene', 'package_qa_setscene']
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
+                       ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
+                       ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
+                       ['mc1:b1:build', 'mc2:b1:build']
+            for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
+                expected.remove(x)
+            self.assertEqual(set(tasks), set(expected))
+
-- 
2.20.1