[PATCH 1/2] cooker: Rework the parsing results submission

[PATCH 1/2] cooker: Rework the parsing results submission

[Richard Purdie]

The loop submitting the parser results was not efficient on many core systems
as may of the parsers could block for a long time waiting to send the results.
While a result was pending, the code wouldn't parse further jobs.

Change the code to parse further jobs even if the results can't be submitted
and lower the result submission timeout to keep the parsers busy.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cooker.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 738849d189..1d347ddc52 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -2107,25 +2107,29 @@ class Parser(multiprocessing.Process):
         multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
 
         pending = []
+        havejobs = True
         try:
-            while True:
+            while havejobs or pending:
                 if self.quit.is_set():
                     break
 
-                if pending:
-                    result = pending.pop()
-                else:
-                    try:
-                        job = self.jobs.pop()
-                    except IndexError:
-                        break
+                job = None
+                try:
+                    job = self.jobs.pop()
+                except IndexError:
+                    havejobs = False
+                if job:
                     result = self.parse(*job)
                     # Clear the siggen cache after parsing to control memory usage, its huge
                     bb.parse.siggen.postparsing_clean_cache()
-                try:
-                    self.results.put(result, timeout=0.25)
-                except queue.Full:
                     pending.append(result)
+
+                if pending:
+                    try:
+                        result = pending.pop()
+                        self.results.put(result, timeout=0.05)
+                    except queue.Full:
+                        pending.append(result)
         finally:
             self.results.close()
             self.results.join_thread()
-- 
2.37.2

[PATCH 2/2] cooker: Clean up inotify idle handler

[Richard Purdie]

We no longer need to abstract the inotify callback handler, remove the
abstraction and simplify/clean up the code.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cooker.py         | 6 ------
 lib/bb/server/process.py | 5 +++--
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 1d347ddc52..d2c42c858d 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -190,12 +190,6 @@ class BBCooker:
 
         self.inotify_modified_files = []
 
-        def _process_inotify_updates(server, cooker, halt):
-            cooker.process_inotify_updates()
-            return 1.0
-
-        self.idleCallBackRegister(_process_inotify_updates, self)
-
         # TOSTOP must not be set or our children will hang when they output
         try:
             fd = sys.stdout.fileno()
diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 78fdc6cf71..4422fd7311 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -158,8 +158,7 @@ class ProcessServer():
     def wait_for_idle(self, timeout=30):
         # Wait for the idle loop to have cleared
         with self.idle_cond:
-            # FIXME - the 1 is the inotify processing in cooker which always runs
-            self.idle_cond.wait_for(lambda: len(self._idlefuns) <= 1, timeout)
+            self.idle_cond.wait_for(lambda: len(self._idlefuns) == 0, timeout)
 
     def main(self):
         self.cooker.pre_serve()
@@ -389,6 +388,8 @@ class ProcessServer():
             nextsleep = 0.1
             fds = []
 
+            self.cooker.process_inotify_updates()
+
             with bb.utils.lock_timeout(self._idlefuncsLock):
                 items = list(self._idlefuns.items())
 
-- 
2.37.2