[PATCH 1/3] fetch/wget: Move files into place atomically

[PATCH 1/3] fetch/wget: Move files into place atomically

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/fetch2/wget.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py
index b3a3de571a..b2b542e1dc 100644
--- a/lib/bb/fetch2/wget.py
+++ b/lib/bb/fetch2/wget.py
@@ -106,10 +106,9 @@ class Wget(FetchMethod):
 
         fetchcmd = self.basecmd
 
-        if 'downloadfilename' in ud.parm:
-            localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
-            bb.utils.mkdirhier(os.path.dirname(localpath))
-            fetchcmd += " -O %s" % shlex.quote(localpath)
+        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp"
+        bb.utils.mkdirhier(os.path.dirname(localpath))
+        fetchcmd += " -O %s" % shlex.quote(localpath)
 
         if ud.user and ud.pswd:
             fetchcmd += " --auth-no-challenge"
@@ -133,6 +132,10 @@ class Wget(FetchMethod):
 
         self._runwget(ud, d, fetchcmd, False)
 
+        # Remove the ".tmp" and move the file into position atomically
+        # Our lock prevents multiple writers but mirroring code may grab incomplete files
+        os.rename(localpath, localpath[:-4])
+
         # Sanity check since wget can pretend it succeed when it didn't
         # Also, this used to happen if sourceforge sent us to the mirror page
         if not os.path.exists(ud.localpath):
-- 
2.34.1

[PATCH 2/3] server/process: Avoid risk of exception deadlocks

[Richard Purdie]

The open coded lock acquire/release in the UI event handler doesn't
cover the case an exception occurs and if one did, it could deadlock
the code. Switch to use 'with' statements which would handle this
possibility.

We have seen deadlocks in the UI at exit this so this removes a
possible cause.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/server/process.py | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 613956f30f..74b74dc39b 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -667,18 +667,14 @@ class BBUIEventQueue:
         self.t.start()
 
     def getEvent(self):
-        self.eventQueueLock.acquire()
-
-        if len(self.eventQueue) == 0:
-            self.eventQueueLock.release()
-            return None
-
-        item = self.eventQueue.pop(0)
+        with self.eventQueueLock:
+            if len(self.eventQueue) == 0:
+                return None
 
-        if len(self.eventQueue) == 0:
-            self.eventQueueNotify.clear()
+            item = self.eventQueue.pop(0)
+            if len(self.eventQueue) == 0:
+                self.eventQueueNotify.clear()
 
-        self.eventQueueLock.release()
         return item
 
     def waitEvent(self, delay):
@@ -686,10 +682,9 @@ class BBUIEventQueue:
         return self.getEvent()
 
     def queue_event(self, event):
-        self.eventQueueLock.acquire()
-        self.eventQueue.append(event)
-        self.eventQueueNotify.set()
-        self.eventQueueLock.release()
+        with self.eventQueueLock:
+            self.eventQueue.append(event)
+            self.eventQueueNotify.set()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))
-- 
2.34.1

[PATCH 3/3] server/process: Remove daemonic thread usage

[Richard Purdie]

We're seeing UI deadlocks occasionally and this is possibly due to the
use of a daemonic thread in the UI event queue processing. This thread
could terminate holding a threading Lock() which would cause issues
for the process when exitting.

Change the shutdown process to handle this more cleanly.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/server/process.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 74b74dc39b..b330520a98 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -437,6 +437,7 @@ class BitBakeProcessServerConnection(object):
         self.socket_connection = sock
 
     def terminate(self):
+        self.events.close()
         self.socket_connection.close()
         self.connection.connection.close()
         self.connection.recv.close()
@@ -662,7 +663,6 @@ class BBUIEventQueue:
         self.reader = ConnectionReader(readfd)
 
         self.t = threading.Thread()
-        self.t.daemon = True
         self.t.run = self.startCallbackHandler
         self.t.start()
 
@@ -693,13 +693,16 @@ class BBUIEventQueue:
         bb.utils.set_process_name("UIEventQueue")
         while True:
             try:
-                self.reader.wait()
+                self.reader.wait(0.25)
                 event = self.reader.get()
                 self.queue_event(event)
-            except EOFError:
+            except (EOFError, OSError):
                 # Easiest way to exit is to close the file descriptor to cause an exit
                 break
+
+    def close(self):
         self.reader.close()
+        self.t.join()
 
 class ConnectionReader(object):
 
-- 
2.34.1