[PATCH] bitbake-worker: Allow shutdown/database flush of pseudo server at task exit

[PATCH] bitbake-worker: Allow shutdown/database flush of pseudo server at task exit

[Richard Purdie]

We have a problem where pseudo server processes exist ater bitbake exits
and hold the pseudo database in memory. In a docker container, the processes
wll be killed as the container is destroyed with no warning and no opportunity
to write the data to disk. This leads to permissions/inode corruptions and
data loss.

Send a shutdown message to pseudo which in new versions of pseudo will flush
the database, thereby fixing some of the issues people using docker containers
see.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 bin/bitbake-worker | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bin/bitbake-worker b/bin/bitbake-worker
index 6ead2da6d1..24dadd997c 100755
--- a/bin/bitbake-worker
+++ b/bin/bitbake-worker
@@ -17,6 +17,8 @@ import signal
 import pickle
 import traceback
 import queue
+import shlex
+import subprocess
 from multiprocessing import Lock
 from threading import Thread
 
@@ -146,6 +148,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
     # a fork() or exec*() activates PSEUDO...
 
     envbackup = {}
+    fakeroot = False
     fakeenv = {}
     umask = None
 
@@ -165,6 +168,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
 
     # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
     if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
+        fakeroot = True
         envvars = (workerdata["fakerootenv"][fn] or "").split()
         for key, value in (var.split('=') for var in envvars):
             envbackup[key] = os.environ.get(key)
@@ -283,7 +287,12 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
             try:
                 if dry_run:
                     return 0
-                return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
+                ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
+                if fakeroot:
+                    fakerootcmd = shlex.split(the_data.getVar("FAKEROOTCMD"))
+                    bb.warn("Shutting down pseudo")
+                    subprocess.run(fakerootcmd + ['-S'], capture_output=True, check=True)
+                return ret
             except:
                 os._exit(1)
         if not profiling:
-- 
2.32.0

RE: [bitbake-devel] [PATCH] bitbake-worker: Allow shutdown/database flush of pseudo server at task exit

[Peter Kjellerstedt]

> -----Original Message-----
> From: bitbake-devel@lists.openembedded.org <bitbake-
> devel@lists.openembedded.org> On Behalf Of Richard Purdie
> Sent: den 25 september 2021 11:41
> To: bitbake-devel@lists.openembedded.org
> Subject: [bitbake-devel] [PATCH] bitbake-worker: Allow shutdown/database
> flush of pseudo server at task exit
> 
> We have a problem where pseudo server processes exist ater bitbake exits

ater -> after

> and hold the pseudo database in memory. In a docker container, the processes
> wll be killed as the container is destroyed with no warning and no opportunity

wll -> will

> to write the data to disk. This leads to permissions/inode corruptions and
> data loss.
> 
> Send a shutdown message to pseudo which in new versions of pseudo will flush
> the database, thereby fixing some of the issues people using docker containers
> see.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  bin/bitbake-worker | 11 ++++++++++-
>  1 file changed, 10 insertions(+), 1 deletion(-)
> 
> diff --git a/bin/bitbake-worker b/bin/bitbake-worker
> index 6ead2da6d1..24dadd997c 100755
> --- a/bin/bitbake-worker
> +++ b/bin/bitbake-worker
> @@ -17,6 +17,8 @@ import signal
>  import pickle
>  import traceback
>  import queue
> +import shlex
> +import subprocess
>  from multiprocessing import Lock
>  from threading import Thread
> 
> @@ -146,6 +148,7 @@ def fork_off_task(cfg, data, databuilder, workerdata,
> fn, task, taskname, taskha
>      # a fork() or exec*() activates PSEUDO...
> 
>      envbackup = {}
> +    fakeroot = False
>      fakeenv = {}
>      umask = None
> 
> @@ -165,6 +168,7 @@ def fork_off_task(cfg, data, databuilder, workerdata,
> fn, task, taskname, taskha
> 
>      # We can't use the fakeroot environment in a dry run as it possibly
> hasn't been built
>      if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not
> dry_run:
> +        fakeroot = True
>          envvars = (workerdata["fakerootenv"][fn] or "").split()
>          for key, value in (var.split('=') for var in envvars):
>              envbackup[key] = os.environ.get(key)
> @@ -283,7 +287,12 @@ def fork_off_task(cfg, data, databuilder, workerdata,
> fn, task, taskname, taskha
>              try:
>                  if dry_run:
>                      return 0
> -                return bb.build.exec_task(fn, taskname, the_data,
> cfg.profile)
> +                ret = bb.build.exec_task(fn, taskname, the_data,
> cfg.profile)
> +                if fakeroot:
> +                    fakerootcmd =
> shlex.split(the_data.getVar("FAKEROOTCMD"))
> +                    bb.warn("Shutting down pseudo")
> +                    subprocess.run(fakerootcmd + ['-S'],
> capture_output=True, check=True)
> +                return ret
>              except:
>                  os._exit(1)
>          if not profiling:
> --
> 2.32.0