* [PATCH 1/2] package_deb, ipk: improve subprocess output on package manager command
@ 2017-09-04 21:35 leonardo.sandoval.gonzalez
2017-09-04 21:35 ` [PATCH 2/2] package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages leonardo.sandoval.gonzalez
0 siblings, 1 reply; 3+ messages in thread
From: leonardo.sandoval.gonzalez @ 2017-09-04 21:35 UTC (permalink / raw)
To: openembedded-core
From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
Redirecting stderr to stdout helps debugging issues, i.e instead of just
getting the return code, get also the error log from the pkg manger
This commit is in the way to figure out the root cause of [YOCTO #12012],
where dpkg-deb fails with a 2 return code and according to the man page,
there are multiple issues leading to the same code.
Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
---
meta/classes/package_deb.bbclass | 4 +++-
meta/classes/package_ipk.bbclass | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index 83baa6c21b..30605344f4 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -286,7 +286,9 @@ def deb_write_pkg(pkg, d):
conffiles.close()
os.chdir(basedir)
- subprocess.check_output("PATH=\"%s\" dpkg-deb -b %s %s" % (localdata.getVar("PATH"), root, pkgoutdir), shell=True)
+ subprocess.check_output("PATH=\"%s\" dpkg-deb -b %s %s" % (localdata.getVar("PATH"), root, pkgoutdir),
+ stderr=subprocess.STDOUT,
+ shell=True)
finally:
cleanupcontrol(root)
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index d58b824d3a..ec90996184 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -244,7 +244,9 @@ def ipk_write_pkg(pkg, d):
os.chdir(basedir)
subprocess.check_output("PATH=\"%s\" %s %s %s" % (localdata.getVar("PATH"),
- d.getVar("OPKGBUILDCMD"), pkg, pkgoutdir), shell=True)
+ d.getVar("OPKGBUILDCMD"), pkg, pkgoutdir),
+ stderr=subprocess.STDOUT,
+ shell=True)
if d.getVar('IPK_SIGN_PACKAGES') == '1':
ipkver = "%s-%s" % (d.getVar('PKGV'), d.getVar('PKGR'))
--
2.12.3
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [PATCH 2/2] package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages
2017-09-04 21:35 [PATCH 1/2] package_deb, ipk: improve subprocess output on package manager command leonardo.sandoval.gonzalez
@ 2017-09-04 21:35 ` leonardo.sandoval.gonzalez
2017-09-05 8:31 ` Joshua Lock
0 siblings, 1 reply; 3+ messages in thread
From: leonardo.sandoval.gonzalez @ 2017-09-04 21:35 UTC (permalink / raw)
To: openembedded-core
From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
Current implementation does not handle possible exceptions coming from child
processes, the latter responsible for creating packages. With the aim to have more
control, use pipes to communicate exceptions and stop package creation in case
of failure.
Helps to debug [YOCTO #12012].
Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
---
meta/classes/package_deb.bbclass | 36 +++++++++++++++++++++++++++++++++---
meta/classes/package_ipk.bbclass | 36 +++++++++++++++++++++++++++++++++---
2 files changed, 66 insertions(+), 6 deletions(-)
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index 30605344f4..5d297939b6 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -41,7 +41,29 @@ def debian_arch_map(arch, tune):
return arch
python do_package_deb () {
- from multiprocessing import Process
+
+ import multiprocessing
+ import traceback
+
+ class DebianWritePkgProcess(multiprocessing.Process):
+ def __init__(self, *args, **kwargs):
+ multiprocessing.Process.__init__(self, *args, **kwargs)
+ self._pconn, self._cconn = multiprocessing.Pipe()
+ self._exception = None
+
+ def run(self):
+ try:
+ multiprocessing.Process.run(self)
+ self._cconn.send(None)
+ except Exception as e:
+ tb = traceback.format_exc()
+ self._cconn.send((e, tb))
+
+ @property
+ def exception(self):
+ if self._pconn.poll():
+ self._exception = self._pconn.recv()
+ return self._exception
oldcwd = os.getcwd()
@@ -56,20 +78,28 @@ python do_package_deb () {
max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
launched = []
+ error = None
pkgs = packages.split()
- while pkgs:
+ while not error and pkgs:
if len(launched) < max_process:
- p = Process(target=deb_write_pkg, args=(pkgs.pop(), d))
+ p = DebianWritePkgProcess(target=deb_write_pkg, args=(pkgs.pop(), d))
p.start()
launched.append(p)
for q in launched:
# The finished processes are joined when calling is_alive()
if not q.is_alive():
launched.remove(q)
+ if q.exception:
+ error, traceback = q.exception
+ break
+
for p in launched:
p.join()
os.chdir(oldcwd)
+
+ if error:
+ raise error
}
do_package_deb[vardeps] += "deb_write_pkg"
do_package_deb[vardepsexclude] = "BB_NUMBER_THREADS"
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index ec90996184..8439cda6dd 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -17,7 +17,29 @@ OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKA
OPKGLIBDIR = "${localstatedir}/lib"
python do_package_ipk () {
- from multiprocessing import Process
+ import multiprocessing
+ import traceback
+
+ class IPKWritePkgProcess(multiprocessing.Process):
+ def __init__(self, *args, **kwargs):
+ multiprocessing.Process.__init__(self, *args, **kwargs)
+ self._pconn, self._cconn = multiprocessing.Pipe()
+ self._exception = None
+
+ def run(self):
+ try:
+ multiprocessing.Process.run(self)
+ self._cconn.send(None)
+ except Exception as e:
+ tb = traceback.format_exc()
+ self._cconn.send((e, tb))
+
+ @property
+ def exception(self):
+ if self._pconn.poll():
+ self._exception = self._pconn.recv()
+ return self._exception
+
oldcwd = os.getcwd()
@@ -41,20 +63,28 @@ python do_package_ipk () {
max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
launched = []
+ error = None
pkgs = packages.split()
- while pkgs:
+ while not error and pkgs:
if len(launched) < max_process:
- p = Process(target=ipk_write_pkg, args=(pkgs.pop(), d))
+ p = IPKWritePkgProcess(target=ipk_write_pkg, args=(pkgs.pop(), d))
p.start()
launched.append(p)
for q in launched:
# The finished processes are joined when calling is_alive()
if not q.is_alive():
launched.remove(q)
+ if q.exception:
+ error, traceback = q.exception
+ break
+
for p in launched:
p.join()
os.chdir(oldcwd)
+
+ if error:
+ raise error
}
do_package_ipk[vardeps] += "ipk_write_pkg"
do_package_ipk[vardepsexclude] = "BB_NUMBER_THREADS"
--
2.12.3
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH 2/2] package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages
2017-09-04 21:35 ` [PATCH 2/2] package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages leonardo.sandoval.gonzalez
@ 2017-09-05 8:31 ` Joshua Lock
0 siblings, 0 replies; 3+ messages in thread
From: Joshua Lock @ 2017-09-05 8:31 UTC (permalink / raw)
To: leonardo.sandoval.gonzalez, openembedded-core
On 04/09/17 22:35, leonardo.sandoval.gonzalez@linux.intel.com wrote:
> From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
>
> Current implementation does not handle possible exceptions coming from child
> processes, the latter responsible for creating packages. With the aim to have more
> control, use pipes to communicate exceptions and stop package creation in case
> of failure.
>
> Helps to debug [YOCTO #12012].
>
> Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
> ---
> meta/classes/package_deb.bbclass | 36 +++++++++++++++++++++++++++++++++---
> meta/classes/package_ipk.bbclass | 36 +++++++++++++++++++++++++++++++++---
> 2 files changed, 66 insertions(+), 6 deletions(-)
>
<chop>
> diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
> index ec90996184..8439cda6dd 100644
> --- a/meta/classes/package_ipk.bbclass
> +++ b/meta/classes/package_ipk.bbclass
> @@ -17,7 +17,29 @@ OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKA
> OPKGLIBDIR = "${localstatedir}/lib"
>
> python do_package_ipk () {
> - from multiprocessing import Process
> + import multiprocessing
> + import traceback
> +
> + class IPKWritePkgProcess(multiprocessing.Process):
> + def __init__(self, *args, **kwargs):
> + multiprocessing.Process.__init__(self, *args, **kwargs)
> + self._pconn, self._cconn = multiprocessing.Pipe()
> + self._exception = None
> +
> + def run(self):
> + try:
> + multiprocessing.Process.run(self)
> + self._cconn.send(None)
> + except Exception as e:
> + tb = traceback.format_exc()
> + self._cconn.send((e, tb))
> +
> + @property
> + def exception(self):
> + if self._pconn.poll():
> + self._exception = self._pconn.recv()
> + return self._exception
> +
Other than the name is IPKWritePkgProcess the same as DebianWritePkgProcess?
If so, could we put a single WritePkgProcess class somewhere it can be
shared in lib/oe ?
Joshua
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2017-09-05 8:31 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2017-09-04 21:35 [PATCH 1/2] package_deb, ipk: improve subprocess output on package manager command leonardo.sandoval.gonzalez
2017-09-04 21:35 ` [PATCH 2/2] package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages leonardo.sandoval.gonzalez
2017-09-05 8:31 ` Joshua Lock
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.