[PATCH 1/4] oeqa/concurrencytest: Rename variables to improve the code

[PATCH 1/4] oeqa/concurrencytest: Rename variables to improve the code

[Richard Purdie]

Each time I look at this code I get confused about what the different
variables represent. Rename a few of them to better indicate what they
represent.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/core/utils/concurrencytest.py | 26 ++++++++++-----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py
index b2eb68fb024..347dc896020 100644
--- a/meta/lib/oeqa/core/utils/concurrencytest.py
+++ b/meta/lib/oeqa/core/utils/concurrencytest.py
@@ -190,13 +190,13 @@ class ConcurrentTestSuite(unittest.TestSuite):
         self.removefunc = removefunc
 
     def run(self, result):
-        tests, totaltests = fork_for_tests(self.processes, self)
+        testservers, totaltests = fork_for_tests(self.processes, self)
         try:
             threads = {}
             queue = Queue()
             semaphore = threading.Semaphore(1)
             result.threadprogress = {}
-            for i, (test, testnum) in enumerate(tests):
+            for i, (testserver, testnum) in enumerate(testservers):
                 result.threadprogress[i] = []
                 process_result = BBThreadsafeForwardingResult(
                         ExtraResultsDecoderTestResult(result),
@@ -210,8 +210,8 @@ class ConcurrentTestSuite(unittest.TestSuite):
                 process_result._stdout_buffer = io.StringIO()
                 process_result._stdout_buffer.buffer = dummybuf(process_result._stdout_buffer)
                 reader_thread = threading.Thread(
-                    target=self._run_test, args=(test, process_result, queue))
-                threads[test] = reader_thread, process_result
+                    target=self._run_test, args=(testserver, process_result, queue))
+                threads[testserver] = reader_thread, process_result
                 reader_thread.start()
             while threads:
                 finished_test = queue.get()
@@ -222,13 +222,13 @@ class ConcurrentTestSuite(unittest.TestSuite):
                 process_result.stop()
             raise
         finally:
-            for test in tests:
-                test[0]._stream.close()
+            for testserver in testservers:
+                testserver[0]._stream.close()
 
-    def _run_test(self, test, process_result, queue):
+    def _run_test(self, testserver, process_result, queue):
         try:
             try:
-                test.run(process_result)
+                testserver.run(process_result)
             except Exception:
                 # The run logic itself failed
                 case = testtools.ErrorHolder(
@@ -236,10 +236,10 @@ class ConcurrentTestSuite(unittest.TestSuite):
                     error=sys.exc_info())
                 case.run(process_result)
         finally:
-            queue.put(test)
+            queue.put(testserver)
 
 def fork_for_tests(concurrency_num, suite):
-    result = []
+    testservers = []
     if 'BUILDDIR' in os.environ:
         selftestdir = get_test_layer()
 
@@ -306,9 +306,9 @@ def fork_for_tests(concurrency_num, suite):
         else:
             os.close(c2pwrite)
             stream = os.fdopen(c2pread, 'rb', 1)
-            test = ProtocolTestCase(stream)
-            result.append((test, numtests))
-    return result, totaltests
+            testserver = ProtocolTestCase(stream)
+            testservers.append((testserver, numtests))
+    return testservers, totaltests
 
 def partition_tests(suite, count):
     # Keep tests from the same class together but allow tests from modules
-- 
2.30.2

[PATCH 2/4] oeqa/concurrencytest: Fix display of test stdout/stderr

[Richard Purdie]

If oe-selftest is run with -j, the output to stdout/stderr is being
lost at present. Capture this and display it upon test failure. We
have code that previously tried to enable this but it wasn't functioning
correctly. This should give more usable error reports on the autobuilder.

This code will mix stdout and stderr as the output is streamed from the test
server without markup. This is most in keeping with subunit/testools though
and the easiest way to handle the various challenges here as far as I can
see.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/core/utils/concurrencytest.py | 33 +++++++++++----------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py
index 347dc896020..161a2f6e904 100644
--- a/meta/lib/oeqa/core/utils/concurrencytest.py
+++ b/meta/lib/oeqa/core/utils/concurrencytest.py
@@ -48,11 +48,15 @@ _all__ = [
 #
 class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
 
-    def __init__(self, target, semaphore, threadnum, totalinprocess, totaltests):
+    def __init__(self, target, semaphore, threadnum, totalinprocess, totaltests, output, finalresult):
         super(BBThreadsafeForwardingResult, self).__init__(target, semaphore)
         self.threadnum = threadnum
         self.totalinprocess = totalinprocess
         self.totaltests = totaltests
+        self.buffer = True
+        self.outputbuf = output
+        self.finalresult = finalresult
+        self.finalresult.buffer = True
 
     def _add_result_with_semaphore(self, method, test, *args, **kwargs):
         self.semaphore.acquire()
@@ -71,6 +75,8 @@ class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
                     test.id())
         finally:
             self.semaphore.release()
+        self.finalresult._stderr_buffer = io.StringIO(initial_value=self.outputbuf.getvalue().decode("utf-8"))
+        self.finalresult._stdout_buffer = io.StringIO()
         super(BBThreadsafeForwardingResult, self)._add_result_with_semaphore(method, test, *args, **kwargs)
 
 class ProxyTestResult:
@@ -196,19 +202,11 @@ class ConcurrentTestSuite(unittest.TestSuite):
             queue = Queue()
             semaphore = threading.Semaphore(1)
             result.threadprogress = {}
-            for i, (testserver, testnum) in enumerate(testservers):
+            for i, (testserver, testnum, output) in enumerate(testservers):
                 result.threadprogress[i] = []
                 process_result = BBThreadsafeForwardingResult(
                         ExtraResultsDecoderTestResult(result),
-                        semaphore, i, testnum, totaltests)
-                # Force buffering of stdout/stderr so the console doesn't get corrupted by test output
-                # as per default in parent code
-                process_result.buffer = True
-                # We have to add a buffer object to stdout to keep subunit happy
-                process_result._stderr_buffer = io.StringIO()
-                process_result._stderr_buffer.buffer = dummybuf(process_result._stderr_buffer)
-                process_result._stdout_buffer = io.StringIO()
-                process_result._stdout_buffer.buffer = dummybuf(process_result._stdout_buffer)
+                        semaphore, i, testnum, totaltests, output, result)
                 reader_thread = threading.Thread(
                     target=self._run_test, args=(testserver, process_result, queue))
                 threads[testserver] = reader_thread, process_result
@@ -273,10 +271,11 @@ def fork_for_tests(concurrency_num, suite):
                 newsi = os.open(os.devnull, os.O_RDWR)
                 os.dup2(newsi, sys.stdin.fileno())
 
+                # Send stdout/stderr over the stream
+                os.dup2(c2pwrite, sys.stdout.fileno())
+                os.dup2(c2pwrite, sys.stderr.fileno())
+
                 subunit_client = TestProtocolClient(stream)
-                # Force buffering of stdout/stderr so the console doesn't get corrupted by test output
-                # as per default in parent code
-                subunit_client.buffer = True
                 subunit_result = AutoTimingTestResultDecorator(subunit_client)
                 unittest_result = process_suite.run(ExtraResultsEncoderTestResult(subunit_result))
                 if ourpid != os.getpid():
@@ -306,8 +305,10 @@ def fork_for_tests(concurrency_num, suite):
         else:
             os.close(c2pwrite)
             stream = os.fdopen(c2pread, 'rb', 1)
-            testserver = ProtocolTestCase(stream)
-            testservers.append((testserver, numtests))
+            # Collect stdout/stderr into an io buffer
+            output = io.BytesIO()
+            testserver = ProtocolTestCase(stream, passthrough=output)
+            testservers.append((testserver, numtests, output))
     return testservers, totaltests
 
 def partition_tests(suite, count):
-- 
2.30.2

[PATCH 3/4] diffoscope: Upgrade 168 -> 172

[Richard Purdie]

In particular 170 includes rpm header fixes which stop the webpages
for rpm diffs breaking web browsers and are important in the context
of the autobuilder.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 .../diffoscope/{diffoscope_168.bb => diffoscope_172.bb}         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename meta/recipes-support/diffoscope/{diffoscope_168.bb => diffoscope_172.bb} (92%)

diff --git a/meta/recipes-support/diffoscope/diffoscope_168.bb b/meta/recipes-support/diffoscope/diffoscope_172.bb
similarity index 92%
rename from meta/recipes-support/diffoscope/diffoscope_168.bb
rename to meta/recipes-support/diffoscope/diffoscope_172.bb
index a7cfdf9ffbe..bf4726e7789 100644
--- a/meta/recipes-support/diffoscope/diffoscope_168.bb
+++ b/meta/recipes-support/diffoscope/diffoscope_172.bb
@@ -12,7 +12,7 @@ PYPI_PACKAGE = "diffoscope"
 
 inherit pypi setuptools3
 
-SRC_URI[sha256sum] = "c6f1dc3e75b7e2e5ceac4f857fbd2ee0ddb3f0169c2b39ea9187af34208e98de"
+SRC_URI[sha256sum] = "5ffe7f38555c6409bc7e7edc277ed77dd78641fe1306fc38d153dbbe445ddea4"
 
 RDEPENDS_${PN} += "binutils vim squashfs-tools python3-libarchive-c python3-magic python3-rpm"
 
-- 
2.30.2

[PATCH 4/4] oeqa/runqemu: Support RUNQEMU_TMPFS_DIR as a location to copy snapshot images to

[Richard Purdie]

We have a working theory that IO queues on the autobuilder are impacting
runtime testing under qemu, particularly async writes which inice does not
influence. We already pass the snapshot option to qemu which copies the
image and runs out of the copy. Add in the ability to copy the image to
a specificed location which can be a tmpfs. This means that writes to the
image would no longer be blocked by other writes to disk in the system.

Preliminary tests show that this does improve the qemu errors at the expense
of sometimes showing qemu startup timeouts as on a loaded system with a large
test image, it can take longer than 120s to copy the image to tmpfs. Having
a most consistent failure mode for loaded tests is probably desireable though.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes/testimage.bbclass        | 1 +
 meta/lib/oeqa/core/target/qemu.py     | 4 ++--
 meta/lib/oeqa/targetcontrol.py        | 2 ++
 meta/lib/oeqa/utils/qemurunner.py     | 6 +++++-
 meta/lib/oeqa/utils/qemutinyrunner.py | 6 +++++-
 scripts/runqemu                       | 9 +++++++++
 6 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 78da4b09bdf..e6137595031 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -305,6 +305,7 @@ def testimage_main(d):
                       'dump_dir'    : d.getVar("TESTIMAGE_DUMP_DIR"),
                       'serial_ports': len(d.getVar("SERIAL_CONSOLES").split()),
                       'ovmf'        : ovmf,
+                      'tmpfsdir'    : d.getVar("RUNQEMU_TMPFS_DIR"),
                     }
 
     if d.getVar("TESTIMAGE_BOOT_PATTERNS"):
diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py
index 0f29414df54..792efca1f84 100644
--- a/meta/lib/oeqa/core/target/qemu.py
+++ b/meta/lib/oeqa/core/target/qemu.py
@@ -21,7 +21,7 @@ class OEQemuTarget(OESSHTarget):
             port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False,
             dump_dir='', dump_host_cmds='', display='', bootlog='',
             tmpdir='', dir_image='', boottime=60, serial_ports=2,
-            boot_patterns = defaultdict(str), ovmf=False, **kwargs):
+            boot_patterns = defaultdict(str), ovmf=False, tmpfsdir=None, **kwargs):
 
         super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout,
                 user, port)
@@ -42,7 +42,7 @@ class OEQemuTarget(OESSHTarget):
                                  use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir,
                                  dump_host_cmds=dump_host_cmds, logger=logger,
                                  serial_ports=serial_ports, boot_patterns = boot_patterns, 
-                                 use_ovmf=ovmf)
+                                 use_ovmf=ovmf, tmpfsdir=tmpfsdir)
         dump_target_cmds = kwargs.get("testimage_dump_target")
         self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner)
         self.target_dumper.create_dir("qemu")
diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py
index 19f5a4ea7e9..12057f855ae 100644
--- a/meta/lib/oeqa/targetcontrol.py
+++ b/meta/lib/oeqa/targetcontrol.py
@@ -131,6 +131,7 @@ class QemuTarget(BaseTarget):
                             logfile = self.qemulog,
                             kernel = self.kernel,
                             boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")),
+                            tmpfsdir = d.getVar("RUNQEMU_TMPFS_DIR"),
                             logger = logger)
         else:
             self.runner = QemuRunner(machine=d.getVar("MACHINE"),
@@ -144,6 +145,7 @@ class QemuTarget(BaseTarget):
                             dump_dir = dump_dir,
                             dump_host_cmds = d.getVar("testimage_dump_host"),
                             logger = logger,
+                            tmpfsdir = d.getVar("RUNQEMU_TMPFS_DIR"),
                             serial_ports = len(d.getVar("SERIAL_CONSOLES").split()))
 
         self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner)
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index eb23dbceb85..278904ba0b1 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -32,7 +32,7 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars)))
 class QemuRunner:
 
     def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds,
-                 use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None):
+                 use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None, tmpfsdir=None):
 
         # Popen object for runqemu
         self.runqemu = None
@@ -61,6 +61,7 @@ class QemuRunner:
         self.serial_ports = serial_ports
         self.msg = ''
         self.boot_patterns = boot_patterns
+        self.tmpfsdir = tmpfsdir
 
         self.runqemutime = 120
         if not workdir:
@@ -150,6 +151,9 @@ class QemuRunner:
         else:
             env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
 
+        if self.tmpfsdir:
+            env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir
+
         if not launch_cmd:
             launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '')
             if self.use_kvm:
diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
index 5c92941c0a6..20009401ca8 100644
--- a/meta/lib/oeqa/utils/qemutinyrunner.py
+++ b/meta/lib/oeqa/utils/qemutinyrunner.py
@@ -19,7 +19,7 @@ from .qemurunner import QemuRunner
 
 class QemuTinyRunner(QemuRunner):
 
-    def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger):
+    def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger, tmpfsdir=None):
 
         # Popen object for runqemu
         self.runqemu = None
@@ -37,6 +37,7 @@ class QemuTinyRunner(QemuRunner):
         self.deploy_dir_image = deploy_dir_image
         self.logfile = logfile
         self.boottime = boottime
+        self.tmpfsdir = tmpfsdir
 
         self.runqemutime = 60
         self.socketfile = "console.sock"
@@ -83,6 +84,9 @@ class QemuTinyRunner(QemuRunner):
             return False
         else:
             os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
+        if self.tmpfsdir:
+            env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir
+
 
         # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact
         # badly with screensavers.
diff --git a/scripts/runqemu b/scripts/runqemu
index 35053de368d..ba0b701aff3 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -1196,6 +1196,15 @@ class BaseConfig(object):
             self.fstype = self.fstype[4:]
         rootfs_format = self.fstype if self.fstype in ('vmdk', 'vhd', 'vhdx', 'qcow2', 'vdi') else 'raw'
 
+        tmpfsdir = os.environ.get("RUNQEMU_TMPFS_DIR", None)
+        if self.snapshot and tmpfsdir:
+            newrootfs = os.path.join(tmpfsdir, os.path.basename(self.rootfs)) + "." + str(os.getpid())
+            shutil.copyfile(self.rootfs, newrootfs)
+            #print("Copying rootfs to tmpfs: %s" % newrootfs)
+            self.rootfs = newrootfs
+            # Don't need a second copy now!
+            self.snapshot = False
+
         qb_rootfs_opt = self.get('QB_ROOTFS_OPT')
         if qb_rootfs_opt:
             self.rootfs_options = qb_rootfs_opt.replace('@ROOTFS@', self.rootfs)
-- 
2.30.2