[PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh Ee Peng]

QA team were testing qemu boot image and shutdown on each
qemu architecture manually. Add automated test to test qemu boot on
ext4 and nfs, finally check that it can shutdown properly.

Original runqemu tests was dedicated for MACHINE=qemux86-64 and
it was testing various live image (iso and hddimg) will be able
to boot while live image was not supported on all qemu architecture.

The new tests were designed as a separate class as this tests
focus on testing qemu boot and shutdown on each qemu architecture.
Furthermore, this tests focus on testing qemu could shutdown
as expected.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/cases/runqemu.py | 61 +++++++++++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py b/meta/lib/oeqa/selftest/cases/runqemu.py
index 47d41f5..7288ab2 100644
--- a/meta/lib/oeqa/selftest/cases/runqemu.py
+++ b/meta/lib/oeqa/selftest/cases/runqemu.py
@@ -3,9 +3,10 @@
 #
 
 import re
-
+import tempfile
+import time
 from oeqa.selftest.case import OESelftestTestCase
-from oeqa.utils.commands import bitbake, runqemu, get_bb_var
+from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
 from oeqa.core.decorator.oeid import OETestID
 
 class RunqemuTests(OESelftestTestCase):
@@ -136,3 +137,59 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s" % (self.cmd_common, rootfs)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+
+class QemuTest(OESelftestTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(QemuTest, cls).setUpClass()
+        cls.recipe = 'core-image-minimal'
+        cls.machine =  get_bb_var('MACHINE')
+        cls.deploy_dir_image =  get_bb_var('DEPLOY_DIR_IMAGE')
+        cls.cmd_common = "runqemu nographic"
+        cls.qemuboot_conf = "%s-%s.qemuboot.conf" % (cls.recipe, cls.machine)
+        cls.qemuboot_conf = os.path.join(cls.deploy_dir_image, cls.qemuboot_conf)
+        result = bitbake(cls.recipe)
+
+    def _start_qemu_shutdown_check_if_shutdown_succeeded(self, qemu, timeout):
+        status, output = qemu.run_serial("shutdown -h now")
+        qemu.runner.stop_thread()
+        print('DEBUG: shutdown and stop thread')
+        time_track = 0
+        while True:
+            is_alive = qemu.check()
+            if not is_alive:
+                return True
+            if time_track > timeout:
+                return False
+            time.sleep(1)
+            time_track += 1
+            print(time_track)
+
+    def test_qemu_can_shutdown(self):
+        if not os.path.exists(self.qemuboot_conf):
+            self.skipTest("%s not found" % self.qemuboot_conf)
+        cmd = "%s %s" % (self.cmd_common, self.qemuboot_conf)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu does not shutdown within timeout(%s)' % shutdown_timeout)
+
+    def test_qemu_can_boot_nfs_and_shutdown(self):
+        bitbake('meta-ide-support')
+        rootfs_tar = "%s-%s.tar.bz2" % (self.recipe, self.machine)
+        rootfs_tar = os.path.join(self.deploy_dir_image, rootfs_tar)
+        if not os.path.exists(rootfs_tar):
+            self.skipTest("%s not found" % rootfs_tar)
+        tmpdir = tempfile.mkdtemp(prefix='qemu_nfs')
+        tmpdir_nfs = os.path.join(tmpdir, 'nfs')
+        cmd_extract_nfs = 'runqemu-extract-sdk %s %s' % (rootfs_tar, tmpdir_nfs)
+        runCmd(cmd_extract_nfs)
+        if not os.path.exists(self.qemuboot_conf):
+            self.skipTest("%s not found" % self.qemuboot_conf)
+        cmd = "%s nfs %s %s" % (self.cmd_common, self.qemuboot_conf, tmpdir_nfs)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu does not shutdown within timeout(%s)' % shutdown_timeout)
+        runCmd('rm -rf %s' % tmpdir)
-- 
2.7.4

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh, Ee Peng]

Hi Alex,

I added comment to that explain the need for a separate class, what the two classes do and how they differ.

Please let me know if you have any inputs. 

Thanks,
Ee Peng 

-----Original Message-----
From: openembedded-core-bounces@lists.openembedded.org [mailto:openembedded-core-bounces@lists.openembedded.org] On Behalf Of Yeoh Ee Peng
Sent: Tuesday, April 10, 2018 1:44 AM
To: openembedded-core@lists.openembedded.org
Subject: [OE-core] [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

QA team were testing qemu boot image and shutdown on each qemu architecture manually. Add automated test to test qemu boot on
ext4 and nfs, finally check that it can shutdown properly.

Original runqemu tests was dedicated for MACHINE=qemux86-64 and it was testing various live image (iso and hddimg) will be able to boot while live image was not supported on all qemu architecture.

The new tests were designed as a separate class as this tests focus on testing qemu boot and shutdown on each qemu architecture.
Furthermore, this tests focus on testing qemu could shutdown as expected.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/cases/runqemu.py | 61 +++++++++++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py b/meta/lib/oeqa/selftest/cases/runqemu.py
index 47d41f5..7288ab2 100644
--- a/meta/lib/oeqa/selftest/cases/runqemu.py
+++ b/meta/lib/oeqa/selftest/cases/runqemu.py
@@ -3,9 +3,10 @@
 #
 
 import re
-
+import tempfile
+import time
 from oeqa.selftest.case import OESelftestTestCase -from oeqa.utils.commands import bitbake, runqemu, get_bb_var
+from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
 from oeqa.core.decorator.oeid import OETestID
 
 class RunqemuTests(OESelftestTestCase):
@@ -136,3 +137,59 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s" % (self.cmd_common, rootfs)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+
+class QemuTest(OESelftestTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(QemuTest, cls).setUpClass()
+        cls.recipe = 'core-image-minimal'
+        cls.machine =  get_bb_var('MACHINE')
+        cls.deploy_dir_image =  get_bb_var('DEPLOY_DIR_IMAGE')
+        cls.cmd_common = "runqemu nographic"
+        cls.qemuboot_conf = "%s-%s.qemuboot.conf" % (cls.recipe, cls.machine)
+        cls.qemuboot_conf = os.path.join(cls.deploy_dir_image, cls.qemuboot_conf)
+        result = bitbake(cls.recipe)
+
+    def _start_qemu_shutdown_check_if_shutdown_succeeded(self, qemu, timeout):
+        status, output = qemu.run_serial("shutdown -h now")
+        qemu.runner.stop_thread()
+        print('DEBUG: shutdown and stop thread')
+        time_track = 0
+        while True:
+            is_alive = qemu.check()
+            if not is_alive:
+                return True
+            if time_track > timeout:
+                return False
+            time.sleep(1)
+            time_track += 1
+            print(time_track)
+
+    def test_qemu_can_shutdown(self):
+        if not os.path.exists(self.qemuboot_conf):
+            self.skipTest("%s not found" % self.qemuboot_conf)
+        cmd = "%s %s" % (self.cmd_common, self.qemuboot_conf)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu does 
+ not shutdown within timeout(%s)' % shutdown_timeout)
+
+    def test_qemu_can_boot_nfs_and_shutdown(self):
+        bitbake('meta-ide-support')
+        rootfs_tar = "%s-%s.tar.bz2" % (self.recipe, self.machine)
+        rootfs_tar = os.path.join(self.deploy_dir_image, rootfs_tar)
+        if not os.path.exists(rootfs_tar):
+            self.skipTest("%s not found" % rootfs_tar)
+        tmpdir = tempfile.mkdtemp(prefix='qemu_nfs')
+        tmpdir_nfs = os.path.join(tmpdir, 'nfs')
+        cmd_extract_nfs = 'runqemu-extract-sdk %s %s' % (rootfs_tar, tmpdir_nfs)
+        runCmd(cmd_extract_nfs)
+        if not os.path.exists(self.qemuboot_conf):
+            self.skipTest("%s not found" % self.qemuboot_conf)
+        cmd = "%s nfs %s %s" % (self.cmd_common, self.qemuboot_conf, tmpdir_nfs)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu does not shutdown within timeout(%s)' % shutdown_timeout)
+        runCmd('rm -rf %s' % tmpdir)
--
2.7.4

--
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Anuj Mittal]

On 04/10/2018 01:43 AM, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each
> qemu architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Original runqemu tests was dedicated for MACHINE=qemux86-64 and
> it was testing various live image (iso and hddimg) will be able
> to boot while live image was not supported on all qemu architecture.
> 
> The new tests were designed as a separate class as this tests
> focus on testing qemu boot and shutdown on each qemu architecture.
> Furthermore, this tests focus on testing qemu could shutdown
> as expected.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/selftest/cases/runqemu.py | 61 +++++++++++++++++++++++++++++++--
>  1 file changed, 59 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py b/meta/lib/oeqa/selftest/cases/runqemu.py
> index 47d41f5..7288ab2 100644
> --- a/meta/lib/oeqa/selftest/cases/runqemu.py
> +++ b/meta/lib/oeqa/selftest/cases/runqemu.py
> @@ -3,9 +3,10 @@
>  #
>  
>  import re
> -
> +import tempfile
> +import time
>  from oeqa.selftest.case import OESelftestTestCase
> -from oeqa.utils.commands import bitbake, runqemu, get_bb_var
> +from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
>  from oeqa.core.decorator.oeid import OETestID
>  
>  class RunqemuTests(OESelftestTestCase):
> @@ -136,3 +137,59 @@ SYSLINUX_TIMEOUT = "10"
>          cmd = "%s %s" % (self.cmd_common, rootfs)
>          with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
>              self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
> +
> +class QemuTest(OESelftestTestCase):
> +
> +    @classmethod
> +    def setUpClass(cls):
> +        super(QemuTest, cls).setUpClass()
> +        cls.recipe = 'core-image-minimal'
> +        cls.machine =  get_bb_var('MACHINE')
> +        cls.deploy_dir_image =  get_bb_var('DEPLOY_DIR_IMAGE')
> +        cls.cmd_common = "runqemu nographic"
> +        cls.qemuboot_conf = "%s-%s.qemuboot.conf" % (cls.recipe, cls.machine)
> +        cls.qemuboot_conf = os.path.join(cls.deploy_dir_image, cls.qemuboot_conf)
> +        result = bitbake(cls.recipe)

This 'result' isn't used.

> +
> +    def _start_qemu_shutdown_check_if_shutdown_succeeded(self, qemu, timeout):
> +        status, output = qemu.run_serial("shutdown -h now")
> +        qemu.runner.stop_thread()

If you call stop_thread like this, this test will probably always pass?
Please check the value of output and then is_alive/check().

> +        print('DEBUG: shutdown and stop thread')
> +        time_track = 0
> +        while True:
> +            is_alive = qemu.check()
> +            if not is_alive:
> +                return True
> +            if time_track > timeout:
> +                return False
> +            time.sleep(1)
> +            time_track += 1
> +            print(time_track)

We shouldn't be printing these values.

> +
> +    def test_qemu_can_shutdown(self):
> +        if not os.path.exists(self.qemuboot_conf):
> +            self.skipTest("%s not found" % self.qemuboot_conf)
> +        cmd = "%s %s" % (self.cmd_common, self.qemuboot_conf)
> +        shutdown_timeout = 120
> +        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
> +            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
> +            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu does not shutdown within timeout(%s)' % shutdown_timeout)
> +
> +    def test_qemu_can_boot_nfs_and_shutdown(self):
> +        bitbake('meta-ide-support')
> +        rootfs_tar = "%s-%s.tar.bz2" % (self.recipe, self.machine)
> +        rootfs_tar = os.path.join(self.deploy_dir_image, rootfs_tar)
> +        if not os.path.exists(rootfs_tar):
> +            self.skipTest("%s not found" % rootfs_tar)
> +        tmpdir = tempfile.mkdtemp(prefix='qemu_nfs')
> +        tmpdir_nfs = os.path.join(tmpdir, 'nfs')
> +        cmd_extract_nfs = 'runqemu-extract-sdk %s %s' % (rootfs_tar, tmpdir_nfs)
> +        runCmd(cmd_extract_nfs)
> +        if not os.path.exists(self.qemuboot_conf):
> +            self.skipTest("%s not found" % self.qemuboot_conf)

This and others should be marked failure I think if env wasn't setup
properly after doing bitbake. Does skipTest result in a failure?

Thanks,
Anuj

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh, Ee Peng]

Hi Anuj,

Thanks for your inputs on comment and code that was not used. I shall remove the comment and code that was not used. 

For the qemu.runner.stop_thread(), this was trigger to stop an instance of a LoggingThread class that was used primary to logging the qemu through serial console. Stopping this thread will not stop the qemu itself. In case that this logging thread object was not stop, when the qemu finally shutdown by the shutdown command, this logging thread will encounter an exception (Exception: Console connection closed unexpectedly). From the testing on multiple qemu architecture, this testcase was able to catch the shutdown hung error facing by existing qemuarm (https://bugzilla.yoctoproject.org/show_bug.cgi?id=12499), while it was as expected on qemux86, qemuppc, and other. I shall add comment to explain the important of qemu.runner.stop_thread() and the reason behind this. 

For the skipTest, I agreed with you, it shall be replace by assertion to highlight environment setup failure. 

Please let me know if you have any more input. Thank you very much!

Thanks,
Yeoh Ee Peng 


-----Original Message-----
From: Mittal, Anuj 
Sent: Tuesday, April 10, 2018 3:01 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

On 04/10/2018 01:43 AM, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each qemu 
> architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Original runqemu tests was dedicated for MACHINE=qemux86-64 and it was 
> testing various live image (iso and hddimg) will be able to boot while 
> live image was not supported on all qemu architecture.
> 
> The new tests were designed as a separate class as this tests focus on 
> testing qemu boot and shutdown on each qemu architecture.
> Furthermore, this tests focus on testing qemu could shutdown as 
> expected.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/selftest/cases/runqemu.py | 61 
> +++++++++++++++++++++++++++++++--
>  1 file changed, 59 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py 
> b/meta/lib/oeqa/selftest/cases/runqemu.py
> index 47d41f5..7288ab2 100644
> --- a/meta/lib/oeqa/selftest/cases/runqemu.py
> +++ b/meta/lib/oeqa/selftest/cases/runqemu.py
> @@ -3,9 +3,10 @@
>  #
>  
>  import re
> -
> +import tempfile
> +import time
>  from oeqa.selftest.case import OESelftestTestCase -from 
> oeqa.utils.commands import bitbake, runqemu, get_bb_var
> +from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
>  from oeqa.core.decorator.oeid import OETestID
>  
>  class RunqemuTests(OESelftestTestCase):
> @@ -136,3 +137,59 @@ SYSLINUX_TIMEOUT = "10"
>          cmd = "%s %s" % (self.cmd_common, rootfs)
>          with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
>              self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
> +
> +class QemuTest(OESelftestTestCase):
> +
> +    @classmethod
> +    def setUpClass(cls):
> +        super(QemuTest, cls).setUpClass()
> +        cls.recipe = 'core-image-minimal'
> +        cls.machine =  get_bb_var('MACHINE')
> +        cls.deploy_dir_image =  get_bb_var('DEPLOY_DIR_IMAGE')
> +        cls.cmd_common = "runqemu nographic"
> +        cls.qemuboot_conf = "%s-%s.qemuboot.conf" % (cls.recipe, cls.machine)
> +        cls.qemuboot_conf = os.path.join(cls.deploy_dir_image, cls.qemuboot_conf)
> +        result = bitbake(cls.recipe)

This 'result' isn't used.

> +
> +    def _start_qemu_shutdown_check_if_shutdown_succeeded(self, qemu, timeout):
> +        status, output = qemu.run_serial("shutdown -h now")
> +        qemu.runner.stop_thread()

If you call stop_thread like this, this test will probably always pass?
Please check the value of output and then is_alive/check().

> +        print('DEBUG: shutdown and stop thread')
> +        time_track = 0
> +        while True:
> +            is_alive = qemu.check()
> +            if not is_alive:
> +                return True
> +            if time_track > timeout:
> +                return False
> +            time.sleep(1)
> +            time_track += 1
> +            print(time_track)

We shouldn't be printing these values.

> +
> +    def test_qemu_can_shutdown(self):
> +        if not os.path.exists(self.qemuboot_conf):
> +            self.skipTest("%s not found" % self.qemuboot_conf)
> +        cmd = "%s %s" % (self.cmd_common, self.qemuboot_conf)
> +        shutdown_timeout = 120
> +        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
> +            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
> +            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu 
> + does not shutdown within timeout(%s)' % shutdown_timeout)
> +
> +    def test_qemu_can_boot_nfs_and_shutdown(self):
> +        bitbake('meta-ide-support')
> +        rootfs_tar = "%s-%s.tar.bz2" % (self.recipe, self.machine)
> +        rootfs_tar = os.path.join(self.deploy_dir_image, rootfs_tar)
> +        if not os.path.exists(rootfs_tar):
> +            self.skipTest("%s not found" % rootfs_tar)
> +        tmpdir = tempfile.mkdtemp(prefix='qemu_nfs')
> +        tmpdir_nfs = os.path.join(tmpdir, 'nfs')
> +        cmd_extract_nfs = 'runqemu-extract-sdk %s %s' % (rootfs_tar, tmpdir_nfs)
> +        runCmd(cmd_extract_nfs)
> +        if not os.path.exists(self.qemuboot_conf):
> +            self.skipTest("%s not found" % self.qemuboot_conf)

This and others should be marked failure I think if env wasn't setup properly after doing bitbake. Does skipTest result in a failure?

Thanks,
Anuj

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Alexander Kanavin]

On 04/09/2018 08:43 PM, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each
> qemu architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Original runqemu tests was dedicated for MACHINE=qemux86-64 and
> it was testing various live image (iso and hddimg) will be able
> to boot while live image was not supported on all qemu architecture.
> 
> The new tests were designed as a separate class as this tests
> focus on testing qemu boot and shutdown on each qemu architecture.
> Furthermore, this tests focus on testing qemu could shutdown
> as expected.

1. I believe the clock on the machine that you use to send the patches 
isn't set correctly, it seems to be several hours in the past.

2. What I meant is that you comment the source code itself, not write a 
longer commit message. So that anyone reading the actual file can 
quickly figure out why there are two classes and what they test. 
Basically take the above, and write it down as comments in the actual 
file; take your time to write a nice, clear explanation (similar to what 
we discussed). Then the commit message can be more brief, basically the 
first paragraph above is enough.

The reason for this is that commit history is less obvious or convenient 
to use when you want to find explanations. You can certainly use 'git 
log <filename>', but it comes with lots of irrelevant commits, and if 
the code was restructured in the past, then the history of changes stops 
there.

Alex

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh, Ee Peng]

Noted, I understood it now. Let me enhance the comment in the source code itself. 
It make sense. Thank you for your inputs! 

-----Original Message-----
From: Alexander Kanavin [mailto:alexander.kanavin@linux.intel.com] 
Sent: Tuesday, April 10, 2018 3:30 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

On 04/09/2018 08:43 PM, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each qemu 
> architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Original runqemu tests was dedicated for MACHINE=qemux86-64 and it was 
> testing various live image (iso and hddimg) will be able to boot while 
> live image was not supported on all qemu architecture.
> 
> The new tests were designed as a separate class as this tests focus on 
> testing qemu boot and shutdown on each qemu architecture.
> Furthermore, this tests focus on testing qemu could shutdown as 
> expected.

1. I believe the clock on the machine that you use to send the patches isn't set correctly, it seems to be several hours in the past.

2. What I meant is that you comment the source code itself, not write a longer commit message. So that anyone reading the actual file can quickly figure out why there are two classes and what they test. 
Basically take the above, and write it down as comments in the actual file; take your time to write a nice, clear explanation (similar to what we discussed). Then the commit message can be more brief, basically the first paragraph above is enough.

The reason for this is that commit history is less obvious or convenient to use when you want to find explanations. You can certainly use 'git log <filename>', but it comes with lots of irrelevant commits, and if the code was restructured in the past, then the history of changes stops there.

Alex

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Richard Purdie]

On Mon, 2018-04-09 at 10:43 -0700, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each
> qemu architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Original runqemu tests was dedicated for MACHINE=qemux86-64 and
> it was testing various live image (iso and hddimg) will be able
> to boot while live image was not supported on all qemu architecture.
> 
> The new tests were designed as a separate class as this tests
> focus on testing qemu boot and shutdown on each qemu architecture.
> Furthermore, this tests focus on testing qemu could shutdown
> as expected.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>

I do have one other question about this. There are various other tests
which boot qemu images. Would we be able to assume that because those
tests work, the start/stop of qemu is working? Or is there a unique
element we need to test specifically here?

Cheers,

Richard

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh, Ee Peng]

Hi Richard,

Yes, we are testing an alternative way to stop qemu here. From my understand, the existing testcases stop qemu through killing the qemu process. While this testcase was testing that after executing the shutdown command inside qemu, qemu will stop as expected. 

Currently, QA team was manually testing the execution of shutdown command on various qemu architecture. The team had found an shutdown hung issue on qemuarm.  
https://bugzilla.yoctoproject.org/show_bug.cgi?id=12499

Hope this explain the intention of this testcase. Please let me you opinion and input. 

Thanks,
Yeoh Ee Peng 

-----Original Message-----
From: Richard Purdie [mailto:richard.purdie@linuxfoundation.org] 
Sent: Tuesday, April 10, 2018 5:41 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

On Mon, 2018-04-09 at 10:43 -0700, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each qemu 
> architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Original runqemu tests was dedicated for MACHINE=qemux86-64 and it was 
> testing various live image (iso and hddimg) will be able to boot while 
> live image was not supported on all qemu architecture.
> 
> The new tests were designed as a separate class as this tests focus on 
> testing qemu boot and shutdown on each qemu architecture.
> Furthermore, this tests focus on testing qemu could shutdown as 
> expected.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>

I do have one other question about this. There are various other tests which boot qemu images. Would we be able to assume that because those tests work, the start/stop of qemu is working? Or is there a unique element we need to test specifically here?

Cheers,

Richard

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh, Ee Peng]

Richard,
Noted, thank you very much!
It was fine, QA team shall be able to manually trigger this selftest, at least it was more efficient than manually executing the testcase. 

Cheers,
Yeoh Ee Peng 

-----Original Message-----
From: Richard Purdie [mailto:richard.purdie@linuxfoundation.org] 
Sent: Friday, April 20, 2018 3:15 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

On Mon, 2018-04-09 at 18:28 -0700, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each qemu 
> architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/selftest/cases/runqemu.py | 70
> ++++++++++++++++++++++++++++++++-
>  1 file changed, 68 insertions(+), 2 deletions(-)

The test is good however it fails on our infrastructure since portmap/rpcbind aren't running. This is a new dependency for the test infrastructure we'd have to document so I don't want to merge this for 2.5. 

Since the code itself is fine, instead I've merged the test but disabled the NFS test by default and we'll need to revisit this in 2.6 to enable it.

Cheers,

Richard

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Richard Purdie]

On Mon, 2018-04-09 at 18:28 -0700, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each
> qemu architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>  meta/lib/oeqa/selftest/cases/runqemu.py | 70
> ++++++++++++++++++++++++++++++++-
>  1 file changed, 68 insertions(+), 2 deletions(-)

The test is good however it fails on our infrastructure since
portmap/rpcbind aren't running. This is a new dependency for the test
infrastructure we'd have to document so I don't want to merge this for
2.5. 

Since the code itself is fine, instead I've merged the test but
disabled the NFS test by default and we'll need to revisit this in 2.6
to enable it.

Cheers,

Richard

[PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh Ee Peng]

QA team were testing qemu boot image and shutdown on each
qemu architecture manually. Add automated test to test qemu boot on
ext4 and nfs, finally check that it can shutdown properly.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/cases/runqemu.py | 70 ++++++++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py b/meta/lib/oeqa/selftest/cases/runqemu.py
index 47d41f5..f3ff015 100644
--- a/meta/lib/oeqa/selftest/cases/runqemu.py
+++ b/meta/lib/oeqa/selftest/cases/runqemu.py
@@ -3,9 +3,10 @@
 #
 
 import re
-
+import tempfile
+import time
 from oeqa.selftest.case import OESelftestTestCase
-from oeqa.utils.commands import bitbake, runqemu, get_bb_var
+from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
 from oeqa.core.decorator.oeid import OETestID
 
 class RunqemuTests(OESelftestTestCase):
@@ -136,3 +137,68 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s" % (self.cmd_common, rootfs)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+
+# This test was designed as a separate class to test that shutdown
+# command will shutdown qemu as expected on each qemu architecture
+# based on the MACHINE configuration inside the config file
+# (eg. local.conf).
+#
+# This was different compared to RunqemuTests, where RunqemuTests was
+# dedicated for MACHINE=qemux86-64 where it test that qemux86-64 will
+# bootup various filesystem types, including live image(iso and hddimg)
+# where live image was not supported on all qemu architecture.
+class QemuTest(OESelftestTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(QemuTest, cls).setUpClass()
+        cls.recipe = 'core-image-minimal'
+        cls.machine =  get_bb_var('MACHINE')
+        cls.deploy_dir_image =  get_bb_var('DEPLOY_DIR_IMAGE')
+        cls.cmd_common = "runqemu nographic"
+        cls.qemuboot_conf = "%s-%s.qemuboot.conf" % (cls.recipe, cls.machine)
+        cls.qemuboot_conf = os.path.join(cls.deploy_dir_image, cls.qemuboot_conf)
+        bitbake(cls.recipe)
+
+    def _start_qemu_shutdown_check_if_shutdown_succeeded(self, qemu, timeout):
+        qemu.run_serial("shutdown -h now")
+        # Stop thread will stop the LoggingThread instance used for logging
+        # qemu through serial console, stop thread will prevent this code
+        # from facing exception (Console connection closed unexpectedly)
+        # when qemu was shutdown by the above shutdown command
+        qemu.runner.stop_thread()
+        time_track = 0
+        while True:
+            is_alive = qemu.check()
+            if not is_alive:
+                return True
+            if time_track > timeout:
+                return False
+            time.sleep(1)
+            time_track += 1
+
+    def test_qemu_can_shutdown(self):
+        self.assertExists(self.qemuboot_conf)
+        cmd = "%s %s" % (self.cmd_common, self.qemuboot_conf)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: %s does not shutdown within timeout(%s)' % (self.machine, shutdown_timeout))
+
+    def test_qemu_can_boot_nfs_and_shutdown(self):
+        self.assertExists(self.qemuboot_conf)
+        bitbake('meta-ide-support')
+        rootfs_tar = "%s-%s.tar.bz2" % (self.recipe, self.machine)
+        rootfs_tar = os.path.join(self.deploy_dir_image, rootfs_tar)
+        self.assertExists(rootfs_tar)
+        tmpdir = tempfile.mkdtemp(prefix='qemu_nfs')
+        tmpdir_nfs = os.path.join(tmpdir, 'nfs')
+        cmd_extract_nfs = 'runqemu-extract-sdk %s %s' % (rootfs_tar, tmpdir_nfs)
+        result = runCmd(cmd_extract_nfs)
+        self.assertEqual(0, result.status, "runqemu-extract-sdk didn't run as expected. %s" % result.output)
+        cmd = "%s nfs %s %s" % (self.cmd_common, self.qemuboot_conf, tmpdir_nfs)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: %s does not shutdown within timeout(%s)' % (self.machine, shutdown_timeout))
+        runCmd('rm -rf %s' % tmpdir)
-- 
2.7.4

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh, Ee Peng]

Noted, will add comment and resend the patch. Thank you for your inputs. 

-----Original Message-----
From: Alexander Kanavin [mailto:alexander.kanavin@linux.intel.com] 
Sent: Monday, April 9, 2018 8:34 PM
To: Yeoh, Ee Peng <ee.peng.yeoh@intel.com>; openembedded-core@lists.openembedded.org
Subject: Re: [OE-core] [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

On 04/09/2018 06:32 AM, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each qemu 
> architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>   meta/lib/oeqa/selftest/cases/runqemu.py | 61 +++++++++++++++++++++++++++++++--
>   1 file changed, 59 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py 
> b/meta/lib/oeqa/selftest/cases/runqemu.py
> index 47d41f5..7288ab2 100644
> --- a/meta/lib/oeqa/selftest/cases/runqemu.py
> +++ b/meta/lib/oeqa/selftest/cases/runqemu.py
> @@ -3,9 +3,10 @@
>   #
>   
>   import re
> -
> +import tempfile
> +import time
>   from oeqa.selftest.case import OESelftestTestCase -from 
> oeqa.utils.commands import bitbake, runqemu, get_bb_var
> +from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
>   from oeqa.core.decorator.oeid import OETestID
>   
>   class RunqemuTests(OESelftestTestCase):
> @@ -136,3 +137,59 @@ SYSLINUX_TIMEOUT = "10"
>           cmd = "%s %s" % (self.cmd_common, rootfs)
>           with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
>               self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
> +
> +class QemuTest(OESelftestTestCase):

Please add comments that explain the need for a separate class, what the two classes do and how they differ.


Alex

Re: [PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Alexander Kanavin]

On 04/09/2018 06:32 AM, Yeoh Ee Peng wrote:
> QA team were testing qemu boot image and shutdown on each
> qemu architecture manually. Add automated test to test qemu boot on
> ext4 and nfs, finally check that it can shutdown properly.
> 
> Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
> ---
>   meta/lib/oeqa/selftest/cases/runqemu.py | 61 +++++++++++++++++++++++++++++++--
>   1 file changed, 59 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py b/meta/lib/oeqa/selftest/cases/runqemu.py
> index 47d41f5..7288ab2 100644
> --- a/meta/lib/oeqa/selftest/cases/runqemu.py
> +++ b/meta/lib/oeqa/selftest/cases/runqemu.py
> @@ -3,9 +3,10 @@
>   #
>   
>   import re
> -
> +import tempfile
> +import time
>   from oeqa.selftest.case import OESelftestTestCase
> -from oeqa.utils.commands import bitbake, runqemu, get_bb_var
> +from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
>   from oeqa.core.decorator.oeid import OETestID
>   
>   class RunqemuTests(OESelftestTestCase):
> @@ -136,3 +137,59 @@ SYSLINUX_TIMEOUT = "10"
>           cmd = "%s %s" % (self.cmd_common, rootfs)
>           with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
>               self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
> +
> +class QemuTest(OESelftestTestCase):

Please add comments that explain the need for a separate class, what the 
two classes do and how they differ.


Alex

[PATCH] oe-selftest: runqemu: add tests for qemu boot and shutdown

[Yeoh Ee Peng]

QA team were testing qemu boot image and shutdown on each
qemu architecture manually. Add automated test to test qemu boot on
ext4 and nfs, finally check that it can shutdown properly.

Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
---
 meta/lib/oeqa/selftest/cases/runqemu.py | 61 +++++++++++++++++++++++++++++++--
 1 file changed, 59 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/runqemu.py b/meta/lib/oeqa/selftest/cases/runqemu.py
index 47d41f5..7288ab2 100644
--- a/meta/lib/oeqa/selftest/cases/runqemu.py
+++ b/meta/lib/oeqa/selftest/cases/runqemu.py
@@ -3,9 +3,10 @@
 #
 
 import re
-
+import tempfile
+import time
 from oeqa.selftest.case import OESelftestTestCase
-from oeqa.utils.commands import bitbake, runqemu, get_bb_var
+from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
 from oeqa.core.decorator.oeid import OETestID
 
 class RunqemuTests(OESelftestTestCase):
@@ -136,3 +137,59 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s" % (self.cmd_common, rootfs)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+
+class QemuTest(OESelftestTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(QemuTest, cls).setUpClass()
+        cls.recipe = 'core-image-minimal'
+        cls.machine =  get_bb_var('MACHINE')
+        cls.deploy_dir_image =  get_bb_var('DEPLOY_DIR_IMAGE')
+        cls.cmd_common = "runqemu nographic"
+        cls.qemuboot_conf = "%s-%s.qemuboot.conf" % (cls.recipe, cls.machine)
+        cls.qemuboot_conf = os.path.join(cls.deploy_dir_image, cls.qemuboot_conf)
+        result = bitbake(cls.recipe)
+
+    def _start_qemu_shutdown_check_if_shutdown_succeeded(self, qemu, timeout):
+        status, output = qemu.run_serial("shutdown -h now")
+        qemu.runner.stop_thread()
+        print('DEBUG: shutdown and stop thread')
+        time_track = 0
+        while True:
+            is_alive = qemu.check()
+            if not is_alive:
+                return True
+            if time_track > timeout:
+                return False
+            time.sleep(1)
+            time_track += 1
+            print(time_track)
+
+    def test_qemu_can_shutdown(self):
+        if not os.path.exists(self.qemuboot_conf):
+            self.skipTest("%s not found" % self.qemuboot_conf)
+        cmd = "%s %s" % (self.cmd_common, self.qemuboot_conf)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu does not shutdown within timeout(%s)' % shutdown_timeout)
+
+    def test_qemu_can_boot_nfs_and_shutdown(self):
+        bitbake('meta-ide-support')
+        rootfs_tar = "%s-%s.tar.bz2" % (self.recipe, self.machine)
+        rootfs_tar = os.path.join(self.deploy_dir_image, rootfs_tar)
+        if not os.path.exists(rootfs_tar):
+            self.skipTest("%s not found" % rootfs_tar)
+        tmpdir = tempfile.mkdtemp(prefix='qemu_nfs')
+        tmpdir_nfs = os.path.join(tmpdir, 'nfs')
+        cmd_extract_nfs = 'runqemu-extract-sdk %s %s' % (rootfs_tar, tmpdir_nfs)
+        runCmd(cmd_extract_nfs)
+        if not os.path.exists(self.qemuboot_conf):
+            self.skipTest("%s not found" % self.qemuboot_conf)
+        cmd = "%s nfs %s %s" % (self.cmd_common, self.qemuboot_conf, tmpdir_nfs)
+        shutdown_timeout = 120
+        with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
+            qemu_shutdown_succeeded = self._start_qemu_shutdown_check_if_shutdown_succeeded(qemu, shutdown_timeout)
+            self.assertTrue(qemu_shutdown_succeeded, 'Failed: qemu does not shutdown within timeout(%s)' % shutdown_timeout)
+        runCmd('rm -rf %s' % tmpdir)
-- 
2.7.4