[PATCH v2 0/3] testimage: additional reports for ptests

[PATCH v2 0/3] testimage: additional reports for ptests

[Adrian Freihofer]

Changes from v1 to v2 of these patches:

- Remove artificial ptest example
- Update the existing service skeleton
  - add support for systemd
  - make it a ptest example

Adrian Freihofer (3):
  meta-skeleton: update service example
  testimage: support additional reports for ptests
  runtime_test.py: add new testimage ptest test case

 .../service/service/run-ptest                 | 45 +++++++++++++++++++
 .../service/service/skeleton.service          |  9 ++++
 .../recipes-skeleton/service/service_0.1.bb   | 32 ++++++++++---
 meta/classes/testimage.bbclass                |  5 +++
 meta/lib/oeqa/runtime/cases/ptest.py          | 24 ++++++++++
 meta/lib/oeqa/selftest/cases/runtime_test.py  | 44 ++++++++++++++++++
 6 files changed, 154 insertions(+), 5 deletions(-)
 create mode 100644 meta-skeleton/recipes-skeleton/service/service/run-ptest
 create mode 100644 meta-skeleton/recipes-skeleton/service/service/skeleton.service

-- 
2.31.1

[PATCH v2 1/3] meta-skeleton: update service example

[Adrian Freihofer]

- Support systemd as well
- Add a simple ptest
---
 .../service/service/run-ptest                 | 45 +++++++++++++++++++
 .../service/service/skeleton.service          |  9 ++++
 .../recipes-skeleton/service/service_0.1.bb   | 32 ++++++++++---
 3 files changed, 81 insertions(+), 5 deletions(-)
 create mode 100644 meta-skeleton/recipes-skeleton/service/service/run-ptest
 create mode 100644 meta-skeleton/recipes-skeleton/service/service/skeleton.service

diff --git a/meta-skeleton/recipes-skeleton/service/service/run-ptest b/meta-skeleton/recipes-skeleton/service/service/run-ptest
new file mode 100644
index 0000000000..16d172609e
--- /dev/null
+++ b/meta-skeleton/recipes-skeleton/service/service/run-ptest
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+failures=0
+
+# The result should be PASS, FAIL, or SKIP, and the testname can be any identifying string.
+result_autostart=0
+testname="skeleton autostart"
+if @@STATUS_COMMAND@@; then
+    echo "PASS:  ${testname}"
+else
+    echo "FAIL:  ${testname}"
+    result_autostart=1
+    failures="$(expr "$failures" + 1)"
+fi
+
+
+# Optional ----------------------
+# A ptest might provide a machine readable report such as JUnit XML.
+# Reports are collected by the ptest imagetest if the TESTIMAGE_PTEST_REPORT_DIR
+# variable is configured for the tested image.
+# Example to fetch a xml report to ${TEST_LOG_DIR}/reports-xml/skeleton-test.xml:
+#   TESTIMAGE_PTEST_REPORT_DIR ?= "/tmp/ptest-xml/*.xml:reports-xml"
+# Usually one of the common unit test frameworks such as googletest or python unittest would generate a xml file.
+# Example for calling a C++ goolgetest test: my-gtest --gtest_output="xml:/tmp/ptest-xml/"
+XML_DIR="/tmp/ptest-xml"
+XML_FILE="${XML_DIR}/skeleton-test.xml"
+timestamp="$(date +"%Y-%m-%dT%H:%M:%S")"
+
+mkdir -p "$XML_DIR"
+cat << xxxEOFxxx > "$XML_FILE"
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="0" disabled="0" errors="0" time="0.010" timestamp="$timestamp" name="AllTests">
+  <testsuite name="ServiceTests" tests="1" failures="$failures" disabled="0" errors="0" time="0.010" timestamp="$timestamp">
+    <testcase name="SkeletonAutostart" status="completed" result="$result_autostart" time="0.010" timestamp="$timestamp" classname="SystemTest"/>
+  </testsuite>
+</testsuites>
+xxxEOFxxx
+# End Optional ----------------------
+
+
+# The ptest-runner evaluates the exit value of a test case: 0 means pass, 1 means fail.
+if [ "$failures" -eq 0 ]; then
+    exit 0
+fi
+exit 1
diff --git a/meta-skeleton/recipes-skeleton/service/service/skeleton.service b/meta-skeleton/recipes-skeleton/service/service/skeleton.service
new file mode 100644
index 0000000000..ee1f8c2e21
--- /dev/null
+++ b/meta-skeleton/recipes-skeleton/service/service/skeleton.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=skeleton demo service
+
+[Service]
+Type=forking
+ExecStart=@SBINDIR@/skeleton-test
+
+[Install]
+WantedBy=multi-user.target
diff --git a/meta-skeleton/recipes-skeleton/service/service_0.1.bb b/meta-skeleton/recipes-skeleton/service/service_0.1.bb
index 669d173ad1..07c8756071 100644
--- a/meta-skeleton/recipes-skeleton/service/service_0.1.bb
+++ b/meta-skeleton/recipes-skeleton/service/service_0.1.bb
@@ -4,16 +4,24 @@ DESCRIPTION = "This recipe is a canonical example of init scripts"
 LICENSE = "GPLv2"
 LIC_FILES_CHKSUM = "file://${WORKDIR}/COPYRIGHT;md5=349c872e0066155e1818b786938876a4"
 
-SRC_URI = "file://skeleton \
-	   file://skeleton_test.c \
-	   file://COPYRIGHT \
-	   "
+SRC_URI = "\
+    file://skeleton \
+    file://skeleton.service \
+    file://skeleton_test.c \
+    file://COPYRIGHT \
+    file://run-ptest \
+"
+
+# A recipe is "ptest-enabled" if it inherits the ptest class
+inherit update-rc.d systemd ptest
+
 
 do_compile () {
 	${CC} ${CFLAGS} ${LDFLAGS} ${WORKDIR}/skeleton_test.c -o ${WORKDIR}/skeleton-test
 }
 
 do_install () {
+	# init script
 	install -d ${D}${sysconfdir}/init.d
 	cat ${WORKDIR}/skeleton | \
 	  sed -e 's,/etc,${sysconfdir},g' \
@@ -23,10 +31,24 @@ do_install () {
 	      -e 's,/usr,${prefix},g' > ${D}${sysconfdir}/init.d/skeleton
 	chmod a+x ${D}${sysconfdir}/init.d/skeleton
 
+	# systemd service file
+	install -d ${D}${systemd_system_unitdir}
+	install -m 0644 ${WORKDIR}/skeleton.service ${D}${systemd_system_unitdir}
+	sed -i -e 's,@SBINDIR@,${sbindir},g' ${D}${systemd_system_unitdir}/skeleton.service
+
+	# skeleton-test
 	install -d ${D}${sbindir}
 	install -m 0755 ${WORKDIR}/skeleton-test ${D}${sbindir}/
 }
 
-RDEPENDS_${PN} = "initscripts"
+do_install_ptest() {
+	if ${@bb.utils.contains("DISTRO_FEATURES", "systemd", "true", "false", d)}; then
+		sed -i -e 's,@@STATUS_COMMAND@@,systemctl is-active --quiet skeleton,g' ${D}${PTEST_PATH}/run-ptest
+	else
+		sed -i -e 's,@@STATUS_COMMAND@@,${sysconfdir}/init.d/skeleton status,g' ${D}${PTEST_PATH}/run-ptest
+	fi
+}
 
 CONFFILES_${PN} += "${sysconfdir}/init.d/skeleton"
+INITSCRIPT_NAME = "skeleton"
+SYSTEMD_SERVICE_${PN} = "skeleton.service"
-- 
2.31.1

[PATCH v2 2/3] testimage: support additional reports for ptests

[Adrian Freihofer]

This adds a new optional feature to the ptest run-time test. Most unit
test frameworks such as googletest generate JUnit like XML reports which
can be processed e.g. by GitLab CI or Jenkins.

Example: A run-ptest script executes a googletest based unit test:
  /usr/bin/my-unittest --gtest_output="xml:/tmp/ptest-xml/"

The new variable TESTIMAGE_PTEST_REPORT_DIR allows to configure
bitbake -c testimage to fetch the reports from the target device and
store them into a subfolder of TEST_LOG_DIR. It's possible to fetch
report files from different locations on the target device to different
subfolders on the host.
---
 meta/classes/testimage.bbclass       |  5 +++++
 meta/lib/oeqa/runtime/cases/ptest.py | 24 ++++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 43de9d4d76..d01892136f 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -47,6 +47,11 @@ TESTIMAGE_AUTO ??= "0"
 # TESTIMAGE_BOOT_PATTERNS[search_login_succeeded] = "webserver@[a-zA-Z0-9\-]+:~#"
 # The accepted flags are the following: search_reached_prompt, send_login_user, search_login_succeeded, search_cmd_finished.
 # They are prefixed with either search/send, to differentiate if the pattern is meant to be sent or searched to/from the target terminal
+# TESTIMAGE_PTEST_REPORT_DIR might be used to fetch additional reports (e.g. JUnit like xml files) generated by ptests from the target device.
+# A ; separate list of remote_path:host_path is expected. The host_path is optional. It defaults to "reports".
+# For example if some ptests (such as ptest-example.bb) create additional reports in /tmp/ptest-xml/ the following line in the image recipe
+# configures the ptest imagetest to fetch the xml reports into a "xml-reports" subfolder of TEST_LOG_DIR:
+# TESTIMAGE_PTEST_REPORT_DIR = "/tmp/ptest-xml/*.xml:xml-reports"
 
 TEST_LOG_DIR ?= "${WORKDIR}/testimage"
 
diff --git a/meta/lib/oeqa/runtime/cases/ptest.py b/meta/lib/oeqa/runtime/cases/ptest.py
index 0800f3c27f..7b3560a4b0 100644
--- a/meta/lib/oeqa/runtime/cases/ptest.py
+++ b/meta/lib/oeqa/runtime/cases/ptest.py
@@ -110,3 +110,27 @@ class PtestRunnerTest(OERuntimeTestCase):
         if failmsg:
             self.logger.warning("There were failing ptests.")
             self.fail(failmsg)
+
+        # Fetch log files e.g. JUnit like xml files from the target device
+        ptest_report_dir = self.td.get('TESTIMAGE_PTEST_REPORT_DIR', '')
+        if ptest_report_dir:
+            for test_log_dir_ptest in ptest_report_dir.split(';'):
+                src_tgt = test_log_dir_ptest.split(':')
+                if len(src_tgt) == 1:
+                    tgt_dir_abs = os.path.join(ptest_log_dir, "reports")
+                elif len(src_tgt) == 2:
+                    tgt_dir_abs = os.path.join(ptest_log_dir, src_tgt[1])
+                else:
+                    self.logger.error("Invalid TESTIMAGE_PTEST_REPORT_DIR setting")
+                self.copy_logs(src_tgt[0], tgt_dir_abs)
+
+    def copy_logs(self, remoteSrc, localDst):
+        self.logger.debug("Fetching from target: %s to %s" % (remoteSrc, localDst))
+        if os.path.exists(localDst):
+            from shutil import rmtree
+            rmtree(localDst)
+        os.makedirs(localDst)
+        try:
+            self.target.copyFrom(remoteSrc, localDst)
+        except AssertionError:
+            pass
-- 
2.31.1

[PATCH v2 3/3] runtime_test.py: add new testimage ptest test case

[Adrian Freihofer]

Add a new selftest for the ptest imagetest. The new feature
TESTIMAGE_PTEST_REPORT_DIR is verified as well.
---
 meta/lib/oeqa/selftest/cases/runtime_test.py | 44 ++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/runtime_test.py b/meta/lib/oeqa/selftest/cases/runtime_test.py
index 84c2cb77e8..d85900b2fd 100644
--- a/meta/lib/oeqa/selftest/cases/runtime_test.py
+++ b/meta/lib/oeqa/selftest/cases/runtime_test.py
@@ -238,6 +238,50 @@ class TestImage(OESelftestTestCase):
         bitbake('core-image-minimal')
         bitbake('-c testimage core-image-minimal')
 
+    def test_testimage_ptest(self):
+        """
+        Summary: Verify ptest runtime test
+        Expected: 1. Verify the ptest imagetest executes the ptest-example-ptest.
+                  2. Verify the xml test report is downloaded to the ptest log files.
+        Product: oe-core
+        Author: Adrian Freihofer <adrian.freihofer@siemens.com>
+        """
+        report_subdir = ""
+
+        features = 'DISTRO_FEATURES_append = " ptest"\n'
+        features += 'IMAGE_CLASSES += "testimage"\n'
+        features += 'IMAGE_INSTALL_append = " service-ptest"\n'
+        features += 'IMAGE_FEATURES_append = " ssh-server-dropbear"\n'
+        features += 'TESTIMAGE_PTEST_REPORT_DIR = "/tmp/ptest-xml/*.xml'
+        if report_subdir:
+            features += ':' + report_subdir
+        features += '"\n'
+        features += 'TEST_SUITES = "ping ssh ptest"\n'
+        self.write_config(features)
+
+        self.append_bblayers_config('BBLAYERS_append = " ${TOPDIR}/../meta-skeleton"')
+
+        bitbake('core-image-minimal')
+        bitbake('-c testimage core-image-minimal')
+
+        vars = get_bb_vars(("TEST_LOG_DIR", "WORKDIR", "TOPDIR", "TESTIMAGE_PTEST_REPORT_DIR"), "core-image-minimal")
+
+        test_log_dir = vars["TEST_LOG_DIR"]
+        if not test_log_dir:
+            test_log_dir = os.path.join(vars["WORKDIR"], 'testimage')
+        if not os.path.isabs(test_log_dir):
+            test_log_dir = os.path.join(vars["TOPDIR"], test_log_dir)
+        ptest_log_dir_link = os.path.join(test_log_dir, 'ptest_log')
+
+        ptest_runner_log = os.path.join(ptest_log_dir_link, 'ptest-runner.log')
+        self.assertTrue(os.path.isfile(ptest_runner_log), "%s is not available." % ptest_runner_log)
+
+        if not report_subdir:
+            report_subdir = 'reports'
+        ptest_example_log = os.path.join(ptest_log_dir_link, report_subdir, 'skeleton-test.xml')
+        self.assertTrue(os.path.isfile(ptest_example_log), "%s is not available." % ptest_example_log)
+
+
 class Postinst(OESelftestTestCase):
 
     def init_manager_loop(self, init_manager):
-- 
2.31.1