* [PATCH v2 1/3] meta-skeleton: update service example
2021-06-04 15:37 [PATCH v2 0/3] testimage: additional reports for ptests Adrian Freihofer
@ 2021-06-04 15:37 ` Adrian Freihofer
2021-06-04 15:37 ` [PATCH v2 2/3] testimage: support additional reports for ptests Adrian Freihofer
2021-06-04 15:37 ` [PATCH v2 3/3] runtime_test.py: add new testimage ptest test case Adrian Freihofer
2 siblings, 0 replies; 4+ messages in thread
From: Adrian Freihofer @ 2021-06-04 15:37 UTC (permalink / raw)
To: openembedded-core; +Cc: Adrian Freihofer
- Support systemd as well
- Add a simple ptest
---
.../service/service/run-ptest | 45 +++++++++++++++++++
.../service/service/skeleton.service | 9 ++++
.../recipes-skeleton/service/service_0.1.bb | 32 ++++++++++---
3 files changed, 81 insertions(+), 5 deletions(-)
create mode 100644 meta-skeleton/recipes-skeleton/service/service/run-ptest
create mode 100644 meta-skeleton/recipes-skeleton/service/service/skeleton.service
diff --git a/meta-skeleton/recipes-skeleton/service/service/run-ptest b/meta-skeleton/recipes-skeleton/service/service/run-ptest
new file mode 100644
index 0000000000..16d172609e
--- /dev/null
+++ b/meta-skeleton/recipes-skeleton/service/service/run-ptest
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+failures=0
+
+# The result should be PASS, FAIL, or SKIP, and the testname can be any identifying string.
+result_autostart=0
+testname="skeleton autostart"
+if @@STATUS_COMMAND@@; then
+ echo "PASS: ${testname}"
+else
+ echo "FAIL: ${testname}"
+ result_autostart=1
+ failures="$(expr "$failures" + 1)"
+fi
+
+
+# Optional ----------------------
+# A ptest might provide a machine readable report such as JUnit XML.
+# Reports are collected by the ptest imagetest if the TESTIMAGE_PTEST_REPORT_DIR
+# variable is configured for the tested image.
+# Example to fetch a xml report to ${TEST_LOG_DIR}/reports-xml/skeleton-test.xml:
+# TESTIMAGE_PTEST_REPORT_DIR ?= "/tmp/ptest-xml/*.xml:reports-xml"
+# Usually one of the common unit test frameworks such as googletest or python unittest would generate a xml file.
+# Example for calling a C++ goolgetest test: my-gtest --gtest_output="xml:/tmp/ptest-xml/"
+XML_DIR="/tmp/ptest-xml"
+XML_FILE="${XML_DIR}/skeleton-test.xml"
+timestamp="$(date +"%Y-%m-%dT%H:%M:%S")"
+
+mkdir -p "$XML_DIR"
+cat << xxxEOFxxx > "$XML_FILE"
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="0" disabled="0" errors="0" time="0.010" timestamp="$timestamp" name="AllTests">
+ <testsuite name="ServiceTests" tests="1" failures="$failures" disabled="0" errors="0" time="0.010" timestamp="$timestamp">
+ <testcase name="SkeletonAutostart" status="completed" result="$result_autostart" time="0.010" timestamp="$timestamp" classname="SystemTest"/>
+ </testsuite>
+</testsuites>
+xxxEOFxxx
+# End Optional ----------------------
+
+
+# The ptest-runner evaluates the exit value of a test case: 0 means pass, 1 means fail.
+if [ "$failures" -eq 0 ]; then
+ exit 0
+fi
+exit 1
diff --git a/meta-skeleton/recipes-skeleton/service/service/skeleton.service b/meta-skeleton/recipes-skeleton/service/service/skeleton.service
new file mode 100644
index 0000000000..ee1f8c2e21
--- /dev/null
+++ b/meta-skeleton/recipes-skeleton/service/service/skeleton.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=skeleton demo service
+
+[Service]
+Type=forking
+ExecStart=@SBINDIR@/skeleton-test
+
+[Install]
+WantedBy=multi-user.target
diff --git a/meta-skeleton/recipes-skeleton/service/service_0.1.bb b/meta-skeleton/recipes-skeleton/service/service_0.1.bb
index 669d173ad1..07c8756071 100644
--- a/meta-skeleton/recipes-skeleton/service/service_0.1.bb
+++ b/meta-skeleton/recipes-skeleton/service/service_0.1.bb
@@ -4,16 +4,24 @@ DESCRIPTION = "This recipe is a canonical example of init scripts"
LICENSE = "GPLv2"
LIC_FILES_CHKSUM = "file://${WORKDIR}/COPYRIGHT;md5=349c872e0066155e1818b786938876a4"
-SRC_URI = "file://skeleton \
- file://skeleton_test.c \
- file://COPYRIGHT \
- "
+SRC_URI = "\
+ file://skeleton \
+ file://skeleton.service \
+ file://skeleton_test.c \
+ file://COPYRIGHT \
+ file://run-ptest \
+"
+
+# A recipe is "ptest-enabled" if it inherits the ptest class
+inherit update-rc.d systemd ptest
+
do_compile () {
${CC} ${CFLAGS} ${LDFLAGS} ${WORKDIR}/skeleton_test.c -o ${WORKDIR}/skeleton-test
}
do_install () {
+ # init script
install -d ${D}${sysconfdir}/init.d
cat ${WORKDIR}/skeleton | \
sed -e 's,/etc,${sysconfdir},g' \
@@ -23,10 +31,24 @@ do_install () {
-e 's,/usr,${prefix},g' > ${D}${sysconfdir}/init.d/skeleton
chmod a+x ${D}${sysconfdir}/init.d/skeleton
+ # systemd service file
+ install -d ${D}${systemd_system_unitdir}
+ install -m 0644 ${WORKDIR}/skeleton.service ${D}${systemd_system_unitdir}
+ sed -i -e 's,@SBINDIR@,${sbindir},g' ${D}${systemd_system_unitdir}/skeleton.service
+
+ # skeleton-test
install -d ${D}${sbindir}
install -m 0755 ${WORKDIR}/skeleton-test ${D}${sbindir}/
}
-RDEPENDS_${PN} = "initscripts"
+do_install_ptest() {
+ if ${@bb.utils.contains("DISTRO_FEATURES", "systemd", "true", "false", d)}; then
+ sed -i -e 's,@@STATUS_COMMAND@@,systemctl is-active --quiet skeleton,g' ${D}${PTEST_PATH}/run-ptest
+ else
+ sed -i -e 's,@@STATUS_COMMAND@@,${sysconfdir}/init.d/skeleton status,g' ${D}${PTEST_PATH}/run-ptest
+ fi
+}
CONFFILES_${PN} += "${sysconfdir}/init.d/skeleton"
+INITSCRIPT_NAME = "skeleton"
+SYSTEMD_SERVICE_${PN} = "skeleton.service"
--
2.31.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH v2 2/3] testimage: support additional reports for ptests
2021-06-04 15:37 [PATCH v2 0/3] testimage: additional reports for ptests Adrian Freihofer
2021-06-04 15:37 ` [PATCH v2 1/3] meta-skeleton: update service example Adrian Freihofer
@ 2021-06-04 15:37 ` Adrian Freihofer
2021-06-04 15:37 ` [PATCH v2 3/3] runtime_test.py: add new testimage ptest test case Adrian Freihofer
2 siblings, 0 replies; 4+ messages in thread
From: Adrian Freihofer @ 2021-06-04 15:37 UTC (permalink / raw)
To: openembedded-core; +Cc: Adrian Freihofer
This adds a new optional feature to the ptest run-time test. Most unit
test frameworks such as googletest generate JUnit like XML reports which
can be processed e.g. by GitLab CI or Jenkins.
Example: A run-ptest script executes a googletest based unit test:
/usr/bin/my-unittest --gtest_output="xml:/tmp/ptest-xml/"
The new variable TESTIMAGE_PTEST_REPORT_DIR allows to configure
bitbake -c testimage to fetch the reports from the target device and
store them into a subfolder of TEST_LOG_DIR. It's possible to fetch
report files from different locations on the target device to different
subfolders on the host.
---
meta/classes/testimage.bbclass | 5 +++++
meta/lib/oeqa/runtime/cases/ptest.py | 24 ++++++++++++++++++++++++
2 files changed, 29 insertions(+)
diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 43de9d4d76..d01892136f 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -47,6 +47,11 @@ TESTIMAGE_AUTO ??= "0"
# TESTIMAGE_BOOT_PATTERNS[search_login_succeeded] = "webserver@[a-zA-Z0-9\-]+:~#"
# The accepted flags are the following: search_reached_prompt, send_login_user, search_login_succeeded, search_cmd_finished.
# They are prefixed with either search/send, to differentiate if the pattern is meant to be sent or searched to/from the target terminal
+# TESTIMAGE_PTEST_REPORT_DIR might be used to fetch additional reports (e.g. JUnit like xml files) generated by ptests from the target device.
+# A ; separate list of remote_path:host_path is expected. The host_path is optional. It defaults to "reports".
+# For example if some ptests (such as ptest-example.bb) create additional reports in /tmp/ptest-xml/ the following line in the image recipe
+# configures the ptest imagetest to fetch the xml reports into a "xml-reports" subfolder of TEST_LOG_DIR:
+# TESTIMAGE_PTEST_REPORT_DIR = "/tmp/ptest-xml/*.xml:xml-reports"
TEST_LOG_DIR ?= "${WORKDIR}/testimage"
diff --git a/meta/lib/oeqa/runtime/cases/ptest.py b/meta/lib/oeqa/runtime/cases/ptest.py
index 0800f3c27f..7b3560a4b0 100644
--- a/meta/lib/oeqa/runtime/cases/ptest.py
+++ b/meta/lib/oeqa/runtime/cases/ptest.py
@@ -110,3 +110,27 @@ class PtestRunnerTest(OERuntimeTestCase):
if failmsg:
self.logger.warning("There were failing ptests.")
self.fail(failmsg)
+
+ # Fetch log files e.g. JUnit like xml files from the target device
+ ptest_report_dir = self.td.get('TESTIMAGE_PTEST_REPORT_DIR', '')
+ if ptest_report_dir:
+ for test_log_dir_ptest in ptest_report_dir.split(';'):
+ src_tgt = test_log_dir_ptest.split(':')
+ if len(src_tgt) == 1:
+ tgt_dir_abs = os.path.join(ptest_log_dir, "reports")
+ elif len(src_tgt) == 2:
+ tgt_dir_abs = os.path.join(ptest_log_dir, src_tgt[1])
+ else:
+ self.logger.error("Invalid TESTIMAGE_PTEST_REPORT_DIR setting")
+ self.copy_logs(src_tgt[0], tgt_dir_abs)
+
+ def copy_logs(self, remoteSrc, localDst):
+ self.logger.debug("Fetching from target: %s to %s" % (remoteSrc, localDst))
+ if os.path.exists(localDst):
+ from shutil import rmtree
+ rmtree(localDst)
+ os.makedirs(localDst)
+ try:
+ self.target.copyFrom(remoteSrc, localDst)
+ except AssertionError:
+ pass
--
2.31.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH v2 3/3] runtime_test.py: add new testimage ptest test case
2021-06-04 15:37 [PATCH v2 0/3] testimage: additional reports for ptests Adrian Freihofer
2021-06-04 15:37 ` [PATCH v2 1/3] meta-skeleton: update service example Adrian Freihofer
2021-06-04 15:37 ` [PATCH v2 2/3] testimage: support additional reports for ptests Adrian Freihofer
@ 2021-06-04 15:37 ` Adrian Freihofer
2 siblings, 0 replies; 4+ messages in thread
From: Adrian Freihofer @ 2021-06-04 15:37 UTC (permalink / raw)
To: openembedded-core; +Cc: Adrian Freihofer
Add a new selftest for the ptest imagetest. The new feature
TESTIMAGE_PTEST_REPORT_DIR is verified as well.
---
meta/lib/oeqa/selftest/cases/runtime_test.py | 44 ++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/meta/lib/oeqa/selftest/cases/runtime_test.py b/meta/lib/oeqa/selftest/cases/runtime_test.py
index 84c2cb77e8..d85900b2fd 100644
--- a/meta/lib/oeqa/selftest/cases/runtime_test.py
+++ b/meta/lib/oeqa/selftest/cases/runtime_test.py
@@ -238,6 +238,50 @@ class TestImage(OESelftestTestCase):
bitbake('core-image-minimal')
bitbake('-c testimage core-image-minimal')
+ def test_testimage_ptest(self):
+ """
+ Summary: Verify ptest runtime test
+ Expected: 1. Verify the ptest imagetest executes the ptest-example-ptest.
+ 2. Verify the xml test report is downloaded to the ptest log files.
+ Product: oe-core
+ Author: Adrian Freihofer <adrian.freihofer@siemens.com>
+ """
+ report_subdir = ""
+
+ features = 'DISTRO_FEATURES_append = " ptest"\n'
+ features += 'IMAGE_CLASSES += "testimage"\n'
+ features += 'IMAGE_INSTALL_append = " service-ptest"\n'
+ features += 'IMAGE_FEATURES_append = " ssh-server-dropbear"\n'
+ features += 'TESTIMAGE_PTEST_REPORT_DIR = "/tmp/ptest-xml/*.xml'
+ if report_subdir:
+ features += ':' + report_subdir
+ features += '"\n'
+ features += 'TEST_SUITES = "ping ssh ptest"\n'
+ self.write_config(features)
+
+ self.append_bblayers_config('BBLAYERS_append = " ${TOPDIR}/../meta-skeleton"')
+
+ bitbake('core-image-minimal')
+ bitbake('-c testimage core-image-minimal')
+
+ vars = get_bb_vars(("TEST_LOG_DIR", "WORKDIR", "TOPDIR", "TESTIMAGE_PTEST_REPORT_DIR"), "core-image-minimal")
+
+ test_log_dir = vars["TEST_LOG_DIR"]
+ if not test_log_dir:
+ test_log_dir = os.path.join(vars["WORKDIR"], 'testimage')
+ if not os.path.isabs(test_log_dir):
+ test_log_dir = os.path.join(vars["TOPDIR"], test_log_dir)
+ ptest_log_dir_link = os.path.join(test_log_dir, 'ptest_log')
+
+ ptest_runner_log = os.path.join(ptest_log_dir_link, 'ptest-runner.log')
+ self.assertTrue(os.path.isfile(ptest_runner_log), "%s is not available." % ptest_runner_log)
+
+ if not report_subdir:
+ report_subdir = 'reports'
+ ptest_example_log = os.path.join(ptest_log_dir_link, report_subdir, 'skeleton-test.xml')
+ self.assertTrue(os.path.isfile(ptest_example_log), "%s is not available." % ptest_example_log)
+
+
class Postinst(OESelftestTestCase):
def init_manager_loop(self, init_manager):
--
2.31.1
^ permalink raw reply related [flat|nested] 4+ messages in thread