[PATCH] gnutls: add initial ptest support

[PATCH] gnutls: add initial ptest support

[Ravineet Singh]

Step 1 of adding gnutls unit-tests into ptest. Only self-contained
test (no external file system dependencies) are included in this step.

Sample output:
root@qemux86-64:~# ptest-runner gnutls
START: ptest-runner
BEGIN: /usr/lib64/gnutls/ptest
PASS: aead-cipher-vec
PASS: alerts
PASS: anti_replay
...
gnutls test summary:
--------------------
total: 375
pass : 364
fail : 0
skip : 11

DURATION: 184
END: /usr/lib64/gnutls/ptest
2023-01-13T11:32
STOP: ptest-runner
TOTAL: 1 FAIL: 0

Signed-off-by: Ravineet Singh <ravineet.a.singh@est.tech>
---
 .../gnutls/gnutls/Add-ptest-support.patch     | 59 ++++++++++++
 meta/recipes-support/gnutls/gnutls/run-ptest  | 90 +++++++++++++++++++
 meta/recipes-support/gnutls/gnutls_3.7.8.bb   |  8 +-
 3 files changed, 156 insertions(+), 1 deletion(-)
 create mode 100644 meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
 create mode 100644 meta/recipes-support/gnutls/gnutls/run-ptest

diff --git a/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch b/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
new file mode 100644
index 0000000000..965568cfd6
--- /dev/null
+++ b/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
@@ -0,0 +1,59 @@
+From 74080376904713a8e6cceefbc25c81512796bf0c Mon Sep 17 00:00:00 2001
+From: Ravineet Singh <ravineet.a.singh@est.tech>
+Date: Tue, 10 Jan 2023 16:11:10 +0100
+Subject: [PATCH] gnutls: add ptest support
+
+Signed-off-by: Ravineet Singh <ravineet.a.singh@est.tech>
+---
+ Makefile.am       | 3 +++
+ configure.ac      | 2 ++
+ tests/Makefile.am | 6 ++++++
+ 3 files changed, 11 insertions(+)
+
+diff --git a/Makefile.am b/Makefile.am
+index 843193f9f..816b09fec 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -191,6 +191,9 @@ dist-hook:
+ 	mv ChangeLog $(distdir)
+ 	touch -c $(distdir)/doc/*.html $(distdir)/doc/*.pdf $(distdir)/doc/*.info
+ 
++install-ptest:
++	 $(MAKE) -C tests DESTDIR=$(DESTDIR)/tests $@
++
+ .PHONY: abi-check abi-dump-versioned abi-dump-latest pic-check symbol-check local-code-coverage-output files-update AUTHORS
+ 
+ include $(top_srcdir)/cligen/cligen.mk
+diff --git a/configure.ac b/configure.ac
+index baff1c007..f9f596abf 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -1150,6 +1150,8 @@ AC_SUBST(LIBGNUTLS_CFLAGS)
+ 
+ AM_CONDITIONAL(NEEDS_LIBRT, test "$gnutls_needs_librt" = "yes")
+ 
++AM_EXTRA_RECURSIVE_TARGETS([buildtest-TESTS])
++
+ AC_DEFINE([GNUTLS_INTERNAL_BUILD], 1, [We allow temporarily usage of deprecated functions - until they are removed.])
+ 
+ hw_features=
+diff --git a/tests/Makefile.am b/tests/Makefile.am
+index 573e911a0..e8a26f8b9 100644
+--- a/tests/Makefile.am
++++ b/tests/Makefile.am
+@@ -644,6 +644,12 @@ SH_LOG_COMPILER = $(SHELL)
+ AM_VALGRINDFLAGS = --suppressions=$(srcdir)/suppressions.valgrind
+ LOG_COMPILER = $(LOG_VALGRIND)
+ 
++install-ptest: $(check_PROGRAMS)
++	@$(INSTALL) -d $(DESTDIR)
++	@for file in $^; do \
++		$(INSTALL_PROGRAM) $$file $(DESTDIR) ; \
++	done
++
+ distclean-local:
+ 	rm -rf softhsm-*.db softhsm-*.config *.tmp tmp-* x509-crt-list-import-url.config.db port.lock.d
+ 
+-- 
+2.31.1
+
diff --git a/meta/recipes-support/gnutls/gnutls/run-ptest b/meta/recipes-support/gnutls/gnutls/run-ptest
new file mode 100644
index 0000000000..84948f487d
--- /dev/null
+++ b/meta/recipes-support/gnutls/gnutls/run-ptest
@@ -0,0 +1,90 @@
+#!/bin/sh
+
+rjob() {
+    local job=$1
+    local log=$2
+
+    # TODO: Output will be garbled
+    ./${job} >> ${log} 2>&1
+
+    ret=$?
+    case $ret in
+    0)
+        echo "PASS: $t" >> ${log}
+        echo "PASS: $t"
+        ;;
+    77)
+        echo "SKIP: $t" >> ${log}
+        echo "SKIP: $t"
+        ;;
+    *)
+        echo "FAIL: $t" >> ${log}
+        echo "FAIL: $t"
+        ;;
+    esac
+}
+
+is_disallowed() {
+    local key=$1
+    $(echo ${test_disallowlist} | grep -w -q ${key})
+    return $?
+}
+
+# TODO
+# This list should probably be in a external file
+# Testcases defined here either take very long time (dtls-stress)
+# or are dependent on local files (certs, etc) in local file system
+# currently not exported to target.
+
+test_disallowlist=""
+test_disallowlist="${test_disallowlist} dtls-stress"
+test_disallowlist="${test_disallowlist} handshake-large-cert"
+test_disallowlist="${test_disallowlist} id-on-xmppAddr"
+test_disallowlist="${test_disallowlist} mini-x509-cas"
+test_disallowlist="${test_disallowlist} pkcs12_simple"
+test_disallowlist="${test_disallowlist} protocol-set-allowlist"
+test_disallowlist="${test_disallowlist} psk-file"
+test_disallowlist="${test_disallowlist} rawpk-api"
+test_disallowlist="${test_disallowlist} set_pkcs12_cred"
+test_disallowlist="${test_disallowlist} system-override-curves-allowlist"
+test_disallowlist="${test_disallowlist} system-override-hash"
+test_disallowlist="${test_disallowlist} system-override-sig"
+test_disallowlist="${test_disallowlist} system-override-sig-tls"
+test_disallowlist="${test_disallowlist} system-prio-file"
+test_disallowlist="${test_disallowlist} x509cert-tl"
+
+LOG=${PWD}/tests.log
+cd tests
+max_njobs=$(grep -c ^processor /proc/cpuinfo)
+njobs=0
+
+for t in *; do
+    [ -x $t ] || continue
+    [ -f $t ] || continue
+
+    is_disallowed ${t}
+    [ $? -eq 0 ] && continue
+
+    rjob ${t} ${LOG} &
+    one=1
+    njobs=$(expr ${njobs} + ${one})
+    if [ ${njobs} -eq ${max_njobs} ]; then
+        wait
+        njobs=0
+    fi
+done
+wait
+
+skipped=$(grep -c SKIP ${LOG})
+passed=$(grep -c PASS ${LOG})
+failed=$(grep -c FAIL ${LOG})
+total=$(expr ${passed} + ${failed} + ${skipped})
+
+echo
+echo "gnutls test summary:"
+echo "--------------------"
+echo "total: ${total}"
+echo "pass : ${passed}"
+echo "fail : ${failed}"
+echo "skip : ${skipped}"
+echo
diff --git a/meta/recipes-support/gnutls/gnutls_3.7.8.bb b/meta/recipes-support/gnutls/gnutls_3.7.8.bb
index 8f979a5b99..fee06393bd 100644
--- a/meta/recipes-support/gnutls/gnutls_3.7.8.bb
+++ b/meta/recipes-support/gnutls/gnutls_3.7.8.bb
@@ -22,11 +22,13 @@ SHRT_VER = "${@d.getVar('PV').split('.')[0]}.${@d.getVar('PV').split('.')[1]}"
 SRC_URI = "https://www.gnupg.org/ftp/gcrypt/gnutls/v${SHRT_VER}/gnutls-${PV}.tar.xz \
            file://arm_eabi.patch \
            file://0001-Creating-.hmac-file-should-be-excuted-in-target-envi.patch \
+           file://run-ptest \
+           file://Add-ptest-support.patch \
            "
 
 SRC_URI[sha256sum] = "c58ad39af0670efe6a8aee5e3a8b2331a1200418b64b7c51977fb396d4617114"
 
-inherit autotools texinfo pkgconfig gettext lib_package gtk-doc
+inherit autotools texinfo pkgconfig gettext lib_package gtk-doc ptest
 
 PACKAGECONFIG ??= "libidn  ${@bb.utils.filter('DISTRO_FEATURES', 'seccomp', d)}"
 
@@ -68,6 +70,10 @@ do_install:append:class-target() {
         fi
 }
 
+do_compile:append() {
+        oe_runmake ${PARALLEL_MAKE} -C tests buildtest-TESTS
+}
+
 PACKAGES =+ "${PN}-openssl ${PN}-xx ${PN}-fips"
 
 FILES:${PN}-dev += "${bindir}/gnutls-cli-debug"
-- 
2.34.1

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Alexander Kanavin]

Thanks! Before this can be added I'd like to clarify two things:
- is execution time of 184 with or without kvm?
- depending on that, you need to add the test to fast or slow list in
ptest-packagelists.inc, so it's actually executed in yocto CI.

Alex

On Wed, 25 Jan 2023 at 14:05, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
>
> Step 1 of adding gnutls unit-tests into ptest. Only self-contained
> test (no external file system dependencies) are included in this step.
>
> Sample output:
> root@qemux86-64:~# ptest-runner gnutls
> START: ptest-runner
> BEGIN: /usr/lib64/gnutls/ptest
> PASS: aead-cipher-vec
> PASS: alerts
> PASS: anti_replay
> ...
> gnutls test summary:
> --------------------
> total: 375
> pass : 364
> fail : 0
> skip : 11
>
> DURATION: 184
> END: /usr/lib64/gnutls/ptest
> 2023-01-13T11:32
> STOP: ptest-runner
> TOTAL: 1 FAIL: 0
>
> Signed-off-by: Ravineet Singh <ravineet.a.singh@est.tech>
> ---
>  .../gnutls/gnutls/Add-ptest-support.patch     | 59 ++++++++++++
>  meta/recipes-support/gnutls/gnutls/run-ptest  | 90 +++++++++++++++++++
>  meta/recipes-support/gnutls/gnutls_3.7.8.bb   |  8 +-
>  3 files changed, 156 insertions(+), 1 deletion(-)
>  create mode 100644 meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
>  create mode 100644 meta/recipes-support/gnutls/gnutls/run-ptest
>
> diff --git a/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch b/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
> new file mode 100644
> index 0000000000..965568cfd6
> --- /dev/null
> +++ b/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
> @@ -0,0 +1,59 @@
> +From 74080376904713a8e6cceefbc25c81512796bf0c Mon Sep 17 00:00:00 2001
> +From: Ravineet Singh <ravineet.a.singh@est.tech>
> +Date: Tue, 10 Jan 2023 16:11:10 +0100
> +Subject: [PATCH] gnutls: add ptest support
> +
> +Signed-off-by: Ravineet Singh <ravineet.a.singh@est.tech>
> +---
> + Makefile.am       | 3 +++
> + configure.ac      | 2 ++
> + tests/Makefile.am | 6 ++++++
> + 3 files changed, 11 insertions(+)
> +
> +diff --git a/Makefile.am b/Makefile.am
> +index 843193f9f..816b09fec 100644
> +--- a/Makefile.am
> ++++ b/Makefile.am
> +@@ -191,6 +191,9 @@ dist-hook:
> +       mv ChangeLog $(distdir)
> +       touch -c $(distdir)/doc/*.html $(distdir)/doc/*.pdf $(distdir)/doc/*.info
> +
> ++install-ptest:
> ++       $(MAKE) -C tests DESTDIR=$(DESTDIR)/tests $@
> ++
> + .PHONY: abi-check abi-dump-versioned abi-dump-latest pic-check symbol-check local-code-coverage-output files-update AUTHORS
> +
> + include $(top_srcdir)/cligen/cligen.mk
> +diff --git a/configure.ac b/configure.ac
> +index baff1c007..f9f596abf 100644
> +--- a/configure.ac
> ++++ b/configure.ac
> +@@ -1150,6 +1150,8 @@ AC_SUBST(LIBGNUTLS_CFLAGS)
> +
> + AM_CONDITIONAL(NEEDS_LIBRT, test "$gnutls_needs_librt" = "yes")
> +
> ++AM_EXTRA_RECURSIVE_TARGETS([buildtest-TESTS])
> ++
> + AC_DEFINE([GNUTLS_INTERNAL_BUILD], 1, [We allow temporarily usage of deprecated functions - until they are removed.])
> +
> + hw_features=
> +diff --git a/tests/Makefile.am b/tests/Makefile.am
> +index 573e911a0..e8a26f8b9 100644
> +--- a/tests/Makefile.am
> ++++ b/tests/Makefile.am
> +@@ -644,6 +644,12 @@ SH_LOG_COMPILER = $(SHELL)
> + AM_VALGRINDFLAGS = --suppressions=$(srcdir)/suppressions.valgrind
> + LOG_COMPILER = $(LOG_VALGRIND)
> +
> ++install-ptest: $(check_PROGRAMS)
> ++      @$(INSTALL) -d $(DESTDIR)
> ++      @for file in $^; do \
> ++              $(INSTALL_PROGRAM) $$file $(DESTDIR) ; \
> ++      done
> ++
> + distclean-local:
> +       rm -rf softhsm-*.db softhsm-*.config *.tmp tmp-* x509-crt-list-import-url.config.db port.lock.d
> +
> +--
> +2.31.1
> +
> diff --git a/meta/recipes-support/gnutls/gnutls/run-ptest b/meta/recipes-support/gnutls/gnutls/run-ptest
> new file mode 100644
> index 0000000000..84948f487d
> --- /dev/null
> +++ b/meta/recipes-support/gnutls/gnutls/run-ptest
> @@ -0,0 +1,90 @@
> +#!/bin/sh
> +
> +rjob() {
> +    local job=$1
> +    local log=$2
> +
> +    # TODO: Output will be garbled
> +    ./${job} >> ${log} 2>&1
> +
> +    ret=$?
> +    case $ret in
> +    0)
> +        echo "PASS: $t" >> ${log}
> +        echo "PASS: $t"
> +        ;;
> +    77)
> +        echo "SKIP: $t" >> ${log}
> +        echo "SKIP: $t"
> +        ;;
> +    *)
> +        echo "FAIL: $t" >> ${log}
> +        echo "FAIL: $t"
> +        ;;
> +    esac
> +}
> +
> +is_disallowed() {
> +    local key=$1
> +    $(echo ${test_disallowlist} | grep -w -q ${key})
> +    return $?
> +}
> +
> +# TODO
> +# This list should probably be in a external file
> +# Testcases defined here either take very long time (dtls-stress)
> +# or are dependent on local files (certs, etc) in local file system
> +# currently not exported to target.
> +
> +test_disallowlist=""
> +test_disallowlist="${test_disallowlist} dtls-stress"
> +test_disallowlist="${test_disallowlist} handshake-large-cert"
> +test_disallowlist="${test_disallowlist} id-on-xmppAddr"
> +test_disallowlist="${test_disallowlist} mini-x509-cas"
> +test_disallowlist="${test_disallowlist} pkcs12_simple"
> +test_disallowlist="${test_disallowlist} protocol-set-allowlist"
> +test_disallowlist="${test_disallowlist} psk-file"
> +test_disallowlist="${test_disallowlist} rawpk-api"
> +test_disallowlist="${test_disallowlist} set_pkcs12_cred"
> +test_disallowlist="${test_disallowlist} system-override-curves-allowlist"
> +test_disallowlist="${test_disallowlist} system-override-hash"
> +test_disallowlist="${test_disallowlist} system-override-sig"
> +test_disallowlist="${test_disallowlist} system-override-sig-tls"
> +test_disallowlist="${test_disallowlist} system-prio-file"
> +test_disallowlist="${test_disallowlist} x509cert-tl"
> +
> +LOG=${PWD}/tests.log
> +cd tests
> +max_njobs=$(grep -c ^processor /proc/cpuinfo)
> +njobs=0
> +
> +for t in *; do
> +    [ -x $t ] || continue
> +    [ -f $t ] || continue
> +
> +    is_disallowed ${t}
> +    [ $? -eq 0 ] && continue
> +
> +    rjob ${t} ${LOG} &
> +    one=1
> +    njobs=$(expr ${njobs} + ${one})
> +    if [ ${njobs} -eq ${max_njobs} ]; then
> +        wait
> +        njobs=0
> +    fi
> +done
> +wait
> +
> +skipped=$(grep -c SKIP ${LOG})
> +passed=$(grep -c PASS ${LOG})
> +failed=$(grep -c FAIL ${LOG})
> +total=$(expr ${passed} + ${failed} + ${skipped})
> +
> +echo
> +echo "gnutls test summary:"
> +echo "--------------------"
> +echo "total: ${total}"
> +echo "pass : ${passed}"
> +echo "fail : ${failed}"
> +echo "skip : ${skipped}"
> +echo
> diff --git a/meta/recipes-support/gnutls/gnutls_3.7.8.bb b/meta/recipes-support/gnutls/gnutls_3.7.8.bb
> index 8f979a5b99..fee06393bd 100644
> --- a/meta/recipes-support/gnutls/gnutls_3.7.8.bb
> +++ b/meta/recipes-support/gnutls/gnutls_3.7.8.bb
> @@ -22,11 +22,13 @@ SHRT_VER = "${@d.getVar('PV').split('.')[0]}.${@d.getVar('PV').split('.')[1]}"
>  SRC_URI = "https://www.gnupg.org/ftp/gcrypt/gnutls/v${SHRT_VER}/gnutls-${PV}.tar.xz \
>             file://arm_eabi.patch \
>             file://0001-Creating-.hmac-file-should-be-excuted-in-target-envi.patch \
> +           file://run-ptest \
> +           file://Add-ptest-support.patch \
>             "
>
>  SRC_URI[sha256sum] = "c58ad39af0670efe6a8aee5e3a8b2331a1200418b64b7c51977fb396d4617114"
>
> -inherit autotools texinfo pkgconfig gettext lib_package gtk-doc
> +inherit autotools texinfo pkgconfig gettext lib_package gtk-doc ptest
>
>  PACKAGECONFIG ??= "libidn  ${@bb.utils.filter('DISTRO_FEATURES', 'seccomp', d)}"
>
> @@ -68,6 +70,10 @@ do_install:append:class-target() {
>          fi
>  }
>
> +do_compile:append() {
> +        oe_runmake ${PARALLEL_MAKE} -C tests buildtest-TESTS
> +}
> +
>  PACKAGES =+ "${PN}-openssl ${PN}-xx ${PN}-fips"
>
>  FILES:${PN}-dev += "${bindir}/gnutls-cli-debug"
> --
> 2.34.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#176347): https://lists.openembedded.org/g/openembedded-core/message/176347
> Mute This Topic: https://lists.openembedded.org/mt/96519581/1686489
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

On Wed, Jan 25, 2023 at 03:02:27PM +0100, Alexander Kanavin wrote:
> Thanks! Before this can be added I'd like to clarify two things:
> - is execution time of 184 with or without kvm?
It is with kvm.
> - depending on that, you need to add the test to fast or slow list in
> ptest-packagelists.inc, so it's actually executed in yocto CI.
I'll add it in the slow list in ptest-packagelists.inc, I assume that is OK.

Ravineet
>
> Alex
>
> On Wed, 25 Jan 2023 at 14:05, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> >
> > Step 1 of adding gnutls unit-tests into ptest. Only self-contained
> > test (no external file system dependencies) are included in this step.
> >
> > Sample output:
> > root@qemux86-64:~# ptest-runner gnutls
> > START: ptest-runner
> > BEGIN: /usr/lib64/gnutls/ptest
> > PASS: aead-cipher-vec
> > PASS: alerts
> > PASS: anti_replay
> > ...
> > gnutls test summary:
> > --------------------
> > total: 375
> > pass : 364
> > fail : 0
> > skip : 11
> >
> > DURATION: 184
> > END: /usr/lib64/gnutls/ptest
> > 2023-01-13T11:32
> > STOP: ptest-runner
> > TOTAL: 1 FAIL: 0
> >
> > Signed-off-by: Ravineet Singh <ravineet.a.singh@est.tech>
> > ---
> >  .../gnutls/gnutls/Add-ptest-support.patch     | 59 ++++++++++++
> >  meta/recipes-support/gnutls/gnutls/run-ptest  | 90 +++++++++++++++++++
> >  meta/recipes-support/gnutls/gnutls_3.7.8.bb   |  8 +-
> >  3 files changed, 156 insertions(+), 1 deletion(-)
> >  create mode 100644 meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
> >  create mode 100644 meta/recipes-support/gnutls/gnutls/run-ptest
> >
> > diff --git a/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch b/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
> > new file mode 100644
> > index 0000000000..965568cfd6
> > --- /dev/null
> > +++ b/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
> > @@ -0,0 +1,59 @@
> > +From 74080376904713a8e6cceefbc25c81512796bf0c Mon Sep 17 00:00:00 2001
> > +From: Ravineet Singh <ravineet.a.singh@est.tech>
> > +Date: Tue, 10 Jan 2023 16:11:10 +0100
> > +Subject: [PATCH] gnutls: add ptest support
> > +
> > +Signed-off-by: Ravineet Singh <ravineet.a.singh@est.tech>
> > +---
> > + Makefile.am       | 3 +++
> > + configure.ac      | 2 ++
> > + tests/Makefile.am | 6 ++++++
> > + 3 files changed, 11 insertions(+)
> > +
> > +diff --git a/Makefile.am b/Makefile.am
> > +index 843193f9f..816b09fec 100644
> > +--- a/Makefile.am
> > ++++ b/Makefile.am
> > +@@ -191,6 +191,9 @@ dist-hook:
> > +       mv ChangeLog $(distdir)
> > +       touch -c $(distdir)/doc/*.html $(distdir)/doc/*.pdf $(distdir)/doc/*.info
> > +
> > ++install-ptest:
> > ++       $(MAKE) -C tests DESTDIR=$(DESTDIR)/tests $@
> > ++
> > + .PHONY: abi-check abi-dump-versioned abi-dump-latest pic-check symbol-check local-code-coverage-output files-update AUTHORS
> > +
> > + include $(top_srcdir)/cligen/cligen.mk
> > +diff --git a/configure.ac b/configure.ac
> > +index baff1c007..f9f596abf 100644
> > +--- a/configure.ac
> > ++++ b/configure.ac
> > +@@ -1150,6 +1150,8 @@ AC_SUBST(LIBGNUTLS_CFLAGS)
> > +
> > + AM_CONDITIONAL(NEEDS_LIBRT, test "$gnutls_needs_librt" = "yes")
> > +
> > ++AM_EXTRA_RECURSIVE_TARGETS([buildtest-TESTS])
> > ++
> > + AC_DEFINE([GNUTLS_INTERNAL_BUILD], 1, [We allow temporarily usage of deprecated functions - until they are removed.])
> > +
> > + hw_features=
> > +diff --git a/tests/Makefile.am b/tests/Makefile.am
> > +index 573e911a0..e8a26f8b9 100644
> > +--- a/tests/Makefile.am
> > ++++ b/tests/Makefile.am
> > +@@ -644,6 +644,12 @@ SH_LOG_COMPILER = $(SHELL)
> > + AM_VALGRINDFLAGS = --suppressions=$(srcdir)/suppressions.valgrind
> > + LOG_COMPILER = $(LOG_VALGRIND)
> > +
> > ++install-ptest: $(check_PROGRAMS)
> > ++      @$(INSTALL) -d $(DESTDIR)
> > ++      @for file in $^; do \
> > ++              $(INSTALL_PROGRAM) $$file $(DESTDIR) ; \
> > ++      done
> > ++
> > + distclean-local:
> > +       rm -rf softhsm-*.db softhsm-*.config *.tmp tmp-* x509-crt-list-import-url.config.db port.lock.d
> > +
> > +--
> > +2.31.1
> > +
> > diff --git a/meta/recipes-support/gnutls/gnutls/run-ptest b/meta/recipes-support/gnutls/gnutls/run-ptest
> > new file mode 100644
> > index 0000000000..84948f487d
> > --- /dev/null
> > +++ b/meta/recipes-support/gnutls/gnutls/run-ptest
> > @@ -0,0 +1,90 @@
> > +#!/bin/sh
> > +
> > +rjob() {
> > +    local job=$1
> > +    local log=$2
> > +
> > +    # TODO: Output will be garbled
> > +    ./${job} >> ${log} 2>&1
> > +
> > +    ret=$?
> > +    case $ret in
> > +    0)
> > +        echo "PASS: $t" >> ${log}
> > +        echo "PASS: $t"
> > +        ;;
> > +    77)
> > +        echo "SKIP: $t" >> ${log}
> > +        echo "SKIP: $t"
> > +        ;;
> > +    *)
> > +        echo "FAIL: $t" >> ${log}
> > +        echo "FAIL: $t"
> > +        ;;
> > +    esac
> > +}
> > +
> > +is_disallowed() {
> > +    local key=$1
> > +    $(echo ${test_disallowlist} | grep -w -q ${key})
> > +    return $?
> > +}
> > +
> > +# TODO
> > +# This list should probably be in a external file
> > +# Testcases defined here either take very long time (dtls-stress)
> > +# or are dependent on local files (certs, etc) in local file system
> > +# currently not exported to target.
> > +
> > +test_disallowlist=""
> > +test_disallowlist="${test_disallowlist} dtls-stress"
> > +test_disallowlist="${test_disallowlist} handshake-large-cert"
> > +test_disallowlist="${test_disallowlist} id-on-xmppAddr"
> > +test_disallowlist="${test_disallowlist} mini-x509-cas"
> > +test_disallowlist="${test_disallowlist} pkcs12_simple"
> > +test_disallowlist="${test_disallowlist} protocol-set-allowlist"
> > +test_disallowlist="${test_disallowlist} psk-file"
> > +test_disallowlist="${test_disallowlist} rawpk-api"
> > +test_disallowlist="${test_disallowlist} set_pkcs12_cred"
> > +test_disallowlist="${test_disallowlist} system-override-curves-allowlist"
> > +test_disallowlist="${test_disallowlist} system-override-hash"
> > +test_disallowlist="${test_disallowlist} system-override-sig"
> > +test_disallowlist="${test_disallowlist} system-override-sig-tls"
> > +test_disallowlist="${test_disallowlist} system-prio-file"
> > +test_disallowlist="${test_disallowlist} x509cert-tl"
> > +
> > +LOG=${PWD}/tests.log
> > +cd tests
> > +max_njobs=$(grep -c ^processor /proc/cpuinfo)
> > +njobs=0
> > +
> > +for t in *; do
> > +    [ -x $t ] || continue
> > +    [ -f $t ] || continue
> > +
> > +    is_disallowed ${t}
> > +    [ $? -eq 0 ] && continue
> > +
> > +    rjob ${t} ${LOG} &
> > +    one=1
> > +    njobs=$(expr ${njobs} + ${one})
> > +    if [ ${njobs} -eq ${max_njobs} ]; then
> > +        wait
> > +        njobs=0
> > +    fi
> > +done
> > +wait
> > +
> > +skipped=$(grep -c SKIP ${LOG})
> > +passed=$(grep -c PASS ${LOG})
> > +failed=$(grep -c FAIL ${LOG})
> > +total=$(expr ${passed} + ${failed} + ${skipped})
> > +
> > +echo
> > +echo "gnutls test summary:"
> > +echo "--------------------"
> > +echo "total: ${total}"
> > +echo "pass : ${passed}"
> > +echo "fail : ${failed}"
> > +echo "skip : ${skipped}"
> > +echo
> > diff --git a/meta/recipes-support/gnutls/gnutls_3.7.8.bb b/meta/recipes-support/gnutls/gnutls_3.7.8.bb
> > index 8f979a5b99..fee06393bd 100644
> > --- a/meta/recipes-support/gnutls/gnutls_3.7.8.bb
> > +++ b/meta/recipes-support/gnutls/gnutls_3.7.8.bb
> > @@ -22,11 +22,13 @@ SHRT_VER = "${@d.getVar('PV').split('.')[0]}.${@d.getVar('PV').split('.')[1]}"
> >  SRC_URI = "https://www.gnupg.org/ftp/gcrypt/gnutls/v${SHRT_VER}/gnutls-${PV}.tar.xz \
> >             file://arm_eabi.patch \
> >             file://0001-Creating-.hmac-file-should-be-excuted-in-target-envi.patch \
> > +           file://run-ptest \
> > +           file://Add-ptest-support.patch \
> >             "
> >
> >  SRC_URI[sha256sum] = "c58ad39af0670efe6a8aee5e3a8b2331a1200418b64b7c51977fb396d4617114"
> >
> > -inherit autotools texinfo pkgconfig gettext lib_package gtk-doc
> > +inherit autotools texinfo pkgconfig gettext lib_package gtk-doc ptest
> >
> >  PACKAGECONFIG ??= "libidn  ${@bb.utils.filter('DISTRO_FEATURES', 'seccomp', d)}"
> >
> > @@ -68,6 +70,10 @@ do_install:append:class-target() {
> >          fi
> >  }
> >
> > +do_compile:append() {
> > +        oe_runmake ${PARALLEL_MAKE} -C tests buildtest-TESTS
> > +}
> > +
> >  PACKAGES =+ "${PN}-openssl ${PN}-xx ${PN}-fips"
> >
> >  FILES:${PN}-dev += "${bindir}/gnutls-cli-debug"
> > --
> > 2.34.1
> >
> >
> > -=-=-=-=-=-=-=-=-=-=-=-
> > Links: You receive all messages sent to this group.
> > View/Reply Online (#176347): https://lists.openembedded.org/g/openembedded-core/message/176347
> > Mute This Topic: https://lists.openembedded.org/mt/96519581/1686489
> > Group Owner: openembedded-core+owner@lists.openembedded.org
> > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> > -=-=-=-=-=-=-=-=-=-=-=-
> >

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Alexander Kanavin]

On Wed, 25 Jan 2023 at 15:20, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
>
> On Wed, Jan 25, 2023 at 03:02:27PM +0100, Alexander Kanavin wrote:
> > Thanks! Before this can be added I'd like to clarify two things:
> > - is execution time of 184 with or without kvm?
> It is with kvm.
> > - depending on that, you need to add the test to fast or slow list in
> > ptest-packagelists.inc, so it's actually executed in yocto CI.
> I'll add it in the slow list in ptest-packagelists.inc, I assume that is OK.

Also the newly added patch is missing Upstream-Status. Aren't you
getting a build failure with that?

Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

On Wed, Jan 25, 2023 at 10:02:22PM +0100, Alexander Kanavin wrote:
> On Wed, 25 Jan 2023 at 15:20, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> >
> > On Wed, Jan 25, 2023 at 03:02:27PM +0100, Alexander Kanavin wrote:
> > > Thanks! Before this can be added I'd like to clarify two things:
> > > - is execution time of 184 with or without kvm?
> > It is with kvm.
> > > - depending on that, you need to add the test to fast or slow list in
> > > ptest-packagelists.inc, so it's actually executed in yocto CI.
> > I'll add it in the slow list in ptest-packagelists.inc, I assume that is OK.
>
> Also the newly added patch is missing Upstream-Status. Aren't you
> getting a build failure with that?
Sorry, I am new to openembedded/yocto hence I do not know where to find the build output/status. Could you point me to it?
I will add 'Upstream-Status: Pending'.
/Ravineet


>
> Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Alexander Kanavin]

On Thu, 26 Jan 2023 at 09:03, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > Also the newly added patch is missing Upstream-Status. Aren't you
> > getting a build failure with that?
> Sorry, I am new to openembedded/yocto hence I do not know where to find the build output/status. Could you point me to it?

I applied your patch, and run 'bitbake gnutls'. The following error occurs:
ERROR: gnutls-native-3.7.8-r0 do_patch: QA Issue: Missing
Upstream-Status in patch
/srv/work/alex/poky/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
Please add according to
https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines#Patch_Header_Recommendations:_Upstream-Status
. [patch-status-core]
ERROR: gnutls-native-3.7.8-r0 do_patch: Fatal QA errors were found,
failing task.
ERROR: Logfile of failure stored in:
/srv/storage/alex/yocto/build-64-alt/tmp/work/x86_64-linux/gnutls-native/3.7.8-r0/temp/log.do_patch.114509
ERROR: Task (virtual:native:/srv/work/alex/poky/meta/recipes-support/gnutls/gnutls_3.7.8.bb:do_patch)
failed with exit code '1'

How did you test your work and get to the point where you can run the
ptest? Are you getting the same error?

> I will add 'Upstream-Status: Pending'.

Is there a reason to use Pending? If yes, you should say what it is,
otherwise please submit the patch upstream.

Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

My bad, will fix and resend the patch.
Thanks Alex!

/RAvineet

On Thu, Jan 26, 2023 at 09:18:06AM +0100, Alexander Kanavin wrote:
> On Thu, 26 Jan 2023 at 09:03, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > > Also the newly added patch is missing Upstream-Status. Aren't you
> > > getting a build failure with that?
> > Sorry, I am new to openembedded/yocto hence I do not know where to find the build output/status. Could you point me to it?
>
> I applied your patch, and run 'bitbake gnutls'. The following error occurs:
> ERROR: gnutls-native-3.7.8-r0 do_patch: QA Issue: Missing
> Upstream-Status in patch
> /srv/work/alex/poky/meta/recipes-support/gnutls/gnutls/Add-ptest-support.patch
> Please add according to
> https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines#Patch_Header_Recommendations:_Upstream-Status
> . [patch-status-core]
> ERROR: gnutls-native-3.7.8-r0 do_patch: Fatal QA errors were found,
> failing task.
> ERROR: Logfile of failure stored in:
> /srv/storage/alex/yocto/build-64-alt/tmp/work/x86_64-linux/gnutls-native/3.7.8-r0/temp/log.do_patch.114509
> ERROR: Task (virtual:native:/srv/work/alex/poky/meta/recipes-support/gnutls/gnutls_3.7.8.bb:do_patch)
> failed with exit code '1'
>
> How did you test your work and get to the point where you can run the
> ptest? Are you getting the same error?
>
> > I will add 'Upstream-Status: Pending'.
>
> Is there a reason to use Pending? If yes, you should say what it is,
> otherwise please submit the patch upstream.
>
> Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Alexander Kanavin]

On Thu, 26 Jan 2023 at 13:07, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> I believe the patch cannot be
> upstreamed into gnutls since gnutls use automake and use 'make check' to verify the same locally.
> This patch enables running gnutls tests under ptest, hence it would tag the patch as:
> Upstream-Status: Inappropriate [embedded specific]
>
> What do you think?

That's ok. The real issue is that libcheck does not support
installable tests, and it should be solved there rather than patched
into specific upstreams that use it.

Can you also try to determine what are the specific missing items
needed by entries in test_disallowlist and see if they can be easily
provided on target via adding to RDEPENDS:${PN}-ptest?

Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

On Thu, Jan 26, 2023 at 03:24:29PM +0100, Alexander Kanavin wrote:
> On Thu, 26 Jan 2023 at 13:07, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > I believe the patch cannot be
> > upstreamed into gnutls since gnutls use automake and use 'make check' to verify the same locally.
> > This patch enables running gnutls tests under ptest, hence it would tag the patch as:
> > Upstream-Status: Inappropriate [embedded specific]
> >
> > What do you think?
>
> That's ok. The real issue is that libcheck does not support
> installable tests, and it should be solved there rather than patched
> into specific upstreams that use it.
>
> Can you also try to determine what are the specific missing items
> needed by entries in test_disallowlist and see if they can be easily
> provided on target via adding to RDEPENDS:${PN}-ptest?
>
Well, these are not package dependencies, rather local file dependencies (pem files, etc)
 Ravineet
> Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Alexander Kanavin]

On Mon, 30 Jan 2023 at 09:25, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > Can you also try to determine what are the specific missing items
> > needed by entries in test_disallowlist and see if they can be easily
> > provided on target via adding to RDEPENDS:${PN}-ptest?
> >
> Well, these are not package dependencies, rather local file dependencies (pem files, etc)

Those local files are provided by something, so you should find out
what that something is. Is it gnutls source tree itself? Or some other
component? Do we have a recipe for it?

Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

On Mon, Jan 30, 2023 at 09:31:27AM +0100, Alexander Kanavin wrote:
> On Mon, 30 Jan 2023 at 09:25, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > > Can you also try to determine what are the specific missing items
> > > needed by entries in test_disallowlist and see if they can be easily
> > > provided on target via adding to RDEPENDS:${PN}-ptest?
> > >
> > Well, these are not package dependencies, rather local file dependencies (pem files, etc)
>
> Those local files are provided by something, so you should find out
> what that something is. Is it gnutls source tree itself? Or some other
> component? Do we have a recipe for it?
Yes, these are from gnutls source tree itself.
I did not want to have the logic to install these files in this patch (want to keep it short),
rather later as I progress in my learning of bitbake magic.
 Ravineet
>
> Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Alexander Kanavin]

On Mon, 30 Jan 2023 at 09:49, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > Those local files are provided by something, so you should find out
> > what that something is. Is it gnutls source tree itself? Or some other
> > component? Do we have a recipe for it?
> Yes, these are from gnutls source tree itself.
> I did not want to have the logic to install these files in this patch (want to keep it short),
> rather later as I progress in my learning of bitbake magic.

That's okay, but making the test set as complete as possible would be
appreciated. You have the knowledge and context for it in your head
right now, and it's going to be harder for someone else, or even for
you later on.

Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

On Mon, Jan 30, 2023 at 10:08:13AM +0100, Alexander Kanavin wrote:
> On Mon, 30 Jan 2023 at 09:49, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > > Those local files are provided by something, so you should find out
> > > what that something is. Is it gnutls source tree itself? Or some other
> > > component? Do we have a recipe for it?
> > Yes, these are from gnutls source tree itself.
> > I did not want to have the logic to install these files in this patch (want to keep it short),
> > rather later as I progress in my learning of bitbake magic.
>
> That's okay, but making the test set as complete as possible would be
> appreciated. You have the knowledge and context for it in your head
> right now, and it's going to be harder for someone else, or even for
> you later on.
makes sense, I'll put some effort into it and send upgraded patch.
Thanks Alex!
 Ravineet
>
> Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

On Mon, Jan 30, 2023 at 10:32:53AM +0100, Ravineet Singh wrote:
> On Mon, Jan 30, 2023 at 10:08:13AM +0100, Alexander Kanavin wrote:
> > On Mon, 30 Jan 2023 at 09:49, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > > > Those local files are provided by something, so you should find out
> > > > what that something is. Is it gnutls source tree itself? Or some other
> > > > component? Do we have a recipe for it?
> > > Yes, these are from gnutls source tree itself.
> > > I did not want to have the logic to install these files in this patch (want to keep it short),
> > > rather later as I progress in my learning of bitbake magic.
> >
> > That's okay, but making the test set as complete as possible would be
> > appreciated. You have the knowledge and context for it in your head
> > right now, and it's going to be harder for someone else, or even for
> > you later on.
> makes sense, I'll put some effort into it and send upgraded patch.
> Thanks Alex!
>  Ravineet
So, tried to add missing files via autotools-brokensep (not recommendded), but still some tests will not
be possible to run since they are expected to be run via wrapper shell scripts. These are the test:
 - system-override-curves-allowlist"
 - system-override-hash"
 - system-override-sig"
 - system-override-sig-tls"
 - system-prio-file"
Giving up at this stage, don't really see a good way forward.

I will post initial patch updated with comments you had before, in case someone else wants to look at it.
I do understand that you may not want to integrate the patch.
/Ravineet

> >
> > Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Alexander Kanavin]

On Tue, 31 Jan 2023 at 18:58, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> So, tried to add missing files via autotools-brokensep (not recommendded), but still some tests will not
> be possible to run since they are expected to be run via wrapper shell scripts. These are the test:
>  - system-override-curves-allowlist"
>  - system-override-hash"
>  - system-override-sig"
>  - system-override-sig-tls"
>  - system-prio-file"
> Giving up at this stage, don't really see a good way forward.
>
> I will post initial patch updated with comments you had before, in case someone else wants to look at it.
> I do understand that you may not want to integrate the patch.

The patch is fine, I only wanted to find out if additional tests can
be added with reasonable effort. If you say that the effort is not
reasonable compared to the outcome, then I trust that, and don't have
any objections to the existing patch going in. Just try to document
your findings (via comments in code or commit message) so that someone
can use them as a starting point later on.

Alex

Re: [OE-core] [PATCH] gnutls: add initial ptest support

[Ravineet Singh]

On Wed, Feb 01, 2023 at 10:10:56AM +0100, Alexander Kanavin wrote:
> On Tue, 31 Jan 2023 at 18:58, Ravineet Singh <ravineet.a.singh@est.tech> wrote:
> > So, tried to add missing files via autotools-brokensep (not recommendded), but still some tests will not
> > be possible to run since they are expected to be run via wrapper shell scripts. These are the test:
> >  - system-override-curves-allowlist"
> >  - system-override-hash"
> >  - system-override-sig"
> >  - system-override-sig-tls"
> >  - system-prio-file"
> > Giving up at this stage, don't really see a good way forward.
> >
> > I will post initial patch updated with comments you had before, in case someone else wants to look at it.
> > I do understand that you may not want to integrate the patch.
>
> The patch is fine, I only wanted to find out if additional tests can
> be added with reasonable effort. If you say that the effort is not
> reasonable compared to the outcome, then I trust that, and don't have
> any objections to the existing patch going in. Just try to document
> your findings (via comments in code or commit message) so that someone
> can use them as a starting point later on.
Will do, thanks!
/Ravineet
>
> Alex