[Fuego] [PATCH] nss: add test case for nss.

[Fuego] [PATCH] nss: add test case for nss.

[Zheng Ruoqin]

Signed-off-by: Zheng Ruoqin <zhengrq.fnst@cn.fujitsu.com>
---
 engine/tests/Functional.nss/fuego_test.sh   | 19 +++++++++++++++++++
 engine/tests/Functional.nss/nss_test.sh     |  4 ++++
 engine/tests/Functional.nss/parser.py       | 22 ++++++++++++++++++++++
 engine/tests/Functional.nss/spec.json       |  7 +++++++
 engine/tests/Functional.nss/tests/nss_01.sh | 18 ++++++++++++++++++
 5 files changed, 70 insertions(+)
 create mode 100644 engine/tests/Functional.nss/fuego_test.sh
 create mode 100644 engine/tests/Functional.nss/nss_test.sh
 create mode 100644 engine/tests/Functional.nss/parser.py
 create mode 100644 engine/tests/Functional.nss/spec.json
 create mode 100644 engine/tests/Functional.nss/tests/nss_01.sh

diff --git a/engine/tests/Functional.nss/fuego_test.sh b/engine/tests/Functional.nss/fuego_test.sh
new file mode 100644
index 0000000..47ba302
--- /dev/null
+++ b/engine/tests/Functional.nss/fuego_test.sh
@@ -0,0 +1,19 @@
+function test_pre_check {
+    is_on_target_path shlibsign PROGRAM_SHLIBSIGN
+    assert_define PROGRAM_SHLIBSIGN "Missing 'shlibsign' program on target board"
+}
+
+function test_deploy {
+    put $TEST_HOME/nss_test.sh $BOARD_TESTDIR/fuego.$TESTDIR/
+    put -r $TEST_HOME/tests $BOARD_TESTDIR/fuego.$TESTDIR/
+}
+
+function test_run {
+    report "cd $BOARD_TESTDIR/fuego.$TESTDIR;\
+    sh -v nss_test.sh"
+}
+
+function test_processing {
+    log_compare "$TESTDIR" "1" "TEST-PASS" "p"
+    log_compare "$TESTDIR" "0" "TEST-FAIL" "n"
+}
diff --git a/engine/tests/Functional.nss/nss_test.sh b/engine/tests/Functional.nss/nss_test.sh
new file mode 100644
index 0000000..dd5ce37
--- /dev/null
+++ b/engine/tests/Functional.nss/nss_test.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+for i in tests/*.sh; do
+    sh $i
+done
diff --git a/engine/tests/Functional.nss/parser.py b/engine/tests/Functional.nss/parser.py
new file mode 100644
index 0000000..d85abd7
--- /dev/null
+++ b/engine/tests/Functional.nss/parser.py
@@ -0,0 +1,22 @@
+#!/usr/bin/python
+# See common.py for description of command-line arguments
+
+import os, sys, collections
+
+sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
+import common as plib
+
+measurements = {}
+measurements = collections.OrderedDict()
+
+regex_string = '^ -> (.*): TEST-(.*)$'
+matches = plib.parse_log(regex_string)
+
+if matches:
+    for m in matches:
+        measurements['default.' + m[0]] = 'PASS' if m[1] == 'PASS' else 'FAIL'
+
+# split the output for each testcase
+plib.split_output_per_testcase(regex_string, measurements)
+
+sys.exit(plib.process(measurements))
diff --git a/engine/tests/Functional.nss/spec.json b/engine/tests/Functional.nss/spec.json
new file mode 100644
index 0000000..eba3d9a
--- /dev/null
+++ b/engine/tests/Functional.nss/spec.json
@@ -0,0 +1,7 @@
+{
+    "testName": "Functional.nss",
+    "specs": {
+        "default": {}
+    }
+}
+
diff --git a/engine/tests/Functional.nss/tests/nss_01.sh b/engine/tests/Functional.nss/tests/nss_01.sh
new file mode 100644
index 0000000..7d0de09
--- /dev/null
+++ b/engine/tests/Functional.nss/tests/nss_01.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+#  In target, run command shlibsign.
+#  To make sure that there is string "Usage" in output.
+
+test="nss01"
+
+expect <<-EOF
+spawn shlibsign --help
+expect {
+ -re ".*Usage.*" {
+           send_user " -> $test: TEST-PASS\n"
+          }
+ default { send_user " -> $test: TEST-FAIL\n" }  
+ } 
+EOF
+
+
-- 
1.8.3.1

Re: [Fuego] [PATCH] nss: add test case for nss.

[Tim.Bird]

> -----Original Message-----
> From: Zheng Ruoqin
> 
> Signed-off-by: Zheng Ruoqin <zhengrq.fnst@cn.fujitsu.com>
> ---
>  engine/tests/Functional.nss/fuego_test.sh   | 19 +++++++++++++++++++
>  engine/tests/Functional.nss/nss_test.sh     |  4 ++++
>  engine/tests/Functional.nss/parser.py       | 22 ++++++++++++++++++++++
>  engine/tests/Functional.nss/spec.json       |  7 +++++++
>  engine/tests/Functional.nss/tests/nss_01.sh | 18 ++++++++++++++++++
>  5 files changed, 70 insertions(+)
>  create mode 100644 engine/tests/Functional.nss/fuego_test.sh
>  create mode 100644 engine/tests/Functional.nss/nss_test.sh
>  create mode 100644 engine/tests/Functional.nss/parser.py
>  create mode 100644 engine/tests/Functional.nss/spec.json
>  create mode 100644 engine/tests/Functional.nss/tests/nss_01.sh
> 
> diff --git a/engine/tests/Functional.nss/fuego_test.sh
> b/engine/tests/Functional.nss/fuego_test.sh
> new file mode 100644
> index 0000000..47ba302
> --- /dev/null
> +++ b/engine/tests/Functional.nss/fuego_test.sh
> @@ -0,0 +1,19 @@
> +function test_pre_check {
> +    is_on_target_path shlibsign PROGRAM_SHLIBSIGN
> +    assert_define PROGRAM_SHLIBSIGN "Missing 'shlibsign' program on
> target board"
> +}
> +
> +function test_deploy {
> +    put $TEST_HOME/nss_test.sh $BOARD_TESTDIR/fuego.$TESTDIR/
> +    put -r $TEST_HOME/tests $BOARD_TESTDIR/fuego.$TESTDIR/
> +}
> +
> +function test_run {
> +    report "cd $BOARD_TESTDIR/fuego.$TESTDIR;\
> +    sh -v nss_test.sh"
> +}
> +
> +function test_processing {
> +    log_compare "$TESTDIR" "1" "TEST-PASS" "p"
> +    log_compare "$TESTDIR" "0" "TEST-FAIL" "n"
> +}
> diff --git a/engine/tests/Functional.nss/nss_test.sh
> b/engine/tests/Functional.nss/nss_test.sh
> new file mode 100644
> index 0000000..dd5ce37
> --- /dev/null
> +++ b/engine/tests/Functional.nss/nss_test.sh
> @@ -0,0 +1,4 @@
> +#!/bin/sh
> +for i in tests/*.sh; do
> +    sh $i
> +done
> diff --git a/engine/tests/Functional.nss/parser.py
> b/engine/tests/Functional.nss/parser.py
> new file mode 100644
> index 0000000..d85abd7
> --- /dev/null
> +++ b/engine/tests/Functional.nss/parser.py
> @@ -0,0 +1,22 @@
> +#!/usr/bin/python
> +# See common.py for description of command-line arguments
> +
> +import os, sys, collections
> +
> +sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
> +import common as plib
> +
> +measurements = {}
> +measurements = collections.OrderedDict()
> +
> +regex_string = '^ -> (.*): TEST-(.*)$'
> +matches = plib.parse_log(regex_string)
> +
> +if matches:
> +    for m in matches:
> +        measurements['default.' + m[0]] = 'PASS' if m[1] == 'PASS' else 'FAIL'
> +
> +# split the output for each testcase
> +plib.split_output_per_testcase(regex_string, measurements)
> +
> +sys.exit(plib.process(measurements))
> diff --git a/engine/tests/Functional.nss/spec.json
> b/engine/tests/Functional.nss/spec.json
> new file mode 100644
> index 0000000..eba3d9a
> --- /dev/null
> +++ b/engine/tests/Functional.nss/spec.json
> @@ -0,0 +1,7 @@
> +{
> +    "testName": "Functional.nss",
> +    "specs": {
> +        "default": {}
> +    }
> +}
> +
> diff --git a/engine/tests/Functional.nss/tests/nss_01.sh
> b/engine/tests/Functional.nss/tests/nss_01.sh
> new file mode 100644
> index 0000000..7d0de09
> --- /dev/null
> +++ b/engine/tests/Functional.nss/tests/nss_01.sh
> @@ -0,0 +1,18 @@
> +#!/bin/sh
> +
> +#  In target, run command shlibsign.
> +#  To make sure that there is string "Usage" in output.
> +
> +test="nss01"
> +
> +expect <<-EOF
> +spawn shlibsign --help
> +expect {
> + -re ".*Usage.*" {
> +           send_user " -> $test: TEST-PASS\n"
> +          }
Using expect to check the output of a command for a regular expression
is way overkill.

> + default { send_user " -> $test: TEST-FAIL\n" }
> + }
> +EOF
> +
> +
> --
> 1.8.3.1

This is a lot of overhead for something that just checks that "shlibsign --help" produces
the string "Usage" on the target

This whole test could have been done with something simpler directly from fuego_test.sh, like so:
report "if sshlibsign --help | grep Usage ; then echo \" -> nss01: TEST-PASS\" else echo \" ->nss01: TEST-FAIL\""

I presume that this is intended to add the framework
needed for additional testing of shlibsign in the future.  Please let me know. 
But for now this is NOT applied.
 -- Tim

Re: [Fuego] [PATCH] nss: add test case for nss.

[Zheng, Ruoqin]

Hi Tim:

When I try to use "sshlibsign --help", the result of echo $? Is 1, and the test will fail.
So I use expect to instead of shell script.

Could you give me some advice about this question?

--------------------------------------------------
Zheng Ruoqin
Nanjing Fujitsu Nanda Software Tech. Co., Ltd.(FNST)
ADDR.: No.6 Wenzhu Road, Software Avenue,
       Nanjing, 210012, China
MAIL : zhengrq.fnst@cn.fujistu.com


-----Original Message-----
From: Tim.Bird@sony.com [mailto:Tim.Bird@sony.com] 
Sent: Wednesday, September 12, 2018 5:34 AM
To: Zheng, Ruoqin/郑 若钦 <zhengrq.fnst@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH] nss: add test case for nss.

> -----Original Message-----
> From: Zheng Ruoqin
> 
> Signed-off-by: Zheng Ruoqin <zhengrq.fnst@cn.fujitsu.com>
> ---
>  engine/tests/Functional.nss/fuego_test.sh   | 19 +++++++++++++++++++
>  engine/tests/Functional.nss/nss_test.sh     |  4 ++++
>  engine/tests/Functional.nss/parser.py       | 22 ++++++++++++++++++++++
>  engine/tests/Functional.nss/spec.json       |  7 +++++++
>  engine/tests/Functional.nss/tests/nss_01.sh | 18 ++++++++++++++++++
>  5 files changed, 70 insertions(+)
>  create mode 100644 engine/tests/Functional.nss/fuego_test.sh
>  create mode 100644 engine/tests/Functional.nss/nss_test.sh
>  create mode 100644 engine/tests/Functional.nss/parser.py
>  create mode 100644 engine/tests/Functional.nss/spec.json
>  create mode 100644 engine/tests/Functional.nss/tests/nss_01.sh
> 
> diff --git a/engine/tests/Functional.nss/fuego_test.sh
> b/engine/tests/Functional.nss/fuego_test.sh
> new file mode 100644
> index 0000000..47ba302
> --- /dev/null
> +++ b/engine/tests/Functional.nss/fuego_test.sh
> @@ -0,0 +1,19 @@
> +function test_pre_check {
> +    is_on_target_path shlibsign PROGRAM_SHLIBSIGN
> +    assert_define PROGRAM_SHLIBSIGN "Missing 'shlibsign' program on
> target board"
> +}
> +
> +function test_deploy {
> +    put $TEST_HOME/nss_test.sh $BOARD_TESTDIR/fuego.$TESTDIR/
> +    put -r $TEST_HOME/tests $BOARD_TESTDIR/fuego.$TESTDIR/ }
> +
> +function test_run {
> +    report "cd $BOARD_TESTDIR/fuego.$TESTDIR;\
> +    sh -v nss_test.sh"
> +}
> +
> +function test_processing {
> +    log_compare "$TESTDIR" "1" "TEST-PASS" "p"
> +    log_compare "$TESTDIR" "0" "TEST-FAIL" "n"
> +}
> diff --git a/engine/tests/Functional.nss/nss_test.sh
> b/engine/tests/Functional.nss/nss_test.sh
> new file mode 100644
> index 0000000..dd5ce37
> --- /dev/null
> +++ b/engine/tests/Functional.nss/nss_test.sh
> @@ -0,0 +1,4 @@
> +#!/bin/sh
> +for i in tests/*.sh; do
> +    sh $i
> +done
> diff --git a/engine/tests/Functional.nss/parser.py
> b/engine/tests/Functional.nss/parser.py
> new file mode 100644
> index 0000000..d85abd7
> --- /dev/null
> +++ b/engine/tests/Functional.nss/parser.py
> @@ -0,0 +1,22 @@
> +#!/usr/bin/python
> +# See common.py for description of command-line arguments
> +
> +import os, sys, collections
> +
> +sys.path.insert(0, os.environ['FUEGO_CORE'] + 
> +'/engine/scripts/parser') import common as plib
> +
> +measurements = {}
> +measurements = collections.OrderedDict()
> +
> +regex_string = '^ -> (.*): TEST-(.*)$'
> +matches = plib.parse_log(regex_string)
> +
> +if matches:
> +    for m in matches:
> +        measurements['default.' + m[0]] = 'PASS' if m[1] == 'PASS' else 'FAIL'
> +
> +# split the output for each testcase
> +plib.split_output_per_testcase(regex_string, measurements)
> +
> +sys.exit(plib.process(measurements))
> diff --git a/engine/tests/Functional.nss/spec.json
> b/engine/tests/Functional.nss/spec.json
> new file mode 100644
> index 0000000..eba3d9a
> --- /dev/null
> +++ b/engine/tests/Functional.nss/spec.json
> @@ -0,0 +1,7 @@
> +{
> +    "testName": "Functional.nss",
> +    "specs": {
> +        "default": {}
> +    }
> +}
> +
> diff --git a/engine/tests/Functional.nss/tests/nss_01.sh
> b/engine/tests/Functional.nss/tests/nss_01.sh
> new file mode 100644
> index 0000000..7d0de09
> --- /dev/null
> +++ b/engine/tests/Functional.nss/tests/nss_01.sh
> @@ -0,0 +1,18 @@
> +#!/bin/sh
> +
> +#  In target, run command shlibsign.
> +#  To make sure that there is string "Usage" in output.
> +
> +test="nss01"
> +
> +expect <<-EOF
> +spawn shlibsign --help
> +expect {
> + -re ".*Usage.*" {
> +           send_user " -> $test: TEST-PASS\n"
> +          }
Using expect to check the output of a command for a regular expression is way overkill.

> + default { send_user " -> $test: TEST-FAIL\n" }  } EOF
> +
> +
> --
> 1.8.3.1

This is a lot of overhead for something that just checks that "shlibsign --help" produces the string "Usage" on the target

This whole test could have been done with something simpler directly from fuego_test.sh, like so:
report "if sshlibsign --help | grep Usage ; then echo \" -> nss01: TEST-PASS\" else echo \" ->nss01: TEST-FAIL\""

I presume that this is intended to add the framework needed for additional testing of shlibsign in the future.  Please let me know. 
But for now this is NOT applied.
 -- Tim

Re: [Fuego] [PATCH] nss: add test case for nss.

[Tim.Bird]

Sorry this took so long to respond to.


> -----Original Message-----
> From: Zheng, Ruoqin 
> Hi Tim:
> 
> When I try to use "sshlibsign --help", the result of echo $? Is 1, and the test
> will fail.
> So I use expect to instead of shell script.
> 
> Could you give me some advice about this question?

There are several ways to avoid the problem.

I'm going to rattle these off without testing them, but you can
select one you think is best for this situation, and test it.

You can ignore the exit code, and put the results in a file, and
grep that for the Usage information (not do it in a pipe) like so:
--------------
sshlibsign --help >sshlibsign-usage.txt
if grep "Usage..." sshlibsign-usage.txt ;
then
...
fi

rm sshlibsign-usage.txt
--------------

if that first line causes the test shell script to exit, then you may
be in 'set -e' mode.  In which case, you can ignore the exit code
by putting it in a conditional sequence, like so:
sshlibsign --help >sshlibsign-usage.txt || true

If you can get rid of the need for 'expect', that will be good.
 -- Tim

> 
> --------------------------------------------------
> Zheng Ruoqin
> Nanjing Fujitsu Nanda Software Tech. Co., Ltd.(FNST)
> ADDR.: No.6 Wenzhu Road, Software Avenue,
>        Nanjing, 210012, China
> MAIL : zhengrq.fnst@cn.fujistu.com
> 
> 
> -----Original Message-----
> From: Tim.Bird@sony.com [mailto:Tim.Bird@sony.com]
> Sent: Wednesday, September 12, 2018 5:34 AM
> To: Zheng, Ruoqin/郑 若钦 <zhengrq.fnst@cn.fujitsu.com>;
> fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH] nss: add test case for nss.
> 
> > -----Original Message-----
> > From: Zheng Ruoqin
> >
> > Signed-off-by: Zheng Ruoqin <zhengrq.fnst@cn.fujitsu.com>
> > ---
> >  engine/tests/Functional.nss/fuego_test.sh   | 19 +++++++++++++++++++
> >  engine/tests/Functional.nss/nss_test.sh     |  4 ++++
> >  engine/tests/Functional.nss/parser.py       | 22
> ++++++++++++++++++++++
> >  engine/tests/Functional.nss/spec.json       |  7 +++++++
> >  engine/tests/Functional.nss/tests/nss_01.sh | 18 ++++++++++++++++++
> >  5 files changed, 70 insertions(+)
> >  create mode 100644 engine/tests/Functional.nss/fuego_test.sh
> >  create mode 100644 engine/tests/Functional.nss/nss_test.sh
> >  create mode 100644 engine/tests/Functional.nss/parser.py
> >  create mode 100644 engine/tests/Functional.nss/spec.json
> >  create mode 100644 engine/tests/Functional.nss/tests/nss_01.sh
> >
> > diff --git a/engine/tests/Functional.nss/fuego_test.sh
> > b/engine/tests/Functional.nss/fuego_test.sh
> > new file mode 100644
> > index 0000000..47ba302
> > --- /dev/null
> > +++ b/engine/tests/Functional.nss/fuego_test.sh
> > @@ -0,0 +1,19 @@
> > +function test_pre_check {
> > +    is_on_target_path shlibsign PROGRAM_SHLIBSIGN
> > +    assert_define PROGRAM_SHLIBSIGN "Missing 'shlibsign' program on
> > target board"
> > +}
> > +
> > +function test_deploy {
> > +    put $TEST_HOME/nss_test.sh $BOARD_TESTDIR/fuego.$TESTDIR/
> > +    put -r $TEST_HOME/tests $BOARD_TESTDIR/fuego.$TESTDIR/ }
> > +
> > +function test_run {
> > +    report "cd $BOARD_TESTDIR/fuego.$TESTDIR;\
> > +    sh -v nss_test.sh"
> > +}
> > +
> > +function test_processing {
> > +    log_compare "$TESTDIR" "1" "TEST-PASS" "p"
> > +    log_compare "$TESTDIR" "0" "TEST-FAIL" "n"
> > +}
> > diff --git a/engine/tests/Functional.nss/nss_test.sh
> > b/engine/tests/Functional.nss/nss_test.sh
> > new file mode 100644
> > index 0000000..dd5ce37
> > --- /dev/null
> > +++ b/engine/tests/Functional.nss/nss_test.sh
> > @@ -0,0 +1,4 @@
> > +#!/bin/sh
> > +for i in tests/*.sh; do
> > +    sh $i
> > +done
> > diff --git a/engine/tests/Functional.nss/parser.py
> > b/engine/tests/Functional.nss/parser.py
> > new file mode 100644
> > index 0000000..d85abd7
> > --- /dev/null
> > +++ b/engine/tests/Functional.nss/parser.py
> > @@ -0,0 +1,22 @@
> > +#!/usr/bin/python
> > +# See common.py for description of command-line arguments
> > +
> > +import os, sys, collections
> > +
> > +sys.path.insert(0, os.environ['FUEGO_CORE'] +
> > +'/engine/scripts/parser') import common as plib
> > +
> > +measurements = {}
> > +measurements = collections.OrderedDict()
> > +
> > +regex_string = '^ -> (.*): TEST-(.*)$'
> > +matches = plib.parse_log(regex_string)
> > +
> > +if matches:
> > +    for m in matches:
> > +        measurements['default.' + m[0]] = 'PASS' if m[1] == 'PASS' else 'FAIL'
> > +
> > +# split the output for each testcase
> > +plib.split_output_per_testcase(regex_string, measurements)
> > +
> > +sys.exit(plib.process(measurements))
> > diff --git a/engine/tests/Functional.nss/spec.json
> > b/engine/tests/Functional.nss/spec.json
> > new file mode 100644
> > index 0000000..eba3d9a
> > --- /dev/null
> > +++ b/engine/tests/Functional.nss/spec.json
> > @@ -0,0 +1,7 @@
> > +{
> > +    "testName": "Functional.nss",
> > +    "specs": {
> > +        "default": {}
> > +    }
> > +}
> > +
> > diff --git a/engine/tests/Functional.nss/tests/nss_01.sh
> > b/engine/tests/Functional.nss/tests/nss_01.sh
> > new file mode 100644
> > index 0000000..7d0de09
> > --- /dev/null
> > +++ b/engine/tests/Functional.nss/tests/nss_01.sh
> > @@ -0,0 +1,18 @@
> > +#!/bin/sh
> > +
> > +#  In target, run command shlibsign.
> > +#  To make sure that there is string "Usage" in output.
> > +
> > +test="nss01"
> > +
> > +expect <<-EOF
> > +spawn shlibsign --help
> > +expect {
> > + -re ".*Usage.*" {
> > +           send_user " -> $test: TEST-PASS\n"
> > +          }
> Using expect to check the output of a command for a regular expression is
> way overkill.
> 
> > + default { send_user " -> $test: TEST-FAIL\n" }  } EOF
> > +
> > +
> > --
> > 1.8.3.1
> 
> This is a lot of overhead for something that just checks that "shlibsign --help"
> produces the string "Usage" on the target
> 
> This whole test could have been done with something simpler directly from
> fuego_test.sh, like so:
> report "if sshlibsign --help | grep Usage ; then echo \" -> nss01: TEST-PASS\"
> else echo \" ->nss01: TEST-FAIL\""
> 
> I presume that this is intended to add the framework needed for additional
> testing of shlibsign in the future.  Please let me know.
> But for now this is NOT applied.
>  -- Tim
> 
> 
> 
>