[Fuego] [PATCH] LTP_one_test: Support use of runltp and ltp-pan

* [Fuego] [PATCH] LTP_one_test: Support use of runltp and ltp-pan
@ 2018-04-18  1:08 Tim.Bird
  2018-04-18  2:02 ` Liu, Wenlong
  0 siblings, 1 reply; 3+ messages in thread
From: Tim.Bird @ 2018-04-18  1:08 UTC (permalink / raw)
  To: fuego

OK - Here's an update to LTP_one_test that offers more flexibility
when running a single test.
----
Add support to the test to run the individual test program
on the board either directly (supported previously) or using
runltp and ltp-pan.  The benefit of running directly is that it
uses a simpler invocation, and you get less 'noise' in the output.
The advantage of running using runltp and ltp-pan is that you
get more setup and test management.

In order to run using runltp, the user must specify the 'scenario'
file (in the runtest directory) which has an entry for the specified
test.

Add some additional specs to demonstrate how to do this.

Signed-off-by: Tim Bird <tim.bird@sony.com>
---
 engine/tests/Functional.LTP_one_test/fuego_test.sh | 54 +++++++++++++++++++---
 engine/tests/Functional.LTP_one_test/parser.py     | 25 ++++++----
 engine/tests/Functional.LTP_one_test/spec.json     | 11 ++++-
 engine/tests/Functional.LTP_one_test/test.yaml     | 12 +++++
 4 files changed, 84 insertions(+), 18 deletions(-)

diff --git a/engine/tests/Functional.LTP_one_test/fuego_test.sh b/engine/tests/Functional.LTP_one_test/fuego_test.sh
index 1d99d9c..deb413e 100755
--- a/engine/tests/Functional.LTP_one_test/fuego_test.sh
+++ b/engine/tests/Functional.LTP_one_test/fuego_test.sh
@@ -6,33 +6,73 @@ one_test=$FUNCTIONAL_LTP_ONE_TEST_TEST
 # individual program to deploy.
 FUNCTIONAL_LTP_ONE_TEST_PER_JOB_BUILD="true"
 
+# This test can either run the test directly, or run it via
+# runltp and ltp-pan.  To use runltp, set the value of "scenario"
+# in the spec to the correct scenario name.  (e.g. 'syscalls')
+# The scenario is the name of the the file in the runtest directory
+# that has an entry (possibly with command line arguments) for this test.
+
 function test_pre_check {
     assert_define FUNCTIONAL_LTP_ONE_TEST_TEST
 }
 
 function test_build {
     # check for LTP build directory
-    LTP_BUILD_DIR=$(echo $JOB_BUILD_DIR | sed s/LTP_one_test/LTP/ | sed s/$TESTSPEC/default/)
+    LTP_BUILD_DIR="${WORKSPACE}/$(echo $JOB_BUILD_DIR | sed s/LTP_one_test/LTP/ | sed s/$TESTSPEC/default/)"
+    echo "LTP_BUILD_DIR=${LTP_BUILD_DIR}"
 
     # if not already built, build LTP
-    if [ ! -e ${WORKSPACE}/${LTP_BUILD_DIR}/fuego_test_successfully_built ] ; then
+    if [ ! -e ${LTP_BUILD_DIR}/fuego_test_successfully_built ] ; then
         echo "Building parent LTP test..."
         ftc run-test -b $NODE_NAME -t Functional.LTP -p b
         # NOTE: vars used in ftc run-test should not leak into this environment
         # that is, none of our test vars should have changed.
     fi
 
-    cp ${WORKSPACE}/${LTP_BUILD_DIR}/target_bin/testcases/bin/$one_test .
+    cp ${LTP_BUILD_DIR}/target_bin/testcases/bin/$one_test .
 }
 
 function test_deploy {
-    put $one_test $BOARD_TESTDIR/fuego.$TESTDIR/
+    # set LTP_BUILD_DIR (possibly again), in case test_build was skipped
+    LTP_BUILD_DIR="${WORKSPACE}/$(echo $JOB_BUILD_DIR | sed s/LTP_one_test/LTP/ | sed s/$TESTSPEC/default/)"
+
+    local bdir="$BOARD_TESTDIR/fuego.$TESTDIR"
+    local scenario=$FUNCTIONAL_LTP_ONE_TEST_SCENARIO
+    if [ -z "$scenario" ] ; then
+        put $one_test $bdir/
+        # if the user hasn't specified any arguments,
+        # warn user if there are default args
+        if [ -z "$FUNCTIONAL_LTP_ONE_TEST_ARGS" ] ; then
+            args=$(grep $one_test ${LTP_BUILD_DIR}/runtest/* | cut -d" " -f3- | head -n 1)
+            if [ -n "$args" ] ; then
+                echo "Warning: No arguments specified, but at least one scenario group for"
+                echo "test '$one_test' has args of '${args}'"
+                echo "Maybe add \"ARGS\":\"${args}\" in spec file?"
+            fi
+        fi
+    else
+        # copy helper files, runltp, ltp-pan, scenario file and the
+        # test program to the board
+        cmd "mkdir -p $bdir/bin $bdir/runtest $bdir/testcases/bin"
+        put ${LTP_BUILD_DIR}/target_bin/IDcheck.sh $bdir/
+        put ${LTP_BUILD_DIR}/target_bin/ver_linux $bdir/
+        put ${LTP_BUILD_DIR}/target_bin/Version $bdir/
+        put ${LTP_BUILD_DIR}/target_bin/runltp $bdir/
+        put ${LTP_BUILD_DIR}/target_bin/bin/ltp-pan $bdir/bin/
+        put ${LTP_BUILD_DIR}/target_bin/runtest/$scenario $bdir/runtest/
+        put $one_test $bdir/testcases/bin/
+    fi
 }
 
 function test_run {
-    # FIXTHIS - deal with arguments better
-    report "cd $BOARD_TESTDIR/fuego.$TESTDIR; \
-        ./$one_test $FUNCTIONAL_LTP_ONE_TEST_ARGS"
+    local bdir="$BOARD_TESTDIR/fuego.$TESTDIR"
+    local scenario=$FUNCTIONAL_LTP_ONE_TEST_SCENARIO
+
+    if [ -z "$scenario" ] ; then
+        report "cd $bdir; ./$one_test $FUNCTIONAL_LTP_ONE_TEST_ARGS"
+    else
+        report "cd $bdir; ./runltp -f $scenario -s $one_test"
+    fi
 }
 
 function test_processing {
diff --git a/engine/tests/Functional.LTP_one_test/parser.py b/engine/tests/Functional.LTP_one_test/parser.py
index fe2336f..855efe1 100755
--- a/engine/tests/Functional.LTP_one_test/parser.py
+++ b/engine/tests/Functional.LTP_one_test/parser.py
@@ -20,15 +20,19 @@ re_pattern = r"(\S+)\s+(\S)+\s+(TPASS|TCONF)\s+:(.*)"
 
 testcase_count = 1
 for line in log_lines:
+    result = None
     #print "line=", line
     if ".c:" in line:
         parts = line.split(":")
         filename = parts[0]
         line_no = parts[1]
-        testcase_num = str(testcase_count)
-        testcase_count += 1
         result = parts[2].strip()
         message = parts[3].strip()
+        if result == "INFO":
+            result = None
+        else:
+            testcase_num = str(testcase_count)
+            testcase_count += 1
     else:
         m = re.match(re_pattern, line)
         #print "m=", m
@@ -39,13 +43,14 @@ for line in log_lines:
             message = m.group(4).strip()
 
     # translate to Fuego results
-    if result == 'PASS':
-        results[testcase_num] = 'PASS'
-    elif result == 'FAIL':
-        results[testcase_num] = 'FAIL'
-    elif result == 'CONF':
-        results[testcase_num] = 'SKIP'
-    else:
-        results[testcase_num] = 'ERR'
+    if result:
+        if result == 'PASS':
+            results[testcase_num] = 'PASS'
+        elif result == 'FAIL':
+            results[testcase_num] = 'FAIL'
+        elif result == 'CONF':
+            results[testcase_num] = 'SKIP'
+        else:
+            results[testcase_num] = 'ERR'
 
 sys.exit(plib.process(results))
diff --git a/engine/tests/Functional.LTP_one_test/spec.json b/engine/tests/Functional.LTP_one_test/spec.json
index 9f0b325..cd78581 100644
--- a/engine/tests/Functional.LTP_one_test/spec.json
+++ b/engine/tests/Functional.LTP_one_test/spec.json
@@ -4,9 +4,18 @@
         "default": {
             "TEST":"brk01"
         },
+        "fanotify06": {
+            "TEST":"fanotify06"
+        },
         "fanotify07": {
             "TEST":"fanotify07"
+        },
+        "mlock03": {
+            "TEST":"mlock03"
+        },
+        "syscalls-mlock03": {
+            "TEST":"mlock03",
+            "scenario":"syscalls"
         }
     }
 }
-
diff --git a/engine/tests/Functional.LTP_one_test/test.yaml b/engine/tests/Functional.LTP_one_test/test.yaml
index 765c98d..2ff0b19 100644
--- a/engine/tests/Functional.LTP_one_test/test.yaml
+++ b/engine/tests/Functional.LTP_one_test/test.yaml
@@ -26,6 +26,18 @@ params:
         description: Command line arguments for the LTP test program.
         example: -s foo
         optional: yes
+    - scenario:
+        description: |
+            Name of the LTP scenario file that has an entry for this test.
+            The scenario files are located in LTP's 'runtest' directory.
+            If this is specified, then Functional.LTP_one_test will 
+            invoke the test using runltp and ltp-pan, which should do
+            more setup, and better match the environment when the test
+            was run in the context of the full Functional.LTP test.
+
+            If this is not specified, then the test is run directly.
+        example: syscalls
+        optional: yes
 data_files:
     - spec.json
     - parser.py
-- 
2.1.4


^ permalink raw reply related	[flat|nested] 3+ messages in thread