[Fuego] [PATCH 01/12] add network testcase to spec

[Fuego] [PATCH 01/12] add network testcase to spec

[qiutt]

include 
 net.features net.ipv6 net.ipv6_lib net.multicast 
 net.nfs net.rpc net.rpc_tests net.sctp
 net_stress.appl net_stress.broken_ip net_stress.interface
 net_stress.ipsec_dccp net_stress.ipsec_icmp net_stress.ipsec_sctp
 net_stress.ipsec_tcp net_stress.ipsec_udp net_stress.multicast
 net_stress.route net.tcp_cmds net.tirpc_tests network_commands smack

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/spec.json | 111 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 111 insertions(+)

diff --git a/tests/Functional.LTP/spec.json b/tests/Functional.LTP/spec.json
index d23edc1..8da77b0 100644
--- a/tests/Functional.LTP/spec.json
+++ b/tests/Functional.LTP/spec.json
@@ -73,6 +73,117 @@
             "tests": "cve",
             "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
             "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.ipv6": {
+            "tests": "net.ipv6",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.rpc_tests": {
+            "tests": "net.rpc_tests",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.features": {
+            "tests": "net.features",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.ipv6_lib": {
+            "tests": "net.ipv6_lib",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "BOARD_NAME": "m3ulcb"
+        },
+        "net.multicast": {
+            "tests": "net.multicast",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.nfs": {
+            "tests": "net.nfs",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.rpc": {
+            "tests": "net.rpc",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.sctp": {
+            "tests": "net.sctp",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.appl": {
+            "tests": "net_stress.appl",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.broken_ip": {
+            "tests": "net_stress.broken_ip",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.interface": {
+            "tests": "net_stress.interface",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.ipsec_dccp": {
+            "tests": "net_stress.ipsec_dccp",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.ipsec_icmp": {
+            "tests": "net_stress.ipsec_icmp",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.ipsec_sctp": {
+            "tests": "net_stress.ipsec_sctp",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.ipsec_tcp": {
+            "tests": "net_stress.ipsec_tcp",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.ipsec_udp": {
+            "tests": "net_stress.ipsec_udp",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.multicast": {
+            "tests": "net_stress.multicast",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net_stress.route": {
+            "tests": "net_stress.route",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.tcp_cmds": {
+            "tests": "net.tcp_cmds ",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "net.tirpc_tests": {
+            "tests": "net.tirpc_tests",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "network_commands": {
+            "tests": "network_commands",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "smack": {
+            "tests": "smack",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
         }
     }
 }
-- 
2.11.0

[Fuego] [PATCH 02/12] skip cases missing ftp cmd

[qiutt]

for m3ulcb, skip cases in  network_commands and net.tcp_cmds

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 7f1a97f..8aaffd8 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -235,6 +235,10 @@ function test_pre_check {
     skip_if_command_unavailable touch "ext4-nsec-timestamps"
     skip_if_command_unavailable quotacheck "quotactl01"
 
+    # some cmds do not exist in m3ulcb
+    skip_if_command_unavailable ftp "ftp04" # for network_commands
+    skip_if_command_unavailable ftp "ftp" # for net.tcp_cmds
+
     echo "Tests skipped depending on the configuration of the target kernel"
     skip_if_kconfig_differs "CONFIG_INOTIFY_USER=y" "inotify_init1_01 inotify_init1_02 inotify01 inotify02 inotify03 inotify04 inotify05 inotify06"
     skip_if_kconfig_differs "CONFIG_FANOTIFY=y CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y" "fanotify01 fanotify02 fanotify03 fanotify04 fanotify05 fanotify06"
-- 
2.11.0

Re: [Fuego] [PATCH 02/12] skip cases missing ftp cmd

[Bird, Tim]

> -----Original Message-----
> From:  qiutt
> 
> for m3ulcb, skip cases in  network_commands and net.tcp_cmds
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
> index 7f1a97f..8aaffd8 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -235,6 +235,10 @@ function test_pre_check {
>      skip_if_command_unavailable touch "ext4-nsec-timestamps"
>      skip_if_command_unavailable quotacheck "quotactl01"
> 
> +    # some cmds do not exist in m3ulcb

In general, this type of comment is not needed in the code,
if there is sufficient explanation in the commit message.

However, this one is innocuous enough that I'm going to leave it.

> +    skip_if_command_unavailable ftp "ftp04" # for network_commands
> +    skip_if_command_unavailable ftp "ftp" # for net.tcp_cmds
> +
>      echo "Tests skipped depending on the configuration of the target kernel"
>      skip_if_kconfig_differs "CONFIG_INOTIFY_USER=y" "inotify_init1_01 inotify_init1_02 inotify01 inotify02 inotify03 inotify04 inotify05
> inotify06"
>      skip_if_kconfig_differs "CONFIG_FANOTIFY=y CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y" "fanotify01 fanotify02 fanotify03
> fanotify04 fanotify05 fanotify06"
> --
> 2.11.0

This looks reasonable.  I'm applying it to the 'master' branch.
 -- Tim

[Fuego] [PATCH 03/12] skip cases missing rsh cmd

[qiutt]

for m3ulcb, skip casesc in net_stress.route and net.tcp_cmds

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 8aaffd8..bde09eb 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -238,6 +238,8 @@ function test_pre_check {
     # some cmds do not exist in m3ulcb
     skip_if_command_unavailable ftp "ftp04" # for network_commands
     skip_if_command_unavailable ftp "ftp" # for net.tcp_cmds
+    skip_if_command_unavailable rsh "route4-change-dst route4-change-gw route4-change-if route4-redirect route4-ifdown route4-rmmod route6-change-dst route6-change-gw route6-change-if route6-redirect route6-ifdown route6-rmmod " # for net_stress.route
+    skip_if_command_unavailable rsh "rsh" # for net.tcp_cmds
 
     echo "Tests skipped depending on the configuration of the target kernel"
     skip_if_kconfig_differs "CONFIG_INOTIFY_USER=y" "inotify_init1_01 inotify_init1_02 inotify01 inotify02 inotify03 inotify04 inotify05 inotify06"
-- 
2.11.0

Re: [Fuego] [PATCH 03/12] skip cases missing rsh cmd

[Bird, Tim]

> -----Original Message-----
> From:  qiutt
> 
> for m3ulcb, skip casesc in net_stress.route and net.tcp_cmds
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 2 ++
>  1 file changed, 2 insertions(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
> index 8aaffd8..bde09eb 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -238,6 +238,8 @@ function test_pre_check {
>      # some cmds do not exist in m3ulcb
>      skip_if_command_unavailable ftp "ftp04" # for network_commands
>      skip_if_command_unavailable ftp "ftp" # for net.tcp_cmds
> +    skip_if_command_unavailable rsh "route4-change-dst route4-change-gw route4-change-if route4-redirect route4-ifdown route4-
> rmmod route6-change-dst route6-change-gw route6-change-if route6-redirect route6-ifdown route6-rmmod " # for net_stress.route

I wish that line wasn't so long, but I guess there's already a precedent for
ridiculously long lines in this test_pre_check() function.

> +    skip_if_command_unavailable rsh "rsh" # for net.tcp_cmds
> 
>      echo "Tests skipped depending on the configuration of the target kernel"
>      skip_if_kconfig_differs "CONFIG_INOTIFY_USER=y" "inotify_init1_01 inotify_init1_02 inotify01 inotify02 inotify03 inotify04 inotify05
> inotify06"
> --
> 2.11.0

Looks OK.  I'm trusting that these dependencies are real. Applied.
 -- Tim

[Fuego] [PATCH 04/12] skip cases missing host cmd

[qiutt]

for m3ulcb, skip cases in net.tcp_cmds

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index bde09eb..1fadf49 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -240,6 +240,7 @@ function test_pre_check {
     skip_if_command_unavailable ftp "ftp" # for net.tcp_cmds
     skip_if_command_unavailable rsh "route4-change-dst route4-change-gw route4-change-if route4-redirect route4-ifdown route4-rmmod route6-change-dst route6-change-gw route6-change-if route6-redirect route6-ifdown route6-rmmod " # for net_stress.route
     skip_if_command_unavailable rsh "rsh" # for net.tcp_cmds
+    skip_if_command_unavailable host "host" # for net.tcp_cmds
 
     echo "Tests skipped depending on the configuration of the target kernel"
     skip_if_kconfig_differs "CONFIG_INOTIFY_USER=y" "inotify_init1_01 inotify_init1_02 inotify01 inotify02 inotify03 inotify04 inotify05 inotify06"
-- 
2.11.0

[Fuego] [PATCH 05/12] skip cases missing rcp cmd

[qiutt]

for m3ulcb, skip cases in net.tcp_cmds

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 1fadf49..156fd39 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -241,6 +241,7 @@ function test_pre_check {
     skip_if_command_unavailable rsh "route4-change-dst route4-change-gw route4-change-if route4-redirect route4-ifdown route4-rmmod route6-change-dst route6-change-gw route6-change-if route6-redirect route6-ifdown route6-rmmod " # for net_stress.route
     skip_if_command_unavailable rsh "rsh" # for net.tcp_cmds
     skip_if_command_unavailable host "host" # for net.tcp_cmds
+    skip_if_command_unavailable rcp "rcp" # for net.tcp_cmds
 
     echo "Tests skipped depending on the configuration of the target kernel"
     skip_if_kconfig_differs "CONFIG_INOTIFY_USER=y" "inotify_init1_01 inotify_init1_02 inotify01 inotify02 inotify03 inotify04 inotify05 inotify06"
-- 
2.11.0

Re: [Fuego] [PATCH 05/12] skip cases missing rcp cmd

[Bird, Tim]

patches 4 and 5 are OK.  Applied.
 -- Tim


> -----Original Message-----
> From: Fuego <fuego-bounces@lists.linuxfoundation.org> On Behalf Of qiutt
> Sent: Wednesday, July 8, 2020 4:26 AM
> To: fuego@lists.linuxfoundation.org
> Subject: [Fuego] [PATCH 05/12] skip cases missing rcp cmd
> 
> for m3ulcb, skip cases in net.tcp_cmds
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
> index 1fadf49..156fd39 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -241,6 +241,7 @@ function test_pre_check {
>      skip_if_command_unavailable rsh "route4-change-dst route4-change-gw route4-change-if route4-redirect route4-ifdown route4-rmmod
> route6-change-dst route6-change-gw route6-change-if route6-redirect route6-ifdown route6-rmmod " # for net_stress.route
>      skip_if_command_unavailable rsh "rsh" # for net.tcp_cmds
>      skip_if_command_unavailable host "host" # for net.tcp_cmds
> +    skip_if_command_unavailable rcp "rcp" # for net.tcp_cmds
> 
>      echo "Tests skipped depending on the configuration of the target kernel"
>      skip_if_kconfig_differs "CONFIG_INOTIFY_USER=y" "inotify_init1_01 inotify_init1_02 inotify01 inotify02 inotify03 inotify04 inotify05
> inotify06"
> --
> 2.11.0
> 
> 
> 
> _______________________________________________
> Fuego mailing list
> Fuego@lists.linuxfoundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/fuego

[Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[qiutt]

for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc
 net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip
 net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp net_stress.ipsec_tcp
 net_stress.ipsec_udp  net_stress.ipsec_sctp net_stress.multicast net.tcp_cmds net.tirpc_tests

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 156fd39..ab4165d 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -251,6 +251,65 @@ function test_pre_check {
     skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
     skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
 
+    # some CONFIGs do not set in m3ulcb
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01 vlan03" # for net.features
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602 sendfile601 traceroute601 dhcpd6" # for net.ipv6
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create rpc_clnt_destroy \ 
+    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror rpc_clnt_sperrno rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \ 
+    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc rpc_clnt_freeres \ 
+    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect \ 
+    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg test_1_to_1_sendto test_1_to_1_shutdown test_1_to_1_socket_bind_listen \ 
+    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort test_assoc_shutdown test_autoclose test_basic test_basic_v6 test_connect test_connectx test_fragments test_fragments_v6 test_getname \ 
+    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style \ 
+    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for net.sctp
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "broken_ip4-version broken_ip4-ihl broken_ip4-totlen broken_ip4-fragment broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr broken_ip6-dstaddr \ 
+    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-route-addlarge if4-mtu-change if6-updown if6-addr-adddel \ 
+    if6-addr-addlarge if6-route-adddel if6-route-addlarge if6-mtu-change" # for net_stress.interface
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01 dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05 dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 dccp4_ipsec10 \ 
+    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14 dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18 dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 dccp4_ipsec23 \ 
+    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27 dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31 dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 dccp4_ipsec36 \ 
+    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04 dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08 dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 dccp6_ipsec13 \ 
+    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17 dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21 dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 dccp6_ipsec26 \ 
+    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30 dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34 dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01 icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \ 
+    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11 icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14 icmp4-uni-basic15 icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \ 
+    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21 icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24 icmp4-uni-basic25 icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \ 
+    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05 icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08 icmp6-uni-basic09 icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \ 
+    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15 icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18 icmp6-uni-basic19 icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \ 
+    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01 tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06 tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \ 
+    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 tcp4_ipsec16 tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20 tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \ 
+    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 tcp4_ipsec30 tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34 tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \ 
+    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 tcp6_ipsec08 tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12 tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \ 
+    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 tcp6_ipsec22 tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26 tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \ 
+    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01 udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05 udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10 udp4_ipsec11 \ 
+    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 udp4_ipsec16 udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20 udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \ 
+    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 udp4_ipsec30 udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34 udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \ 
+    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 udp6_ipsec08 udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12 udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \ 
+    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 udp6_ipsec22 udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26 udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \ 
+    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01 sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05 sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 sctp4_ipsec10 \ 
+    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14 sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18 sctp4_ipsec19 sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \ 
+    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27 sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31 sctp4_ipsec32 sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \ 
+    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04 sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08 sctp6_ipsec09 sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \ 
+    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17 sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21 sctp6_ipsec22 sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \ 
+    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30 sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34 sctp6_ipsec35 sctp6_ipsec36" # for net_stress.ipsec_sctp
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mcast4-group-single-socket mcast4-group-multiple-socket mcast4-group-same-group mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \ 
+    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03 mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 mcast6-group-single-socket mcast6-group-multiple-socket mcast6-group-same-group \ 
+    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02 mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for net_stress.multicast
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
+    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tirpc_rpcb_getaddr tirpc_rpcb_getmaps tirpc_authdes_create tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \ 
+    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers tirpc_svcerr_systemerr tirpc_svcerr_weakauth \ 
+    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call tirpc_clnt_tp_create tirpc_clnt_tp_create_timed tirpc_svc_tp_create \ 
+    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp tirpc_clnt_create tirpc_clnt_create_timed tirpc_svc_create \ 
+    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" # for net.tirpc_tests
+
     echo "Tests skipped depending on the architecture"
     if [ "$ARCHITECTURE" != "s390" ]; then
         skip_tests "sbrk03" # Only works in 32bit on s390 series system
-- 
2.11.0

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Bird, Tim]

This one doesn't look right. See comments below.

> -----Original Message-----
> From: qiutt
> 
> for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc
>  net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip
>  net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp net_stress.ipsec_tcp
>  net_stress.ipsec_udp  net_stress.ipsec_sctp net_stress.multicast net.tcp_cmds net.tirpc_tests
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 59 ++++++++++++++++++++++++++++++++++++++
>  1 file changed, 59 insertions(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
> index 156fd39..ab4165d 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -251,6 +251,65 @@ function test_pre_check {
>      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
>      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> 
> +    # some CONFIGs do not set in m3ulcb
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01 vlan03"
> # for net.features
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602 sendfile601 traceroute601 dhcpd6" # for net.ipv6

Do ping and traceroute depend on CONFIG_XFRM_USER=y?
That doesn't seem right.  

Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the
tests to see if this was the case or not.

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create rpc_clnt_destroy \
> +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror rpc_clnt_sperrno
> rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc rpc_clnt_freeres \
> +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect \
> +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> test_1_to_1_sendto test_1_to_1_shutdown test_1_to_1_socket_bind_listen \
> +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort test_assoc_shutdown test_autoclose test_basic test_basic_v6 test_connect
> test_connectx test_fragments test_fragments_v6 test_getname \
> +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style \
> +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for net.sctp

A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
Are you just using CONFIG_XFRM_USER as a placeholder for networking config
for the m3ulcb board?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "broken_ip4-version broken_ip4-ihl broken_ip4-totlen broken_ip4-fragment
> broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr broken_ip6-dstaddr \
> +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> +    if6-addr-addlarge if6-route-adddel if6-route-addlarge if6-mtu-change" # for net_stress.interface

Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01 dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 dccp4_ipsec10 \
> +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14 dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 dccp4_ipsec23 \
> +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27 dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 dccp4_ipsec36 \
> +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04 dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 dccp6_ipsec13 \
> +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17 dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 dccp6_ipsec26 \
> +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30 dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp

This is a LOT of dependencies!  Is there any way to use wildcards here,
to avoid having to list all of these testcases individually?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01 icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11 icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14 icmp4-uni-basic15
> icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21 icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24 icmp4-uni-basic25
> icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05 icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08 icmp6-uni-basic09
> icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15 icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18 icmp6-uni-basic19
> icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01 tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 tcp4_ipsec16 tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 tcp4_ipsec30 tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 tcp6_ipsec08 tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 tcp6_ipsec22 tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01 udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10 udp4_ipsec11 \
> +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 udp4_ipsec16 udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 udp4_ipsec30 udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 udp6_ipsec08 udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 udp6_ipsec22 udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01 sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 sctp4_ipsec10 \
> +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14 sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18 sctp4_ipsec19
> sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27 sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31 sctp4_ipsec32
> sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04 sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08 sctp6_ipsec09
> sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17 sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21 sctp6_ipsec22
> sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30 sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34 sctp6_ipsec35
> sctp6_ipsec36" # for net_stress.ipsec_sctp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mcast4-group-single-socket mcast4-group-multiple-socket mcast4-group-same-group
> mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03 mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 mcast6-group-
> single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02 mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for net_stress.multicast

A lot of these lines are very long, and the dependency seems a bit sketch.

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tirpc_rpcb_getaddr tirpc_rpcb_getmaps tirpc_authdes_create
> tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call tirpc_clnt_tp_create
> tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp tirpc_clnt_create
> tirpc_clnt_create_timed tirpc_svc_create \
> +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" # for net.tirpc_tests
> +
>      echo "Tests skipped depending on the architecture"
>      if [ "$ARCHITECTURE" != "s390" ]; then
>          skip_tests "sbrk03" # Only works in 32bit on s390 series system
> --
> 2.11.0

I have too many questions about this one.  I did NOT apply it.
 -- Tim

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Qiu, Tingting]

Hi,Tim

Cases such as ping, tcp, udp and traceroute are all based on the virtual network card. 
Initializing the virtual network card is implemented by the common script(test_net.sh). 
The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
Therefore, all cases that use virtualized network cards will be affected in the same way. 
It is reasonable to use the "CONFIG_XFRM_USER=y" to check.

The following is my guess:
There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
when initializing the virtualized network card, using "ip xfrm" is necessary, 
because many ipsec-related content need to be tested in the network test set (at least 504 cases).

*1:
tst_init_iface()
{
         local type="${1:-lhost}"
         local link_num="${2:-0}"
         local iface="$(tst_iface $type $link_num)"
         tst_resm TINFO "initialize '$type' '$iface' interface"

         if [ "$type" = "lhost" ]; then
                 ip xfrm policy flush || return $?
                 ip xfrm state flush || return $?
                 ip link set $iface down || return $?
                 ip route flush dev $iface || return $?
                 ip addr flush dev $iface || return $?
                 ip link set $iface up
                 return $?
         fi

         tst_rhost_run -c "ip xfrm policy flush" || return $?
         tst_rhost_run -c "ip xfrm state flush" || return $?
         tst_rhost_run -c "ip link set $iface down" || return $?
         tst_rhost_run -c "ip route flush dev $iface" || return $?
         tst_rhost_run -c "ip addr flush dev $iface" || return $?
         tst_rhost_run -c "ip link set $iface up"
}

Thanks,
 -- QiuTingting

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Thursday, July 9, 2020 6:13 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

This one doesn't look right. See comments below.

> -----Original Message-----
> From: qiutt
> 
> for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc  
> net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip  
> net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp 
> net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp 
> net_stress.multicast net.tcp_cmds net.tirpc_tests
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 59 
> ++++++++++++++++++++++++++++++++++++++
>  1 file changed, 59 insertions(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh 
> b/tests/Functional.LTP/fuego_test.sh
> index 156fd39..ab4165d 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -251,6 +251,65 @@ function test_pre_check {
>      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
>      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> 
> +    # some CONFIGs do not set in m3ulcb
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01 vlan03"
> # for net.features
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602 
> + sendfile601 traceroute601 dhcpd6" # for net.ipv6

Do ping and traceroute depend on CONFIG_XFRM_USER=y?
That doesn't seem right.  

Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set 
> + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create 
> rpc_clnt_destroy \
> +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create 
> + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror 
> + rpc_clnt_sperrno
> rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth 
> + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc 
> rpc_clnt_freeres \
> +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" 
> + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect \
> +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg 
> + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> test_1_to_1_sendto test_1_to_1_shutdown test_1_to_1_socket_bind_listen 
> \
> +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort 
> + test_assoc_shutdown test_autoclose test_basic test_basic_v6 
> + test_connect
> test_connectx test_fragments test_fragments_v6 test_getname \
> +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff 
> + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style \
> +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for 
> + net.sctp

A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "broken_ip4-version 
> + broken_ip4-ihl broken_ip4-totlen broken_ip4-fragment
> broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr 
> broken_ip6-dstaddr \
> +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change 
> + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> +    if6-addr-addlarge if6-route-adddel if6-route-addlarge 
> + if6-mtu-change" # for net_stress.interface

Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01 
> + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 dccp4_ipsec10 
> \
> +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14 
> + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 dccp4_ipsec23 
> \
> +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27 
> + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 dccp4_ipsec36 
> \
> +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04 
> + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 dccp6_ipsec13 
> \
> +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17 
> + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 dccp6_ipsec26 
> \
> +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30 
> + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp

This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01 
> + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11 
> + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14 
> + icmp4-uni-basic15
> icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21 
> + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24 
> + icmp4-uni-basic25
> icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05 
> + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08 
> + icmp6-uni-basic09
> icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15 
> + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18 
> + icmp6-uni-basic19
> icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01 
> + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 tcp4_ipsec16 
> + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 tcp4_ipsec30 
> + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 tcp6_ipsec08 
> + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 tcp6_ipsec22 
> + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01 
> + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10 
> udp4_ipsec11 \
> +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 udp4_ipsec16 
> + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 udp4_ipsec30 
> + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 udp6_ipsec08 
> + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 udp6_ipsec22 
> + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01 
> + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 sctp4_ipsec10 
> \
> +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14 
> + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18 
> + sctp4_ipsec19
> sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27 
> + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31 
> + sctp4_ipsec32
> sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04 
> + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08 
> + sctp6_ipsec09
> sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17 
> + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21 
> + sctp6_ipsec22
> sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30 
> + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34 
> + sctp6_ipsec35
> sctp6_ipsec36" # for net_stress.ipsec_sctp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" 
> + "mcast4-group-single-socket mcast4-group-multiple-socket 
> + mcast4-group-same-group
> mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03 
> + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 mcast6-group-
> single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02 
> + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for 
> net_stress.multicast

A lot of these lines are very long, and the dependency seems a bit sketch.

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tirpc_rpcb_getaddr 
> + tirpc_rpcb_getmaps tirpc_authdes_create
> tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror 
> + tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset 
> + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call 
> + tirpc_clnt_tp_create
> tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg 
> + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp 
> + tirpc_clnt_create
> tirpc_clnt_create_timed tirpc_svc_create \
> +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" # 
> + for net.tirpc_tests
> +
>      echo "Tests skipped depending on the architecture"
>      if [ "$ARCHITECTURE" != "s390" ]; then
>          skip_tests "sbrk03" # Only works in 32bit on s390 series 
> system
> --
> 2.11.0

I have too many questions about this one.  I did NOT apply it.
 -- Tim

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Qiu, Tingting]

[-- Attachment #1: Type: text/plain, Size: 16998 bytes --]

Hi,Tim

> This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?

We can add wildcard processing in fuego_test.sh, but this is only used in some cases, such as using
skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "^icmp4-uni* ^icmp6-uni*" 
instead of
skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01 icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \ 
    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11 icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14 icmp4-uni-basic15 icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \ 
    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21 icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24 icmp4-uni-basic25 icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \ 
    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05 icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08 icmp6-uni-basic09 icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \ 
    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15 icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18 icmp6-uni-basic19 icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \ 
    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
But there are still some cases(preliminary estimate is at least 80+), the code is not concise after using wildcards. Becase the name gap is very small (some cases need to be tested, but some need skip) 

Therefore, we suggest whether it is possible to write all the cases that depend on "CONFIG_XFRM_USER=y" into a CONFIG_XFRM_USER_skiplist.txt (naming rule: ${config name}_skiplist.txt), and add the contents to skiplist.txt during run test. 
I think this can achieve requirements of skip case, and concise code.
See patch : 0006-skip-some-cases-because-ONFIG_XFRM_USER-is-not-set.patch

Thanks,
 -- QiuTingting

-----Original Message-----
From: Fuego [mailto:fuego-bounces@lists.linuxfoundation.org] On Behalf Of Qiu, Tingting
Sent: Thursday, July 30, 2020 1:49 PM
To: Bird, Tim <Tim.Bird@sony.com>
Cc: fuego@lists.linuxfoundation.org
Subject: Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

Hi,Tim

Cases such as ping, tcp, udp and traceroute are all based on the virtual network card. 
Initializing the virtual network card is implemented by the common script(test_net.sh). 
The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
Therefore, all cases that use virtualized network cards will be affected in the same way. 
It is reasonable to use the "CONFIG_XFRM_USER=y" to check.

The following is my guess:
There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
when initializing the virtualized network card, using "ip xfrm" is necessary, because many ipsec-related content need to be tested in the network test set (at least 504 cases).

*1:
tst_init_iface()
{
         local type="${1:-lhost}"
         local link_num="${2:-0}"
         local iface="$(tst_iface $type $link_num)"
         tst_resm TINFO "initialize '$type' '$iface' interface"

         if [ "$type" = "lhost" ]; then
                 ip xfrm policy flush || return $?
                 ip xfrm state flush || return $?
                 ip link set $iface down || return $?
                 ip route flush dev $iface || return $?
                 ip addr flush dev $iface || return $?
                 ip link set $iface up
                 return $?
         fi

         tst_rhost_run -c "ip xfrm policy flush" || return $?
         tst_rhost_run -c "ip xfrm state flush" || return $?
         tst_rhost_run -c "ip link set $iface down" || return $?
         tst_rhost_run -c "ip route flush dev $iface" || return $?
         tst_rhost_run -c "ip addr flush dev $iface" || return $?
         tst_rhost_run -c "ip link set $iface up"
}

Thanks,
 -- QiuTingting

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com]
Sent: Thursday, July 9, 2020 6:13 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

This one doesn't look right. See comments below.

> -----Original Message-----
> From: qiutt
> 
> for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc 
> net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip 
> net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp 
> net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp 
> net_stress.multicast net.tcp_cmds net.tirpc_tests
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 59
> ++++++++++++++++++++++++++++++++++++++
>  1 file changed, 59 insertions(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh
> b/tests/Functional.LTP/fuego_test.sh
> index 156fd39..ab4165d 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -251,6 +251,65 @@ function test_pre_check {
>      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
>      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> 
> +    # some CONFIGs do not set in m3ulcb
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01 vlan03"
> # for net.features
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602
> + sendfile601 traceroute601 dhcpd6" # for net.ipv6

Do ping and traceroute depend on CONFIG_XFRM_USER=y?
That doesn't seem right.  

Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set 
> + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create 
> rpc_clnt_destroy \
> +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create 
> + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror 
> + rpc_clnt_sperrno
> rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth 
> + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc 
> rpc_clnt_freeres \
> +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" 
> + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect \
> +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg 
> + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> test_1_to_1_sendto test_1_to_1_shutdown test_1_to_1_socket_bind_listen 
> \
> +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort 
> + test_assoc_shutdown test_autoclose test_basic test_basic_v6 
> + test_connect
> test_connectx test_fragments test_fragments_v6 test_getname \
> +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff
> + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style \
> +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for 
> + net.sctp

A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "broken_ip4-version 
> + broken_ip4-ihl broken_ip4-totlen broken_ip4-fragment
> broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr 
> broken_ip6-dstaddr \
> +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change 
> + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> +    if6-addr-addlarge if6-route-adddel if6-route-addlarge 
> + if6-mtu-change" # for net_stress.interface

Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01
> + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 dccp4_ipsec10 
> \
> +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14
> + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 dccp4_ipsec23 
> \
> +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27
> + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 dccp4_ipsec36 
> \
> +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04
> + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 dccp6_ipsec13 
> \
> +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17
> + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 dccp6_ipsec26 
> \
> +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30
> + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp

This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01 
> + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11 
> + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14 
> + icmp4-uni-basic15
> icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21 
> + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24 
> + icmp4-uni-basic25
> icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05 
> + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08 
> + icmp6-uni-basic09
> icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15 
> + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18 
> + icmp6-uni-basic19
> icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01 
> + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 tcp4_ipsec16 
> + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 tcp4_ipsec30 
> + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 tcp6_ipsec08 
> + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 tcp6_ipsec22 
> + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01 
> + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10 
> udp4_ipsec11 \
> +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 udp4_ipsec16 
> + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 udp4_ipsec30 
> + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 udp6_ipsec08 
> + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 udp6_ipsec22 
> + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01 
> + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 sctp4_ipsec10 
> \
> +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14 
> + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18 
> + sctp4_ipsec19
> sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27 
> + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31 
> + sctp4_ipsec32
> sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04 
> + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08 
> + sctp6_ipsec09
> sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17 
> + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21 
> + sctp6_ipsec22
> sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30 
> + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34 
> + sctp6_ipsec35
> sctp6_ipsec36" # for net_stress.ipsec_sctp
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" 
> + "mcast4-group-single-socket mcast4-group-multiple-socket 
> + mcast4-group-same-group
> mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03 
> + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 mcast6-group-
> single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02 
> + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for 
> net_stress.multicast

A lot of these lines are very long, and the dependency seems a bit sketch.

> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tirpc_rpcb_getaddr 
> + tirpc_rpcb_getmaps tirpc_authdes_create
> tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror 
> + tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset 
> + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call 
> + tirpc_clnt_tp_create
> tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg 
> + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp 
> + tirpc_clnt_create
> tirpc_clnt_create_timed tirpc_svc_create \
> +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" # 
> + for net.tirpc_tests
> +
>      echo "Tests skipped depending on the architecture"
>      if [ "$ARCHITECTURE" != "s390" ]; then
>          skip_tests "sbrk03" # Only works in 32bit on s390 series 
> system
> --
> 2.11.0

I have too many questions about this one.  I did NOT apply it.
 -- Tim





_______________________________________________
Fuego mailing list
Fuego@lists.linuxfoundation.org
https://lists.linuxfoundation.org/mailman/listinfo/fuego



[-- Attachment #2: 0006-skip-some-cases-because-ONFIG_XFRM_USER-is-not-set.patch --]
[-- Type: application/octet-stream, Size: 11271 bytes --]

From 10a05fb5bd0cda19a3af1ed6dc59e16a2c07d4e8 Mon Sep 17 00:00:00 2001
From: qiutt <qiutt@cn.fujitsu.com>
Date: Thu, 30 Jul 2020 14:14:48 +0800
Subject: [PATCH 1/2] skip some cases because ONFIG_XFRM_USER is not set

add a CONFIG_XFRM_USER_skiplist.txt, including all cases need to skip
if CONFIG_XFRM_USER is not equall to y.

for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc
net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip
net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp
net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp
net_stress.multicast net.tcp_cmds net.tirpc_tests

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/CONFIG_XFRM_USER_skiplist.txt | 541 +++++++++++++++++++++
 tests/Functional.LTP/fuego_test.sh                 |  18 +
 2 files changed, 559 insertions(+)
 create mode 100644 tests/Functional.LTP/CONFIG_XFRM_USER_skiplist.txt

diff --git a/tests/Functional.LTP/CONFIG_XFRM_USER_skiplist.txt b/tests/Functional.LTP/CONFIG_XFRM_USER_skiplist.txt
new file mode 100644
index 0000000..5524205
--- /dev/null
+++ b/tests/Functional.LTP/CONFIG_XFRM_USER_skiplist.txt
@@ -0,0 +1,541 @@
+dccp01
+dccp01_ipv6
+sctp01
+sctp01_ipv6
+tcp_fastopen
+tcp_fastopen6
+vlan01
+vlan03
+
+ping601
+ping602
+sendfile601
+traceroute601
+dhcpd6
+
+mc_cmds
+mc_commo
+mc_member
+mc_opts
+
+rpc01
+
+rpc_pmap_set
+rpc_pmap_unset
+rpc_pmap_getport
+rpc_pmap_getmaps
+rpc_pmap_rmtcall
+rpc_clnt_broadcast
+rpc_svc_destroy
+rpc_svcfd_create
+rpc_clnt_destroy
+rpc_clnt_create
+rpc_clnttcp_create
+rpc_clntudp_create
+rpc_clntudp_bufcreate
+rpc_clnt_perrno
+rpc_clnt_perror
+rpc_clnt_sperrno
+rpc_clnt_sperror
+rpc_svcerr_noproc
+rpc_svcerr_noprog
+rpc_svcerr_progvers
+rpc_svcerr_systemerr
+rpc_svcerr_auth
+rpc_svcerr_weakauth
+rpc_xprt_register
+rpc_xprt_unregister
+rpc_svc_register
+rpc_registerrpc
+rpc_clnt_call
+rpc_callrpc
+rpc_clnt_freeres
+rpc_clnt_geterr
+rpc_clnt_control
+rpc_svc_getcaller
+rpc_svc_getargs
+rpc_svc_sendreply
+
+test_1_to_1_accept_close
+test_1_to_1_addrs
+test_1_to_1_connect
+test_1_to_1_connectx
+test_1_to_1_events
+test_1_to_1_initmsg_connect
+test_1_to_1_nonblock
+test_1_to_1_recvfrom
+test_1_to_1_recvmsg
+test_1_to_1_rtoinfo
+test_1_to_1_send
+test_1_to_1_sendmsg
+test_1_to_1_sendto
+test_1_to_1_shutdown
+test_1_to_1_socket_bind_listen
+test_1_to_1_sockopt
+test_1_to_1_threads
+test_assoc_abort
+test_assoc_shutdown
+test_autoclose
+test_basic
+test_basic_v6
+test_connect
+test_connectx
+test_fragments
+test_fragments_v6
+test_getname
+test_getname_v6
+test_inaddr_any
+test_inaddr_any_v6
+test_peeloff
+test_peeloff_v6
+test_recvmsg
+test_sctp_sendrecvmsg
+test_sctp_sendrecvmsg_v6
+test_sockopt
+test_sockopt_v6
+test_tcp_style
+test_tcp_style_v6
+test_timetolive
+test_timetolive_v6
+
+ssh4-stress
+ssh6-stress
+
+broken_ip4-version
+broken_ip4-ihl
+broken_ip4-totlen
+broken_ip4-fragment
+broken_ip4-protcol
+broken_ip4-checksum
+broken_ip4-dstaddr
+broken_ip6-dstaddr
+broken_ip6-nexthdr
+broken_ip6-plen
+broken_ip6-version
+
+if4-addr-change
+if4-updown
+if4-addr-adddel
+if4-addr-addlarge
+if4-route-adddel
+if4-route-addlarge
+if4-mtu-change
+if6-updown
+if6-addr-adddel
+if6-addr-addlarge
+if6-route-adddel
+if6-route-addlarge
+if6-mtu-change
+
+dccp4_ipsec01
+dccp4_ipsec02
+dccp4_ipsec03
+dccp4_ipsec04
+dccp4_ipsec05
+dccp4_ipsec06
+dccp4_ipsec07
+dccp4_ipsec08
+dccp4_ipsec09
+dccp4_ipsec10
+dccp4_ipsec11
+dccp4_ipsec12
+dccp4_ipsec13
+dccp4_ipsec14
+dccp4_ipsec15
+dccp4_ipsec16
+dccp4_ipsec17
+dccp4_ipsec18
+dccp4_ipsec19
+dccp4_ipsec20
+dccp4_ipsec21
+dccp4_ipsec22
+dccp4_ipsec23
+dccp4_ipsec24
+dccp4_ipsec25
+dccp4_ipsec26
+dccp4_ipsec27
+dccp4_ipsec28
+dccp4_ipsec29
+dccp4_ipsec30
+dccp4_ipsec31
+dccp4_ipsec32
+dccp4_ipsec33
+dccp4_ipsec34
+dccp4_ipsec35
+dccp4_ipsec36
+dccp6_ipsec01
+dccp6_ipsec02
+dccp6_ipsec03
+dccp6_ipsec04
+dccp6_ipsec05
+dccp6_ipsec06
+dccp6_ipsec07
+dccp6_ipsec08
+dccp6_ipsec09
+dccp6_ipsec10
+dccp6_ipsec11
+dccp6_ipsec12
+dccp6_ipsec13
+dccp6_ipsec14
+dccp6_ipsec15
+dccp6_ipsec16
+dccp6_ipsec17
+dccp6_ipsec18
+dccp6_ipsec19
+dccp6_ipsec20
+dccp6_ipsec21
+dccp6_ipsec22
+dccp6_ipsec23
+dccp6_ipsec24
+dccp6_ipsec25
+dccp6_ipsec26
+dccp6_ipsec27
+dccp6_ipsec28
+dccp6_ipsec29
+dccp6_ipsec30
+dccp6_ipsec31
+dccp6_ipsec32
+dccp6_ipsec33
+dccp6_ipsec34
+dccp6_ipsec35
+dccp6_ipsec36
+
+icmp4-uni-basic01
+icmp4-uni-basic02
+icmp4-uni-basic03
+icmp4-uni-basic04
+icmp4-uni-basic05
+icmp4-uni-basic06
+icmp4-uni-basic07
+icmp4-uni-basic08
+icmp4-uni-basic09
+icmp4-uni-basic10
+icmp4-uni-basic11
+icmp4-uni-basic12
+icmp4-uni-basic13
+icmp4-uni-basic14
+icmp4-uni-basic15
+icmp4-uni-basic16
+icmp4-uni-basic17
+icmp4-uni-basic18
+icmp4-uni-basic19
+icmp4-uni-basic20
+icmp4-uni-basic21
+icmp4-uni-basic22
+icmp4-uni-basic23
+icmp4-uni-basic24
+icmp4-uni-basic25
+icmp4-uni-basic26
+icmp6-uni-basic01
+icmp6-uni-basic02
+icmp6-uni-basic03
+icmp6-uni-basic04
+icmp6-uni-basic05
+icmp6-uni-basic06
+icmp6-uni-basic07
+icmp6-uni-basic08
+icmp6-uni-basic09
+icmp6-uni-basic10
+icmp6-uni-basic11
+icmp6-uni-basic12
+icmp6-uni-basic13
+icmp6-uni-basic14
+icmp6-uni-basic15
+icmp6-uni-basic16
+icmp6-uni-basic17
+icmp6-uni-basic18
+icmp6-uni-basic19
+icmp6-uni-basic20
+icmp6-uni-basic21
+icmp6-uni-basic22
+icmp6-uni-basic23
+icmp6-uni-basic24
+icmp6-uni-basic25
+icmp6-uni-basic26
+
+tcp4_ipsec01
+tcp4_ipsec02
+tcp4_ipsec03
+tcp4_ipsec04
+tcp4_ipsec05
+tcp4_ipsec06
+tcp4_ipsec07
+tcp4_ipsec08
+tcp4_ipsec09
+tcp4_ipsec10
+tcp4_ipsec11
+tcp4_ipsec12
+tcp4_ipsec13
+tcp4_ipsec14
+tcp4_ipsec15
+tcp4_ipsec16
+tcp4_ipsec17
+tcp4_ipsec18
+tcp4_ipsec19
+tcp4_ipsec20
+tcp4_ipsec21
+tcp4_ipsec22
+tcp4_ipsec23
+tcp4_ipsec24
+tcp4_ipsec25
+tcp4_ipsec26
+tcp4_ipsec27
+tcp4_ipsec28
+tcp4_ipsec29
+tcp4_ipsec30
+tcp4_ipsec31
+tcp4_ipsec32
+tcp4_ipsec33
+tcp4_ipsec34
+tcp4_ipsec35
+tcp4_ipsec36
+tcp6_ipsec01
+tcp6_ipsec02
+tcp6_ipsec03
+tcp6_ipsec04
+tcp6_ipsec05
+tcp6_ipsec06
+tcp6_ipsec07
+tcp6_ipsec08
+tcp6_ipsec09
+tcp6_ipsec10
+tcp6_ipsec11
+tcp6_ipsec12
+tcp6_ipsec13
+tcp6_ipsec14
+tcp6_ipsec15
+tcp6_ipsec16
+tcp6_ipsec17
+tcp6_ipsec18
+tcp6_ipsec19
+tcp6_ipsec20
+tcp6_ipsec21
+tcp6_ipsec22
+tcp6_ipsec23
+tcp6_ipsec24
+tcp6_ipsec25
+tcp6_ipsec26
+tcp6_ipsec27
+tcp6_ipsec28
+tcp6_ipsec29
+tcp6_ipsec30
+tcp6_ipsec31
+tcp6_ipsec32
+tcp6_ipsec33
+tcp6_ipsec34
+tcp6_ipsec35
+tcp6_ipsec36
+
+udp4_ipsec01
+udp4_ipsec02
+udp4_ipsec03
+udp4_ipsec04
+udp4_ipsec05
+udp4_ipsec06
+udp4_ipsec07
+udp4_ipsec08
+udp4_ipsec09
+udp4_ipsec10
+udp4_ipsec11
+udp4_ipsec12
+udp4_ipsec13
+udp4_ipsec14
+udp4_ipsec15
+udp4_ipsec16
+udp4_ipsec17
+udp4_ipsec18
+udp4_ipsec19
+udp4_ipsec20
+udp4_ipsec21
+udp4_ipsec22
+udp4_ipsec23
+udp4_ipsec24
+udp4_ipsec25
+udp4_ipsec26
+udp4_ipsec27
+udp4_ipsec28
+udp4_ipsec29
+udp4_ipsec30
+udp4_ipsec31
+udp4_ipsec32
+udp4_ipsec33
+udp4_ipsec34
+udp4_ipsec35
+udp4_ipsec36
+udp6_ipsec01
+udp6_ipsec02
+udp6_ipsec03
+udp6_ipsec04
+udp6_ipsec05
+udp6_ipsec06
+udp6_ipsec07
+udp6_ipsec08
+udp6_ipsec09
+udp6_ipsec10
+udp6_ipsec11
+udp6_ipsec12
+udp6_ipsec13
+udp6_ipsec14
+udp6_ipsec15
+udp6_ipsec16
+udp6_ipsec17
+udp6_ipsec18
+udp6_ipsec19
+udp6_ipsec20
+udp6_ipsec21
+udp6_ipsec22
+udp6_ipsec23
+udp6_ipsec24
+udp6_ipsec25
+udp6_ipsec26
+udp6_ipsec27
+udp6_ipsec28
+udp6_ipsec29
+udp6_ipsec30
+udp6_ipsec31
+udp6_ipsec32
+udp6_ipsec33
+udp6_ipsec34
+udp6_ipsec35
+udp6_ipsec36
+
+sctp4_ipsec01
+sctp4_ipsec02
+sctp4_ipsec03
+sctp4_ipsec04
+sctp4_ipsec05
+sctp4_ipsec06
+sctp4_ipsec07
+sctp4_ipsec08
+sctp4_ipsec09
+sctp4_ipsec10
+sctp4_ipsec11
+sctp4_ipsec12
+sctp4_ipsec13
+sctp4_ipsec14
+sctp4_ipsec15
+sctp4_ipsec16
+sctp4_ipsec17
+sctp4_ipsec18
+sctp4_ipsec19
+sctp4_ipsec20
+sctp4_ipsec21
+sctp4_ipsec22
+sctp4_ipsec23
+sctp4_ipsec24
+sctp4_ipsec25
+sctp4_ipsec26
+sctp4_ipsec27
+sctp4_ipsec28
+sctp4_ipsec29
+sctp4_ipsec30
+sctp4_ipsec31
+sctp4_ipsec32
+sctp4_ipsec33
+sctp4_ipsec34
+sctp4_ipsec35
+sctp4_ipsec36
+sctp6_ipsec01
+sctp6_ipsec02
+sctp6_ipsec03
+sctp6_ipsec04
+sctp6_ipsec05
+sctp6_ipsec06
+sctp6_ipsec07
+sctp6_ipsec08
+sctp6_ipsec09
+sctp6_ipsec10
+sctp6_ipsec11
+sctp6_ipsec12
+sctp6_ipsec13
+sctp6_ipsec14
+sctp6_ipsec15
+sctp6_ipsec16
+sctp6_ipsec17
+sctp6_ipsec18
+sctp6_ipsec19
+sctp6_ipsec20
+sctp6_ipsec21
+sctp6_ipsec22
+sctp6_ipsec23
+sctp6_ipsec24
+sctp6_ipsec25
+sctp6_ipsec26
+sctp6_ipsec27
+sctp6_ipsec28
+sctp6_ipsec29
+sctp6_ipsec30
+sctp6_ipsec31
+sctp6_ipsec32
+sctp6_ipsec33
+sctp6_ipsec34
+sctp6_ipsec35
+sctp6_ipsec36
+
+mcast4-group-single-socket
+mcast4-group-multiple-socket
+mcast4-group-same-group
+mcast4-group-source-filter
+mcast4-pktfld01
+mcast4-pktfld02
+mcast4-queryfld01
+mcast4-queryfld02
+mcast4-queryfld03
+mcast4-queryfld04
+mcast4-queryfld05
+mcast4-queryfld06
+mcast6-group-single-socket
+mcast6-group-multiple-socket
+mcast6-group-same-group
+mcast6-group-source-filter
+mcast6-pktfld01
+mcast6-pktfld02
+mcast6-queryfld01
+mcast6-queryfld02
+mcast6-queryfld03
+mcast6-queryfld04
+mcast6-queryfld05
+mcast6-queryfld06
+
+ping01
+ping02
+sendfile
+iproute
+traceroute01
+dhcpd
+
+tirpc_rpcb_getaddr
+tirpc_rpcb_getmaps
+tirpc_authdes_create
+tirpc_authdes_seccreate
+tirpc_clnt_dg_create
+tirpc_clnt_vc_create
+tirpc_bottomlevel_clnt_call
+tirpc_clnt_perrno
+tirpc_clnt_perror
+tirpc_svcerr_noproc
+tirpc_svcerr_noprog
+tirpc_svcerr_progvers
+tirpc_svcerr_systemerr
+tirpc_svcerr_weakauth
+tirpc_clnt_tli_create
+tirpc_rpcb_set
+tirpc_rpcb_unset
+tirpc_rpcb_rmtcall
+tirpc_svc_reg
+tirpc_expertlevel_clnt_call
+tirpc_clnt_tp_create
+tirpc_clnt_tp_create_timed
+tirpc_svc_tp_create
+tirpc_interlevel_clnt_call
+tirpc_clnt_control
+tirpc_rpc_reg
+tirpc_rpc_call
+tirpc_rpc_broadcast
+tirpc_rpc_broadcast_exp
+tirpc_clnt_create
+tirpc_clnt_create_timed
+tirpc_svc_create
+tirpc_toplevel_clnt_call
+tirpc_clnt_destroy
+tirpc_svc_destroy
diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 02fb250..88dea7e 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -152,6 +152,21 @@ function skip_if_kconfig_differs {
     done
 }
 
+# $1: string with kernel config option (only support one config option)
+# $2: file name that contains skip case list
+function skip_tests_with_config_skiplist {
+    local KCONFIG="$1"
+    local CONFIG_SKIPLIST_FILE=${TEST_HOME}/"$2"
+    if [ ! -f $CONFIG_SKIPLIST_FILE ]; then
+        echo "No $CONFIG_SKIPLIST_FILE"
+        return 0
+    fi
+    if ! check_kconfig "$KCONFIG"; then
+        echo "CONFIG_SKIPLIST_FILE:$CONFIG_SKIPLIST_FILE"
+        cat $CONFIG_SKIPLIST_FILE >> ${LOGDIR}/skiplist.txt
+    fi
+}
+
 function test_pre_check {
     if [ -n "${FUNCTIONAL_LTP_HOMEDIR}" ] ; then
         # user or spec has specified a home directory for LTP.
@@ -251,6 +266,9 @@ function test_pre_check {
     skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
     skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
 
+    echo "Tests skipped depending on the config skiplist file"
+    skip_tests_with_config_skiplist "CONFIG_XFRM_USER=y" "CONFIG_XFRM_USER_skiplist.txt"
+
     echo "Tests skipped depending on the architecture"
     if [ "$ARCHITECTURE" != "s390" ]; then
         skip_tests "sbrk03" # Only works in 32bit on s390 series system
-- 
2.11.0

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Bird, Tim]

> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> 
> Hi,Tim
> 
> > This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> 
> We can add wildcard processing in fuego_test.sh, but this is only used in some cases, such as using
> skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "^icmp4-uni* ^icmp6-uni*"
> instead of
> skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01 icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-uni-
> basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
>     icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11 icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14 icmp4-uni-basic15
> icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
>     icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21 icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24 icmp4-uni-basic25
> icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
>     icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05 icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08 icmp6-uni-basic09
> icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
>     icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15 icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18 icmp6-uni-basic19
> icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
>     icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> But there are still some cases(preliminary estimate is at least 80+), the code is not concise after using wildcards. Becase the name gap is
> very small (some cases need to be tested, but some need skip)
> 
> Therefore, we suggest whether it is possible to write all the cases that depend on "CONFIG_XFRM_USER=y" into a
> CONFIG_XFRM_USER_skiplist.txt (naming rule: ${config name}_skiplist.txt), and add the contents to skiplist.txt during run test.
> I think this can achieve requirements of skip case, and concise code.
> See patch : 0006-skip-some-cases-because-ONFIG_XFRM_USER-is-not-set.patch

The approach is intriguing, but seems a bit complicated.  Also, with this particular dependency,
based on my experimentation with dccp01 I don't trust that all these tests are actually
dependent on CONFIG_XFRM_USER.

However, that issue is separate from the issue of how to handle large numbers of dependencies.

With regards to using wildcards, 
wildcard dependencies only work at the level of fuego_test.sh if Fuego can expand the
wildcard to come up with the list of items to match - which are individual testcase tags
specified in LTP's runtest files (also called 'scenario' files or command files).  I'd rather
not have LTP have to go parsing the files in that directory to process wildcards.

Passing around strings with wildcard patterns in them is quite problematical in shell
code, so I don't think it's feasible to try to keep the regexes with wildcards in the
skiplist itself (to pass through eventually to runltp), and with that solution, runltp
would need to be modified to support wildcards.  Maybe it would work, but it seems
error prone.

So, for now, I think I'd like to withdraw my wildcard suggestion.

Putting the lists into a file seems like a good idea.  I don't like them cluttering up
the fuego_test.sh code with huge lists.

I'm not sure I want to fully generalize this solution yet.  If we kept this list of
files dependent on a config, I think I'd rather do something like modifying
skip_if_kconfig_differs as follows:

function skip_if_kconfig_differs {
    set -f
    local KCONFIGS=($1)
    set +f
    local TESTS="$2"

    for cfg in ${KCONFIGS[@]}; do
        if ! check_kconfig "$cfg"; then
            for item in "${TESTS}"; do
                if [ -f "$item" ]; then
                    cat "$item" >> ${LOGDIR}/skiplist.txt
               else
                    echo "$item" >> ${LOGDIR}/skiplist.txt
               fi
           done
        fi
    done
}

And then call it with:

skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "$TEST_HOME/CONFIG_XFRM_USER_skiplist.txt"

 -- Tim

> 
> Thanks,
>  -- QiuTingting
> 
> -----Original Message-----
> From: Fuego [mailto:fuego-bounces@lists.linuxfoundation.org] On Behalf Of Qiu, Tingting
> Sent: Thursday, July 30, 2020 1:49 PM
> To: Bird, Tim <Tim.Bird@sony.com>
> Cc: fuego@lists.linuxfoundation.org
> Subject: Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set
> 
> Hi,Tim
> 
> Cases such as ping, tcp, udp and traceroute are all based on the virtual network card.
> Initializing the virtual network card is implemented by the common script(test_net.sh).
> The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
> Therefore, all cases that use virtualized network cards will be affected in the same way.
> It is reasonable to use the "CONFIG_XFRM_USER=y" to check.
> 
> The following is my guess:
> There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
> when initializing the virtualized network card, using "ip xfrm" is necessary, because many ipsec-related content need to be tested in the
> network test set (at least 504 cases).
> 
> *1:
> tst_init_iface()
> {
>          local type="${1:-lhost}"
>          local link_num="${2:-0}"
>          local iface="$(tst_iface $type $link_num)"
>          tst_resm TINFO "initialize '$type' '$iface' interface"
> 
>          if [ "$type" = "lhost" ]; then
>                  ip xfrm policy flush || return $?
>                  ip xfrm state flush || return $?
>                  ip link set $iface down || return $?
>                  ip route flush dev $iface || return $?
>                  ip addr flush dev $iface || return $?
>                  ip link set $iface up
>                  return $?
>          fi
> 
>          tst_rhost_run -c "ip xfrm policy flush" || return $?
>          tst_rhost_run -c "ip xfrm state flush" || return $?
>          tst_rhost_run -c "ip link set $iface down" || return $?
>          tst_rhost_run -c "ip route flush dev $iface" || return $?
>          tst_rhost_run -c "ip addr flush dev $iface" || return $?
>          tst_rhost_run -c "ip link set $iface up"
> }
> 
> Thanks,
>  -- QiuTingting
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Thursday, July 9, 2020 6:13 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set
> 
> This one doesn't look right. See comments below.
> 
> > -----Original Message-----
> > From: qiutt
> >
> > for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc
> > net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip
> > net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp
> > net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp
> > net_stress.multicast net.tcp_cmds net.tirpc_tests
> >
> > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > ---
> >  tests/Functional.LTP/fuego_test.sh | 59
> > ++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 59 insertions(+)
> >
> > diff --git a/tests/Functional.LTP/fuego_test.sh
> > b/tests/Functional.LTP/fuego_test.sh
> > index 156fd39..ab4165d 100755
> > --- a/tests/Functional.LTP/fuego_test.sh
> > +++ b/tests/Functional.LTP/fuego_test.sh
> > @@ -251,6 +251,65 @@ function test_pre_check {
> >      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
> >      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> >
> > +    # some CONFIGs do not set in m3ulcb
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01
> vlan03"
> > # for net.features
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602
> > + sendfile601 traceroute601 dhcpd6" # for net.ipv6
> 
> Do ping and traceroute depend on CONFIG_XFRM_USER=y?
> That doesn't seem right.
> 
> Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set
> > + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> > rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create
> > rpc_clnt_destroy \
> > +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create
> > + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror
> > + rpc_clnt_sperrno
> > rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> > +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth
> > + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> > rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc
> > rpc_clnt_freeres \
> > +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> > test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect \
> > +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg
> > + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> > test_1_to_1_sendto test_1_to_1_shutdown test_1_to_1_socket_bind_listen
> > \
> > +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort
> > + test_assoc_shutdown test_autoclose test_basic test_basic_v6
> > + test_connect
> > test_connectx test_fragments test_fragments_v6 test_getname \
> > +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff
> > + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> > test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style \
> > +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for
> > + net.sctp
> 
> A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
> Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "broken_ip4-version
> > + broken_ip4-ihl broken_ip4-totlen broken_ip4-fragment
> > broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr
> > broken_ip6-dstaddr \
> > +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change
> > + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> > route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> > +    if6-addr-addlarge if6-route-adddel if6-route-addlarge
> > + if6-mtu-change" # for net_stress.interface
> 
> Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01
> > + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> > dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 dccp4_ipsec10
> > \
> > +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14
> > + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> > dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 dccp4_ipsec23
> > \
> > +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27
> > + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> > dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 dccp4_ipsec36
> > \
> > +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04
> > + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> > dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 dccp6_ipsec13
> > \
> > +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17
> > + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> > dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 dccp6_ipsec26
> > \
> > +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30
> > + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> > dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp
> 
> This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01
> > + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> > uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> > +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > + icmp4-uni-basic15
> > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> > +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > + icmp4-uni-basic25
> > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> > +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > + icmp6-uni-basic09
> > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> > +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > + icmp6-uni-basic19
> > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> > +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01
> > + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> > tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> > +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 tcp4_ipsec16
> > + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> > tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> > +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 tcp4_ipsec30
> > + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> > tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> > +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 tcp6_ipsec08
> > + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> > tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> > +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 tcp6_ipsec22
> > + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> > tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> > +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01
> > + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> > udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10
> > udp4_ipsec11 \
> > +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 udp4_ipsec16
> > + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> > udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> > +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 udp4_ipsec30
> > + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> > udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> > +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 udp6_ipsec08
> > + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> > udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> > +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 udp6_ipsec22
> > + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> > udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> > +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01
> > + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> > sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 sctp4_ipsec10
> > \
> > +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14
> > + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18
> > + sctp4_ipsec19
> > sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> > +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27
> > + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31
> > + sctp4_ipsec32
> > sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> > +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04
> > + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08
> > + sctp6_ipsec09
> > sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> > +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17
> > + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21
> > + sctp6_ipsec22
> > sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> > +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30
> > + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34
> > + sctp6_ipsec35
> > sctp6_ipsec36" # for net_stress.ipsec_sctp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "mcast4-group-single-socket mcast4-group-multiple-socket
> > + mcast4-group-same-group
> > mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> > +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03
> > + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 mcast6-group-
> > single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> > +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02
> > + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> > queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for
> > net_stress.multicast
> 
> A lot of these lines are very long, and the dependency seems a bit sketch.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tirpc_rpcb_getaddr
> > + tirpc_rpcb_getmaps tirpc_authdes_create
> > tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> > +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror
> > + tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> > tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> > +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset
> > + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call
> > + tirpc_clnt_tp_create
> > tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> > +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg
> > + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp
> > + tirpc_clnt_create
> > tirpc_clnt_create_timed tirpc_svc_create \
> > +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" #
> > + for net.tirpc_tests
> > +
> >      echo "Tests skipped depending on the architecture"
> >      if [ "$ARCHITECTURE" != "s390" ]; then
> >          skip_tests "sbrk03" # Only works in 32bit on s390 series
> > system
> > --
> > 2.11.0
> 
> I have too many questions about this one.  I did NOT apply it.
>  -- Tim
> 
> 
> 
> 
> 
> _______________________________________________
> Fuego mailing list
> Fuego@lists.linuxfoundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/fuego
> 

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Qiu, Tingting]

[-- Attachment #1: Type: text/plain, Size: 21011 bytes --]

Hi,Tim

Thanks for your reply.
We modified tst_init_iface() refer to 20200515-164-ga6c8680, and then retested. 
It took some time.

Some cases(such as ping601) are really unrelated to CONFIG_XFRM_USER(or ip xfrm).
So we add a file case-dependency-list-with-config.txt, to describe the relationship between testcase and the CONFIG, 
and add a function skip_tests_with_config_skiplist to add testcases to skiplist.txt. 
refer to 0001-skip-some-cases-based-on-skiplist-with-config.patch

As the same way, we add case-dependency-list-with-config.txt and a function skip_tests_with_cmd_skiplist, 
for the relationship between testcases and the cmds.
refer to 0002-skip-some-cases-based-on-skiplist-with-cmd.patch

If these patches is ok, we will commit them.

Thanks,
  -- QiuTingting

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Friday, August 7, 2020 8:03 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
Cc: fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> 
> Hi,Tim
> 
> > This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> 
> We can add wildcard processing in fuego_test.sh, but this is only used 
> in some cases, such as using skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "^icmp4-uni* ^icmp6-uni*"
> instead of
> skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01 
> icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-uni-
> basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
>     icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11 
> icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14 
> icmp4-uni-basic15
> icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
>     icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21 
> icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24 
> icmp4-uni-basic25
> icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
>     icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05 
> icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08 
> icmp6-uni-basic09
> icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
>     icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15 
> icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18 
> icmp6-uni-basic19
> icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
>     icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 
> icmp6-uni-basic26" # for net_stress.ipsec_icmp But there are still 
> some cases(preliminary estimate is at least 80+), the code is not 
> concise after using wildcards. Becase the name gap is very small (some 
> cases need to be tested, but some need skip)
> 
> Therefore, we suggest whether it is possible to write all the cases 
> that depend on "CONFIG_XFRM_USER=y" into a CONFIG_XFRM_USER_skiplist.txt (naming rule: ${config name}_skiplist.txt), and add the contents to skiplist.txt during run test.
> I think this can achieve requirements of skip case, and concise code.
> See patch : 
> 0006-skip-some-cases-because-ONFIG_XFRM_USER-is-not-set.patch

The approach is intriguing, but seems a bit complicated.  Also, with this particular dependency, based on my experimentation with dccp01 I don't trust that all these tests are actually dependent on CONFIG_XFRM_USER.

However, that issue is separate from the issue of how to handle large numbers of dependencies.

With regards to using wildcards,
wildcard dependencies only work at the level of fuego_test.sh if Fuego can expand the wildcard to come up with the list of items to match - which are individual testcase tags specified in LTP's runtest files (also called 'scenario' files or command files).  I'd rather not have LTP have to go parsing the files in that directory to process wildcards.

Passing around strings with wildcard patterns in them is quite problematical in shell code, so I don't think it's feasible to try to keep the regexes with wildcards in the skiplist itself (to pass through eventually to runltp), and with that solution, runltp would need to be modified to support wildcards.  Maybe it would work, but it seems error prone.

So, for now, I think I'd like to withdraw my wildcard suggestion.

Putting the lists into a file seems like a good idea.  I don't like them cluttering up the fuego_test.sh code with huge lists.

I'm not sure I want to fully generalize this solution yet.  If we kept this list of files dependent on a config, I think I'd rather do something like modifying skip_if_kconfig_differs as follows:

function skip_if_kconfig_differs {
    set -f
    local KCONFIGS=($1)
    set +f
    local TESTS="$2"

    for cfg in ${KCONFIGS[@]}; do
        if ! check_kconfig "$cfg"; then
            for item in "${TESTS}"; do
                if [ -f "$item" ]; then
                    cat "$item" >> ${LOGDIR}/skiplist.txt
               else
                    echo "$item" >> ${LOGDIR}/skiplist.txt
               fi
           done
        fi
    done
}

And then call it with:

skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "$TEST_HOME/CONFIG_XFRM_USER_skiplist.txt"

 -- Tim

> 
> Thanks,
>  -- QiuTingting
> 
> -----Original Message-----
> From: Fuego [mailto:fuego-bounces@lists.linuxfoundation.org] On Behalf 
> Of Qiu, Tingting
> Sent: Thursday, July 30, 2020 1:49 PM
> To: Bird, Tim <Tim.Bird@sony.com>
> Cc: fuego@lists.linuxfoundation.org
> Subject: Re: [Fuego] [PATCH 06/12] skip some cases because 
> ONFIG_XFRM_USER is not set
> 
> Hi,Tim
> 
> Cases such as ping, tcp, udp and traceroute are all based on the virtual network card.
> Initializing the virtual network card is implemented by the common script(test_net.sh).
> The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
> Therefore, all cases that use virtualized network cards will be affected in the same way.
> It is reasonable to use the "CONFIG_XFRM_USER=y" to check.
> 
> The following is my guess:
> There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
> when initializing the virtualized network card, using "ip xfrm" is 
> necessary, because many ipsec-related content need to be tested in the network test set (at least 504 cases).
> 
> *1:
> tst_init_iface()
> {
>          local type="${1:-lhost}"
>          local link_num="${2:-0}"
>          local iface="$(tst_iface $type $link_num)"
>          tst_resm TINFO "initialize '$type' '$iface' interface"
> 
>          if [ "$type" = "lhost" ]; then
>                  ip xfrm policy flush || return $?
>                  ip xfrm state flush || return $?
>                  ip link set $iface down || return $?
>                  ip route flush dev $iface || return $?
>                  ip addr flush dev $iface || return $?
>                  ip link set $iface up
>                  return $?
>          fi
> 
>          tst_rhost_run -c "ip xfrm policy flush" || return $?
>          tst_rhost_run -c "ip xfrm state flush" || return $?
>          tst_rhost_run -c "ip link set $iface down" || return $?
>          tst_rhost_run -c "ip route flush dev $iface" || return $?
>          tst_rhost_run -c "ip addr flush dev $iface" || return $?
>          tst_rhost_run -c "ip link set $iface up"
> }
> 
> Thanks,
>  -- QiuTingting
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Thursday, July 9, 2020 6:13 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; 
> fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 06/12] skip some cases because 
> ONFIG_XFRM_USER is not set
> 
> This one doesn't look right. See comments below.
> 
> > -----Original Message-----
> > From: qiutt
> >
> > for m3ulcb, skip cases in  net.features net.multicast net.ipv6 
> > net.rpc net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip 
> > net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp 
> > net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp 
> > net_stress.multicast net.tcp_cmds net.tirpc_tests
> >
> > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > ---
> >  tests/Functional.LTP/fuego_test.sh | 59
> > ++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 59 insertions(+)
> >
> > diff --git a/tests/Functional.LTP/fuego_test.sh
> > b/tests/Functional.LTP/fuego_test.sh
> > index 156fd39..ab4165d 100755
> > --- a/tests/Functional.LTP/fuego_test.sh
> > +++ b/tests/Functional.LTP/fuego_test.sh
> > @@ -251,6 +251,65 @@ function test_pre_check {
> >      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
> >      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> >
> > +    # some CONFIGs do not set in m3ulcb
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 
> > + dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01
> vlan03"
> > # for net.features
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602
> > + sendfile601 traceroute601 dhcpd6" # for net.ipv6
> 
> Do ping and traceroute depend on CONFIG_XFRM_USER=y?
> That doesn't seem right.
> 
> Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set 
> > + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> > rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create 
> > rpc_clnt_destroy \
> > +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create 
> > + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror 
> > + rpc_clnt_sperrno
> > rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> > +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth 
> > + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> > rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc 
> > rpc_clnt_freeres \
> > +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> > test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect 
> > \
> > +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg 
> > + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> > test_1_to_1_sendto test_1_to_1_shutdown 
> > test_1_to_1_socket_bind_listen \
> > +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort 
> > + test_assoc_shutdown test_autoclose test_basic test_basic_v6 
> > + test_connect
> > test_connectx test_fragments test_fragments_v6 test_getname \
> > +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff
> > + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> > test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style 
> > \
> > +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for 
> > + net.sctp
> 
> A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
> Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" 
> > + "broken_ip4-version broken_ip4-ihl broken_ip4-totlen 
> > + broken_ip4-fragment
> > broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr 
> > broken_ip6-dstaddr \
> > +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change 
> > + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> > route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> > +    if6-addr-addlarge if6-route-adddel if6-route-addlarge 
> > + if6-mtu-change" # for net_stress.interface
> 
> Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01
> > + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> > dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 
> > dccp4_ipsec10 \
> > +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14
> > + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> > dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 
> > dccp4_ipsec23 \
> > +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27
> > + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> > dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 
> > dccp4_ipsec36 \
> > +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04
> > + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> > dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 
> > dccp6_ipsec13 \
> > +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17
> > + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> > dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 
> > dccp6_ipsec26 \
> > +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30
> > + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> > dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp
> 
> This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01
> > + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> > uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> > +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > + icmp4-uni-basic15
> > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> > +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > + icmp4-uni-basic25
> > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> > +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > + icmp6-uni-basic09
> > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> > +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > + icmp6-uni-basic19
> > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> > +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01
> > + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> > tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> > +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 
> > + tcp4_ipsec16
> > + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> > tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> > +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 
> > + tcp4_ipsec30
> > + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> > tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> > +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 
> > + tcp6_ipsec08
> > + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> > tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> > +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 
> > + tcp6_ipsec22
> > + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> > tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> > +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01
> > + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> > udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10
> > udp4_ipsec11 \
> > +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 
> > + udp4_ipsec16
> > + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> > udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> > +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 
> > + udp4_ipsec30
> > + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> > udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> > +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 
> > + udp6_ipsec08
> > + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> > udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> > +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 
> > + udp6_ipsec22
> > + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> > udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> > +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01
> > + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> > sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 
> > sctp4_ipsec10 \
> > +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14
> > + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18
> > + sctp4_ipsec19
> > sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> > +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27
> > + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31
> > + sctp4_ipsec32
> > sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> > +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04
> > + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08
> > + sctp6_ipsec09
> > sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> > +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17
> > + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21
> > + sctp6_ipsec22
> > sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> > +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30
> > + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34
> > + sctp6_ipsec35
> > sctp6_ipsec36" # for net_stress.ipsec_sctp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "mcast4-group-single-socket mcast4-group-multiple-socket 
> > + mcast4-group-same-group
> > mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> > +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03
> > + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 
> > + mcast6-group-
> > single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> > +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02
> > + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> > queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for 
> > net_stress.multicast
> 
> A lot of these lines are very long, and the dependency seems a bit sketch.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" 
> > + "tirpc_rpcb_getaddr tirpc_rpcb_getmaps tirpc_authdes_create
> > tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> > +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror 
> > + tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> > tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> > +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset 
> > + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call 
> > + tirpc_clnt_tp_create
> > tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> > +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg 
> > + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp 
> > + tirpc_clnt_create
> > tirpc_clnt_create_timed tirpc_svc_create \
> > +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" 
> > + # for net.tirpc_tests
> > +
> >      echo "Tests skipped depending on the architecture"
> >      if [ "$ARCHITECTURE" != "s390" ]; then
> >          skip_tests "sbrk03" # Only works in 32bit on s390 series 
> > system
> > --
> > 2.11.0
> 
> I have too many questions about this one.  I did NOT apply it.
>  -- Tim
> 
> 
> 
> 
> 
> _______________________________________________
> Fuego mailing list
> Fuego@lists.linuxfoundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/fuego
> 






[-- Attachment #2: 0001-skip-some-cases-based-on-skiplist-with-config.patch --]
[-- Type: application/octet-stream, Size: 29799 bytes --]

From 0566bb41cee3a46407326ddfff571890e8d4d2a0 Mon Sep 17 00:00:00 2001
From: qiutt <qiutt@cn.fujitsu.com>
Date: Thu, 10 Sep 2020 16:41:18 +0800
Subject: [PATCH 1/3] skip some cases based on skiplist-with-config

datails:
add a case-dependency-list-with-config.txt, including
cases and their dependent CONFIG options

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 .../case-dependency-list-with-config.txt           | 382 +++++++++++++++++++++
 tests/Functional.LTP/fuego_test.sh                 |  22 ++
 2 files changed, 404 insertions(+)
 create mode 100644 tests/Functional.LTP/case-dependency-list-with-config.txt

diff --git a/tests/Functional.LTP/case-dependency-list-with-config.txt b/tests/Functional.LTP/case-dependency-list-with-config.txt
new file mode 100644
index 0000000..c4a3b34
--- /dev/null
+++ b/tests/Functional.LTP/case-dependency-list-with-config.txt
@@ -0,0 +1,382 @@
+#for net_stress.ipsec_dccp
+dccp4_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+dccp4_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+dccp4_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+dccp4_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+dccp4_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp4_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+dccp4_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+dccp4_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+dccp4_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+dccp4_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+dccp4_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+dccp4_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+dccp4_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+dccp4_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+dccp4_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+dccp4_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+dccp6_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+dccp6_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+dccp6_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAST5=y
+dccp6_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAST5=y
+dccp6_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_BLOWFISH=y
+dccp6_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_BLOWFISH=y
+dccp6_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_TWOFISH=y
+dccp6_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_TWOFISH=y
+dccp6_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAMELLIA=y
+dccp6_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAMELLIA=y
+dccp6_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+dccp6_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+dccp6_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_BEET=y
+
+#for net_stress.ipsec_sctp
+sctp4_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+sctp4_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+sctp4_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+sctp4_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+sctp4_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp4_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+sctp4_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+sctp4_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+sctp4_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+sctp4_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+sctp4_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+sctp4_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+sctp4_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+sctp4_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+sctp4_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+sctp4_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+sctp6_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+sctp6_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+sctp6_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAST5=y
+sctp6_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAST5=y
+sctp6_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_BLOWFISH=y
+sctp6_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_BLOWFISH=y
+sctp6_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_TWOFISH=y
+sctp6_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_TWOFISH=y
+sctp6_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAMELLIA=y
+sctp6_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAMELLIA=y
+sctp6_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+sctp6_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+sctp6_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_BEET=y
+
+#for net_stress.ipsec_tcp
+tcp4_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+tcp4_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+tcp4_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+tcp4_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+tcp4_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp4_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+tcp4_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+tcp4_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+tcp4_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+tcp4_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+tcp4_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+tcp4_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+tcp4_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+tcp4_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+tcp4_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+tcp4_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+tcp6_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+tcp6_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+tcp6_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAST5=y
+tcp6_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAST5=y
+tcp6_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_BLOWFISH=y
+tcp6_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_BLOWFISH=y
+tcp6_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_TWOFISH=y
+tcp6_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_TWOFISH=y
+tcp6_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAMELLIA=y
+tcp6_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAMELLIA=y
+tcp6_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+tcp6_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+tcp6_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_BEET=y
+
+#for net_stress.ipsec_udp
+udp4_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+udp4_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+udp4_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+udp4_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+udp4_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp4_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+udp4_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAST5=y
+udp4_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+udp4_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_BLOWFISH=y
+udp4_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+udp4_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_TWOFISH=y
+udp4_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+udp4_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_CAMELLIA=y
+udp4_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+udp4_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+udp4_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+udp6_ipsec02: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec03: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec04: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec05: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec06: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec07: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec08: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec09: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec10: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec11: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec12: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec13: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec14: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec15: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec16: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec17: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec18: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec19: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec20: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec21: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec22: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec23: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec24: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+udp6_ipsec25: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+udp6_ipsec26: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAST5=y
+udp6_ipsec27: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAST5=y
+udp6_ipsec28: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_BLOWFISH=y
+udp6_ipsec29: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_BLOWFISH=y
+udp6_ipsec30: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_TWOFISH=y
+udp6_ipsec31: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_TWOFISH=y
+udp6_ipsec32: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_CAMELLIA=y
+udp6_ipsec33: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_CAMELLIA=y
+udp6_ipsec34: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+udp6_ipsec35: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y CONFIG_CRYPTO_SERPENT=y CONFIG_CRYPTO_RMD160=y
+udp6_ipsec36: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_BEET=y
+
+#for net_stress.ipsec_icmp
+icmp4-uni-basic02: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+icmp4-uni-basic03: CONFIG_XFRM_USER=y CONFIG_INET_AH=y
+icmp4-uni-basic04: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic05: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic06: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+icmp4-uni-basic07: CONFIG_XFRM_USER=y CONFIG_INET_IPCOMP=y
+icmp4-uni-basic08: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic09: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic10: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic11: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic12: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic13: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic14: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic15: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic16: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic17: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic18: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic19: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic20: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic21: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic22: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic23: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic24: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic25: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp4-uni-basic26: CONFIG_XFRM_USER=y CONFIG_INET_ESP=y
+icmp6-uni-basic02: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic03: CONFIG_XFRM_USER=y CONFIG_INET6_AH=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic04: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic05: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic06: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic07: CONFIG_XFRM_USER=y CONFIG_INET6_IPCOMP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic08: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic09: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic10: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic11: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic12: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic13: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic14: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic15: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic16: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic17: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic18: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic19: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic20: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic21: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic22: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic23: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic24: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+icmp6-uni-basic25: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_TUNNEL=y
+icmp6-uni-basic26: CONFIG_XFRM_USER=y CONFIG_INET6_ESP=y CONFIG_INET6_XFRM_MODE_BEET=y
+# for net.sctp
+test_1_to_1_accept_close:  CONFIG_IP_SCTP=y
+test_1_to_1_addrs:  CONFIG_IP_SCTP=y
+test_1_to_1_connect:  CONFIG_IP_SCTP=y
+test_1_to_1_connectx:  CONFIG_IP_SCTP=y
+test_1_to_1_events:  CONFIG_IP_SCTP=y
+test_1_to_1_initmsg_connect:  CONFIG_IP_SCTP=y
+test_1_to_1_nonblock:  CONFIG_IP_SCTP=y
+test_1_to_1_recvfrom:  CONFIG_IP_SCTP=y
+test_1_to_1_recvmsg:  CONFIG_IP_SCTP=y
+test_1_to_1_rtoinfo:  CONFIG_IP_SCTP=y
+test_1_to_1_send:  CONFIG_IP_SCTP=y
+test_1_to_1_sendmsg:  CONFIG_IP_SCTP=y
+test_1_to_1_sendto:  CONFIG_IP_SCTP=y
+test_1_to_1_shutdown:  CONFIG_IP_SCTP=y
+test_1_to_1_socket_bind_listen:  CONFIG_IP_SCTP=y
+test_1_to_1_sockopt:  CONFIG_IP_SCTP=y
+test_1_to_1_threads:  CONFIG_IP_SCTP=y
+test_assoc_abort:  CONFIG_IP_SCTP=y
+test_assoc_shutdown:  CONFIG_IP_SCTP=y
+test_autoclose:  CONFIG_IP_SCTP=y
+test_basic:  CONFIG_IP_SCTP=y
+test_basic_v6:  CONFIG_IP_SCTP=y
+test_connect:  CONFIG_IP_SCTP=y
+test_connectx:  CONFIG_IP_SCTP=y
+test_fragments:  CONFIG_IP_SCTP=y
+test_fragments_v6:  CONFIG_IP_SCTP=y
+test_getname:  CONFIG_IP_SCTP=y
+test_getname_v6:  CONFIG_IP_SCTP=y
+test_inaddr_any:  CONFIG_IP_SCTP=y
+test_inaddr_any_v6:  CONFIG_IP_SCTP=y
+test_peeloff:  CONFIG_IP_SCTP=y
+test_peeloff_v6:  CONFIG_IP_SCTP=y
+test_recvmsg:  CONFIG_IP_SCTP=y
+test_sctp_sendrecvmsg:  CONFIG_IP_SCTP=y
+test_sctp_sendrecvmsg_v6:  CONFIG_IP_SCTP=y
+test_sockopt:  CONFIG_IP_SCTP=y
+test_sockopt_v6:  CONFIG_IP_SCTP=y
+test_tcp_style:  CONFIG_IP_SCTP=y
+test_tcp_style_v6:  CONFIG_IP_SCTP=y
+test_timetolive:  CONFIG_IP_SCTP=y
+test_timetolive_v6:  CONFIG_IP_SCTP=y
+
diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 02fb250..516d13e 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -79,6 +79,24 @@ function skip_tests {
     done
 }
 
+function skip_tests_with_config_skiplist {
+    local CONFIG_SKIPLIST_FILE_NAME="${1:-case-dependency-list-with-config.txt}"
+    local CONFIG_SKIPLIST_FILE=${TEST_HOME}/${CONFIG_SKIPLIST_FILE_NAME}
+    if [ ! -f $CONFIG_SKIPLIST_FILE ]; then
+        echo "No $CONFIG_SKIPLIST_FILE"
+        return 0
+    fi
+
+    while read line
+    do
+        if [ "${line:0:1}" != '#' ]; then
+            TESTS=$(echo $line |awk -F ':' '{print $1}')
+            KCONFIGS=$(echo $line |awk -F ':' '{print $2}')
+            skip_if_kconfig_differs "$KCONFIGS" "$TESTS"
+        fi
+    done < $CONFIG_SKIPLIST_FILE
+}
+
 # $1: command/program to look for
 # $2: string with LTP test names separated by spaces
 function skip_if_command_unavailable {
@@ -251,6 +269,10 @@ function test_pre_check {
     skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
     skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
 
+    #skip tests depend on case-dependency-list-with-config.txt
+    echo "Tests skipped depending on the configuration of the target kernel, based on case-dependency-list-with-config.txt"
+    skip_tests_with_config_skiplist
+
     echo "Tests skipped depending on the architecture"
     if [ "$ARCHITECTURE" != "s390" ]; then
         skip_tests "sbrk03" # Only works in 32bit on s390 series system
-- 
2.11.0


[-- Attachment #3: 0002-skip-some-cases-based-on-skiplist-with-cmd.patch --]
[-- Type: application/octet-stream, Size: 9191 bytes --]

From 23c56afc77753049b02bf5f4c0ce8bec2dcf2f3a Mon Sep 17 00:00:00 2001
From: qiutt <qiutt@cn.fujitsu.com>
Date: Thu, 10 Sep 2020 16:52:02 +0800
Subject: [PATCH 2/3] skip some cases based on skiplist-with-cmd

details:
add a case-dependency-list-with-cmd.txt, including
cases and their dependent cmds

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 .../case-dependency-list-with-cmd.txt              | 322 +++++++++++++++++++++
 tests/Functional.LTP/fuego_test.sh                 |  26 ++
 2 files changed, 348 insertions(+)
 create mode 100644 tests/Functional.LTP/case-dependency-list-with-cmd.txt

diff --git a/tests/Functional.LTP/case-dependency-list-with-cmd.txt b/tests/Functional.LTP/case-dependency-list-with-cmd.txt
new file mode 100644
index 0000000..728be6b
--- /dev/null
+++ b/tests/Functional.LTP/case-dependency-list-with-cmd.txt
@@ -0,0 +1,322 @@
+#net_stress.ipsec_dccp
+dccp4_ipsec01: ss
+dccp4_ipsec02: ss
+dccp4_ipsec03: ss
+dccp4_ipsec04: ss
+dccp4_ipsec05: ss
+dccp4_ipsec06: ss
+dccp4_ipsec07: ss
+dccp4_ipsec08: ss
+dccp4_ipsec09: ss
+dccp4_ipsec10: ss
+dccp4_ipsec11: ss
+dccp4_ipsec12: ss
+dccp4_ipsec13: ss
+dccp4_ipsec14: ss
+dccp4_ipsec15: ss
+dccp4_ipsec16: ss
+dccp4_ipsec17: ss
+dccp4_ipsec18: ss
+dccp4_ipsec19: ss
+dccp4_ipsec20: ss
+dccp4_ipsec21: ss
+dccp4_ipsec22: ss
+dccp4_ipsec23: ss
+dccp4_ipsec24: ss
+dccp4_ipsec25: ss
+dccp4_ipsec26: ss
+dccp4_ipsec27: ss
+dccp4_ipsec28: ss
+dccp4_ipsec29: ss
+dccp4_ipsec30: ss
+dccp4_ipsec31: ss
+dccp4_ipsec32: ss
+dccp4_ipsec33: ss
+dccp4_ipsec34: ss
+dccp4_ipsec35: ss
+dccp4_ipsec36: ss
+dccp6_ipsec01: ss
+dccp6_ipsec02: ss
+dccp6_ipsec03: ss
+dccp6_ipsec04: ss
+dccp6_ipsec05: ss
+dccp6_ipsec06: ss
+dccp6_ipsec07: ss
+dccp6_ipsec08: ss
+dccp6_ipsec09: ss
+dccp6_ipsec10: ss
+dccp6_ipsec11: ss
+dccp6_ipsec12: ss
+dccp6_ipsec13: ss
+dccp6_ipsec14: ss
+dccp6_ipsec15: ss
+dccp6_ipsec16: ss
+dccp6_ipsec17: ss
+dccp6_ipsec18: ss
+dccp6_ipsec19: ss
+dccp6_ipsec20: ss
+dccp6_ipsec21: ss
+dccp6_ipsec22: ss
+dccp6_ipsec23: ss
+dccp6_ipsec24: ss
+dccp6_ipsec25: ss
+dccp6_ipsec26: ss
+dccp6_ipsec27: ss
+dccp6_ipsec28: ss
+dccp6_ipsec29: ss
+dccp6_ipsec30: ss
+dccp6_ipsec31: ss
+dccp6_ipsec32: ss
+dccp6_ipsec33: ss
+dccp6_ipsec34: ss
+dccp6_ipsec35: ss
+dccp6_ipsec36: ss
+
+#net_stress.ipsec_tcp
+tcp4_ipsec01: ss
+tcp4_ipsec02: ss
+tcp4_ipsec03: ss
+tcp4_ipsec04: ss
+tcp4_ipsec05: ss
+tcp4_ipsec06: ss
+tcp4_ipsec07: ss
+tcp4_ipsec08: ss
+tcp4_ipsec09: ss
+tcp4_ipsec10: ss
+tcp4_ipsec11: ss
+tcp4_ipsec12: ss
+tcp4_ipsec13: ss
+tcp4_ipsec14: ss
+tcp4_ipsec15: ss
+tcp4_ipsec16: ss
+tcp4_ipsec17: ss
+tcp4_ipsec18: ss
+tcp4_ipsec19: ss
+tcp4_ipsec20: ss
+tcp4_ipsec21: ss
+tcp4_ipsec22: ss
+tcp4_ipsec23: ss
+tcp4_ipsec24: ss
+tcp4_ipsec25: ss
+tcp4_ipsec26: ss
+tcp4_ipsec27: ss
+tcp4_ipsec28: ss
+tcp4_ipsec29: ss
+tcp4_ipsec30: ss
+tcp4_ipsec31: ss
+tcp4_ipsec32: ss
+tcp4_ipsec33: ss
+tcp4_ipsec34: ss
+tcp4_ipsec35: ss
+tcp4_ipsec36: ss
+tcp6_ipsec01: ss
+tcp6_ipsec02: ss
+tcp6_ipsec03: ss
+tcp6_ipsec04: ss
+tcp6_ipsec05: ss
+tcp6_ipsec06: ss
+tcp6_ipsec07: ss
+tcp6_ipsec08: ss
+tcp6_ipsec09: ss
+tcp6_ipsec10: ss
+tcp6_ipsec11: ss
+tcp6_ipsec12: ss
+tcp6_ipsec13: ss
+tcp6_ipsec14: ss
+tcp6_ipsec15: ss
+tcp6_ipsec16: ss
+tcp6_ipsec17: ss
+tcp6_ipsec18: ss
+tcp6_ipsec19: ss
+tcp6_ipsec20: ss
+tcp6_ipsec21: ss
+tcp6_ipsec22: ss
+tcp6_ipsec23: ss
+tcp6_ipsec24: ss
+tcp6_ipsec25: ss
+tcp6_ipsec26: ss
+tcp6_ipsec27: ss
+tcp6_ipsec28: ss
+tcp6_ipsec29: ss
+tcp6_ipsec30: ss
+tcp6_ipsec31: ss
+tcp6_ipsec32: ss
+tcp6_ipsec33: ss
+tcp6_ipsec34: ss
+tcp6_ipsec35: ss
+tcp6_ipsec36: ss
+
+#net_stress.ipsec_udp
+udp4_ipsec01: ss
+udp4_ipsec02: ss
+udp4_ipsec03: ss
+udp4_ipsec04: ss
+udp4_ipsec05: ss
+udp4_ipsec06: ss
+udp4_ipsec07: ss
+udp4_ipsec08: ss
+udp4_ipsec09: ss
+udp4_ipsec10: ss
+udp4_ipsec11: ss
+udp4_ipsec12: ss
+udp4_ipsec13: ss
+udp4_ipsec14: ss
+udp4_ipsec15: ss
+udp4_ipsec16: ss
+udp4_ipsec17: ss
+udp4_ipsec18: ss
+udp4_ipsec19: ss
+udp4_ipsec20: ss
+udp4_ipsec21: ss
+udp4_ipsec22: ss
+udp4_ipsec23: ss
+udp4_ipsec24: ss
+udp4_ipsec25: ss
+udp4_ipsec26: ss
+udp4_ipsec27: ss
+udp4_ipsec28: ss
+udp4_ipsec29: ss
+udp4_ipsec30: ss
+udp4_ipsec31: ss
+udp4_ipsec32: ss
+udp4_ipsec33: ss
+udp4_ipsec34: ss
+udp4_ipsec35: ss
+udp4_ipsec36: ss
+udp6_ipsec01: ss
+udp6_ipsec02: ss
+udp6_ipsec03: ss
+udp6_ipsec04: ss
+udp6_ipsec05: ss
+udp6_ipsec06: ss
+udp6_ipsec07: ss
+udp6_ipsec08: ss
+udp6_ipsec09: ss
+udp6_ipsec10: ss
+udp6_ipsec11: ss
+udp6_ipsec12: ss
+udp6_ipsec13: ss
+udp6_ipsec14: ss
+udp6_ipsec15: ss
+udp6_ipsec16: ss
+udp6_ipsec17: ss
+udp6_ipsec18: ss
+udp6_ipsec19: ss
+udp6_ipsec20: ss
+udp6_ipsec21: ss
+udp6_ipsec22: ss
+udp6_ipsec23: ss
+udp6_ipsec24: ss
+udp6_ipsec25: ss
+udp6_ipsec26: ss
+udp6_ipsec27: ss
+udp6_ipsec28: ss
+udp6_ipsec29: ss
+udp6_ipsec30: ss
+udp6_ipsec31: ss
+udp6_ipsec32: ss
+udp6_ipsec33: ss
+udp6_ipsec34: ss
+udp6_ipsec35: ss
+udp6_ipsec36: ss
+
+# for net.features
+tcp_fastopen: tc
+tcp_fastopen6: tc
+vlan03: ss
+
+# for net_stress.multicast 
+mcast4-pktfld01: rsh
+mcast4-pktfld02: rsh
+mcast4-queryfld01: rsh
+mcast4-queryfld02: rsh
+mcast4-queryfld03: rsh
+mcast4-queryfld04: rsh
+mcast4-queryfld05: rsh
+mcast4-queryfld06: rsh
+mcast6-pktfld01: rsh
+mcast6-pktfld02: rsh
+mcast6-queryfld01: rsh
+mcast6-queryfld02: rsh
+mcast6-queryfld03: rsh
+mcast6-queryfld04: rsh
+mcast6-queryfld05: rsh
+mcast6-queryfld06: rsh
+
+# for net.rpc
+rpc01: rpcinfo
+
+# for net.rpc_tests
+rpc_pmap_set: rpcinfo
+rpc_pmap_unset: rpcinfo
+rpc_pmap_getport: rpcinfo
+rpc_pmap_getmaps: rpcinfo
+rpc_pmap_rmtcall: rpcinfo
+rpc_clnt_broadcast: rpcinfo
+rpc_svc_destroy: rpcinfo
+rpc_svcfd_create: rpcinfo
+rpc_clnt_destroy: rpcinfo
+rpc_clnt_create: rpcinfo
+rpc_clnttcp_create: rpcinfo
+rpc_clntudp_create: rpcinfo
+rpc_clntudp_bufcreate: rpcinfo
+rpc_clnt_perrno: rpcinfo
+rpc_clnt_perror: rpcinfo
+rpc_clnt_sperrno: rpcinfo
+rpc_clnt_sperror: rpcinfo
+rpc_svcerr_noproc: rpcinfo
+rpc_svcerr_noprog: rpcinfo
+rpc_svcerr_progvers: rpcinfo
+rpc_svcerr_systemerr: rpcinfo
+rpc_svcerr_auth: rpcinfo
+rpc_svcerr_weakauth: rpcinfo
+rpc_xprt_register: rpcinfo
+rpc_xprt_unregister: rpcinfo
+rpc_svc_register: rpcinfo
+rpc_registerrpc: rpcinfo
+rpc_clnt_call: rpcinfo
+rpc_callrpc: rpcinfo
+rpc_clnt_freeres: rpcinfo
+rpc_clnt_geterr: rpcinfo
+rpc_clnt_control: rpcinfo
+rpc_svc_getcaller: rpcinfo
+rpc_svc_getargs: rpcinfo
+rpc_svc_sendreply: rpcinfo
+
+#for net.tirpc_tests 
+tirpc_rpcb_getaddr: rpcinfo
+tirpc_rpcb_getmaps: rpcinfo
+tirpc_authdes_create: rpcinfo
+tirpc_authdes_seccreate: rpcinfo
+tirpc_clnt_dg_create: rpcinfo
+tirpc_clnt_vc_create: rpcinfo
+tirpc_bottomlevel_clnt_call: rpcinfo
+tirpc_clnt_perrno: rpcinfo
+tirpc_clnt_perror: rpcinfo
+tirpc_svcerr_noproc: rpcinfo
+tirpc_svcerr_noprog: rpcinfo
+tirpc_svcerr_progvers: rpcinfo
+tirpc_svcerr_systemerr: rpcinfo
+tirpc_svcerr_weakauth: rpcinfo
+tirpc_clnt_tli_create: rpcinfo
+tirpc_rpcb_set: rpcinfo
+tirpc_rpcb_unset: rpcinfo
+tirpc_rpcb_rmtcallrpcinfo
+tirpc_svc_regrpcinfo
+tirpc_expertlevel_clnt_callrpcinfo
+tirpc_clnt_tp_createrpcinfo
+tirpc_clnt_tp_create_timedrpcinfo
+tirpc_svc_tp_createrpcinfo
+tirpc_interlevel_clnt_callrpcinfo
+tirpc_clnt_controlrpcinfo
+tirpc_rpc_regrpcinfo
+tirpc_rpc_callrpcinfo
+tirpc_rpc_broadcastrpcinfo
+tirpc_rpc_broadcast_exprpcinfo
+tirpc_clnt_createrpcinfo
+tirpc_clnt_create_timedrpcinfo
+tirpc_svc_createrpcinfo
+tirpc_toplevel_clnt_callrpcinfo
+tirpc_clnt_destroyrpcinfo
+tirpc_svc_destroyrpcinfo
+
diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 516d13e..c4c8129 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -97,6 +97,28 @@ function skip_tests_with_config_skiplist {
     done < $CONFIG_SKIPLIST_FILE
 }
 
+function skip_tests_with_cmd_skiplist {
+    local CMD_SKIPLIST_FILE_NAME="${1:-case-dependency-list-with-cmd.txt}"
+    local CMD_SKIPLIST_FILE=${TEST_HOME}/${CMD_SKIPLIST_FILE_NAME}
+    if [ ! -f $CMD_SKIPLIST_FILE ]; then
+        echo "No $CMD_SKIPLIST_FILE"
+        return 0
+    fi
+
+    #change IFS to '\n' to avoid line separated by space
+    IFS_bak=$IFS
+    IFS=$'\n'
+    for line in $(cat $CMD_SKIPLIST_FILE)
+    do
+        if [ "${line:0:1}" != '#' ]; then
+            TESTS=$(echo $line |awk -F ':' '{print $1}')
+            KCMD=$(echo $line |awk -F ':' '{print $2}')
+            IFS=$IFS_bak
+            skip_if_command_unavailable $KCMD $TESTS
+        fi
+    done
+}
+
 # $1: command/program to look for
 # $2: string with LTP test names separated by spaces
 function skip_if_command_unavailable {
@@ -273,6 +295,10 @@ function test_pre_check {
     echo "Tests skipped depending on the configuration of the target kernel, based on case-dependency-list-with-config.txt"
     skip_tests_with_config_skiplist
 
+    #skip tests depend on case-dependency-list-with-cmd.txt
+    echo "Tests skipped depending on the availability of a command on the target, based on case-dependency-list-with-cmd.txt"
+    skip_tests_with_cmd_skiplist
+
     echo "Tests skipped depending on the architecture"
     if [ "$ARCHITECTURE" != "s390" ]; then
         skip_tests "sbrk03" # Only works in 32bit on s390 series system
-- 
2.11.0

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Bird, Tim]

Tingting,

I have examined the first patch, and it will work.  But I would like to measure the
overhead of the operations that this test is performing.  Already, the time
spent in test_pre_check for Functional.LTP is quite long.  I'd like to prevent it from
getting too long.

The current mechanisms in Fuego are not very efficient for testing a large number
of dependencies.  There may be several program invocations and file operations for
each dependency check. So I am thinking about writing an alternate mechanism.

I am considering writing a small python program (which I'll call 'check-dependencies" to perform dependency-checking based on dependencies listed in a file.

With such a program I could pass the dependency file ( case-dependency-list-with-config.txt),
and process them all at once, rather than one at a time.

The program would do the following:
 - read the kernel config file once and keep it in memory
 - read the case-dependency file (or a list of files) specified, and add items to
   the skiplist, if their dependencies are not met

It would be nice to keep an extra file (maybe in json format), that we would
place in the log directory, to hold the reason for each skipped item (missing kernel config,
wrong architecture, missing command on target).  This information would be quite 
helpful to diagnose problems with a test (why particular tests were skipped).

BTW - I'd rather call the case-dependency file something like:
"testcase-dependencies.txt", and have syntax in the file to indicate the
type of dependency.

Let me know what you think.  I'll try to get a prototype written next week
that we can review and discuss.
 -- Tim


> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> 
> Hi,Tim
> 
> Thanks for your reply.
> We modified tst_init_iface() refer to 20200515-164-ga6c8680, and then retested.
> It took some time.
> 
> Some cases(such as ping601) are really unrelated to CONFIG_XFRM_USER(or ip xfrm).
> So we add a file case-dependency-list-with-config.txt, to describe the relationship between testcase and the CONFIG,
> and add a function skip_tests_with_config_skiplist to add testcases to skiplist.txt.
> refer to 0001-skip-some-cases-based-on-skiplist-with-config.patch
> 
> As the same way, we add case-dependency-list-with-config.txt and a function skip_tests_with_cmd_skiplist,
> for the relationship between testcases and the cmds.
> refer to 0002-skip-some-cases-based-on-skiplist-with-cmd.patch
> 
> If these patches is ok, we will commit them.
> 
> Thanks,
>   -- QiuTingting
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Friday, August 7, 2020 8:03 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
> Cc: fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set
> 
> > -----Original Message-----
> > From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> >
> > Hi,Tim
> >
> > > This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> >
> > We can add wildcard processing in fuego_test.sh, but this is only used
> > in some cases, such as using skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "^icmp4-uni* ^icmp6-uni*"
> > instead of
> > skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01
> > icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-uni-
> > basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> >     icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > icmp4-uni-basic15
> > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> >     icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > icmp4-uni-basic25
> > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> >     icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > icmp6-uni-basic09
> > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> >     icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > icmp6-uni-basic19
> > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> >     icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25
> > icmp6-uni-basic26" # for net_stress.ipsec_icmp But there are still
> > some cases(preliminary estimate is at least 80+), the code is not
> > concise after using wildcards. Becase the name gap is very small (some
> > cases need to be tested, but some need skip)
> >
> > Therefore, we suggest whether it is possible to write all the cases
> > that depend on "CONFIG_XFRM_USER=y" into a CONFIG_XFRM_USER_skiplist.txt (naming rule: ${config name}_skiplist.txt), and add the
> contents to skiplist.txt during run test.
> > I think this can achieve requirements of skip case, and concise code.
> > See patch :
> > 0006-skip-some-cases-because-ONFIG_XFRM_USER-is-not-set.patch
> 
> The approach is intriguing, but seems a bit complicated.  Also, with this particular dependency, based on my experimentation with dccp01
> I don't trust that all these tests are actually dependent on CONFIG_XFRM_USER.
> 
> However, that issue is separate from the issue of how to handle large numbers of dependencies.
> 
> With regards to using wildcards,
> wildcard dependencies only work at the level of fuego_test.sh if Fuego can expand the wildcard to come up with the list of items to match
> - which are individual testcase tags specified in LTP's runtest files (also called 'scenario' files or command files).  I'd rather not have LTP
> have to go parsing the files in that directory to process wildcards.
> 
> Passing around strings with wildcard patterns in them is quite problematical in shell code, so I don't think it's feasible to try to keep the
> regexes with wildcards in the skiplist itself (to pass through eventually to runltp), and with that solution, runltp would need to be modified
> to support wildcards.  Maybe it would work, but it seems error prone.
> 
> So, for now, I think I'd like to withdraw my wildcard suggestion.
> 
> Putting the lists into a file seems like a good idea.  I don't like them cluttering up the fuego_test.sh code with huge lists.
> 
> I'm not sure I want to fully generalize this solution yet.  If we kept this list of files dependent on a config, I think I'd rather do something
> like modifying skip_if_kconfig_differs as follows:
> 
> function skip_if_kconfig_differs {
>     set -f
>     local KCONFIGS=($1)
>     set +f
>     local TESTS="$2"
> 
>     for cfg in ${KCONFIGS[@]}; do
>         if ! check_kconfig "$cfg"; then
>             for item in "${TESTS}"; do
>                 if [ -f "$item" ]; then
>                     cat "$item" >> ${LOGDIR}/skiplist.txt
>                else
>                     echo "$item" >> ${LOGDIR}/skiplist.txt
>                fi
>            done
>         fi
>     done
> }
> 
> And then call it with:
> 
> skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "$TEST_HOME/CONFIG_XFRM_USER_skiplist.txt"
> 
>  -- Tim
> 
> >
> > Thanks,
> >  -- QiuTingting
> >
> > -----Original Message-----
> > From: Fuego [mailto:fuego-bounces@lists.linuxfoundation.org] On Behalf
> > Of Qiu, Tingting
> > Sent: Thursday, July 30, 2020 1:49 PM
> > To: Bird, Tim <Tim.Bird@sony.com>
> > Cc: fuego@lists.linuxfoundation.org
> > Subject: Re: [Fuego] [PATCH 06/12] skip some cases because
> > ONFIG_XFRM_USER is not set
> >
> > Hi,Tim
> >
> > Cases such as ping, tcp, udp and traceroute are all based on the virtual network card.
> > Initializing the virtual network card is implemented by the common script(test_net.sh).
> > The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
> > Therefore, all cases that use virtualized network cards will be affected in the same way.
> > It is reasonable to use the "CONFIG_XFRM_USER=y" to check.
> >
> > The following is my guess:
> > There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
> > when initializing the virtualized network card, using "ip xfrm" is
> > necessary, because many ipsec-related content need to be tested in the network test set (at least 504 cases).
> >
> > *1:
> > tst_init_iface()
> > {
> >          local type="${1:-lhost}"
> >          local link_num="${2:-0}"
> >          local iface="$(tst_iface $type $link_num)"
> >          tst_resm TINFO "initialize '$type' '$iface' interface"
> >
> >          if [ "$type" = "lhost" ]; then
> >                  ip xfrm policy flush || return $?
> >                  ip xfrm state flush || return $?
> >                  ip link set $iface down || return $?
> >                  ip route flush dev $iface || return $?
> >                  ip addr flush dev $iface || return $?
> >                  ip link set $iface up
> >                  return $?
> >          fi
> >
> >          tst_rhost_run -c "ip xfrm policy flush" || return $?
> >          tst_rhost_run -c "ip xfrm state flush" || return $?
> >          tst_rhost_run -c "ip link set $iface down" || return $?
> >          tst_rhost_run -c "ip route flush dev $iface" || return $?
> >          tst_rhost_run -c "ip addr flush dev $iface" || return $?
> >          tst_rhost_run -c "ip link set $iface up"
> > }
> >
> > Thanks,
> >  -- QiuTingting
> >
> > -----Original Message-----
> > From: Bird, Tim [mailto:Tim.Bird@sony.com]
> > Sent: Thursday, July 9, 2020 6:13 AM
> > To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>;
> > fuego@lists.linuxfoundation.org
> > Subject: RE: [Fuego] [PATCH 06/12] skip some cases because
> > ONFIG_XFRM_USER is not set
> >
> > This one doesn't look right. See comments below.
> >
> > > -----Original Message-----
> > > From: qiutt
> > >
> > > for m3ulcb, skip cases in  net.features net.multicast net.ipv6
> > > net.rpc net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip
> > > net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp
> > > net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp
> > > net_stress.multicast net.tcp_cmds net.tirpc_tests
> > >
> > > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > > ---
> > >  tests/Functional.LTP/fuego_test.sh | 59
> > > ++++++++++++++++++++++++++++++++++++++
> > >  1 file changed, 59 insertions(+)
> > >
> > > diff --git a/tests/Functional.LTP/fuego_test.sh
> > > b/tests/Functional.LTP/fuego_test.sh
> > > index 156fd39..ab4165d 100755
> > > --- a/tests/Functional.LTP/fuego_test.sh
> > > +++ b/tests/Functional.LTP/fuego_test.sh
> > > @@ -251,6 +251,65 @@ function test_pre_check {
> > >      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
> > >      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> > >
> > > +    # some CONFIGs do not set in m3ulcb
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01
> > > + dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01
> > vlan03"
> > > # for net.features
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602
> > > + sendfile601 traceroute601 dhcpd6" # for net.ipv6
> >
> > Do ping and traceroute depend on CONFIG_XFRM_USER=y?
> > That doesn't seem right.
> >
> > Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set
> > > + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> > > rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create
> > > rpc_clnt_destroy \
> > > +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create
> > > + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror
> > > + rpc_clnt_sperrno
> > > rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> > > +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth
> > > + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> > > rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc
> > > rpc_clnt_freeres \
> > > +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> > > test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect
> > > \
> > > +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg
> > > + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> > > test_1_to_1_sendto test_1_to_1_shutdown
> > > test_1_to_1_socket_bind_listen \
> > > +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort
> > > + test_assoc_shutdown test_autoclose test_basic test_basic_v6
> > > + test_connect
> > > test_connectx test_fragments test_fragments_v6 test_getname \
> > > +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff
> > > + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> > > test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style
> > > \
> > > +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for
> > > + net.sctp
> >
> > A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
> > Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "broken_ip4-version broken_ip4-ihl broken_ip4-totlen
> > > + broken_ip4-fragment
> > > broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr
> > > broken_ip6-dstaddr \
> > > +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change
> > > + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> > > route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> > > +    if6-addr-addlarge if6-route-adddel if6-route-addlarge
> > > + if6-mtu-change" # for net_stress.interface
> >
> > Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01
> > > + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> > > dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09
> > > dccp4_ipsec10 \
> > > +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14
> > > + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> > > dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22
> > > dccp4_ipsec23 \
> > > +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27
> > > + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> > > dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35
> > > dccp4_ipsec36 \
> > > +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04
> > > + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> > > dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12
> > > dccp6_ipsec13 \
> > > +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17
> > > + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> > > dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25
> > > dccp6_ipsec26 \
> > > +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30
> > > + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> > > dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp
> >
> > This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01
> > > + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> > > uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> > > +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > > + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > > + icmp4-uni-basic15
> > > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> > > +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > > + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > > + icmp4-uni-basic25
> > > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> > > +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > > + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > > + icmp6-uni-basic09
> > > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> > > +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > > + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > > + icmp6-uni-basic19
> > > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> > > +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01
> > > + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> > > tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> > > +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15
> > > + tcp4_ipsec16
> > > + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> > > tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> > > +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29
> > > + tcp4_ipsec30
> > > + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> > > tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> > > +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07
> > > + tcp6_ipsec08
> > > + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> > > tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> > > +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21
> > > + tcp6_ipsec22
> > > + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> > > tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> > > +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01
> > > + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> > > udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10
> > > udp4_ipsec11 \
> > > +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15
> > > + udp4_ipsec16
> > > + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> > > udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> > > +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29
> > > + udp4_ipsec30
> > > + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> > > udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> > > +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07
> > > + udp6_ipsec08
> > > + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> > > udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> > > +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21
> > > + udp6_ipsec22
> > > + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> > > udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> > > +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01
> > > + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> > > sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09
> > > sctp4_ipsec10 \
> > > +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14
> > > + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18
> > > + sctp4_ipsec19
> > > sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> > > +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27
> > > + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31
> > > + sctp4_ipsec32
> > > sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> > > +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04
> > > + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08
> > > + sctp6_ipsec09
> > > sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> > > +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17
> > > + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21
> > > + sctp6_ipsec22
> > > sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> > > +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30
> > > + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34
> > > + sctp6_ipsec35
> > > sctp6_ipsec36" # for net_stress.ipsec_sctp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "mcast4-group-single-socket mcast4-group-multiple-socket
> > > + mcast4-group-same-group
> > > mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> > > +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03
> > > + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06
> > > + mcast6-group-
> > > single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> > > +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02
> > > + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> > > queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for
> > > net_stress.multicast
> >
> > A lot of these lines are very long, and the dependency seems a bit sketch.
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "tirpc_rpcb_getaddr tirpc_rpcb_getmaps tirpc_authdes_create
> > > tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> > > +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror
> > > + tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> > > tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> > > +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset
> > > + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call
> > > + tirpc_clnt_tp_create
> > > tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> > > +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg
> > > + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp
> > > + tirpc_clnt_create
> > > tirpc_clnt_create_timed tirpc_svc_create \
> > > +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy"
> > > + # for net.tirpc_tests
> > > +
> > >      echo "Tests skipped depending on the architecture"
> > >      if [ "$ARCHITECTURE" != "s390" ]; then
> > >          skip_tests "sbrk03" # Only works in 32bit on s390 series
> > > system
> > > --
> > > 2.11.0
> >
> > I have too many questions about this one.  I did NOT apply it.
> >  -- Tim
> >
> >
> >
> >
> >
> > _______________________________________________
> > Fuego mailing list
> > Fuego@lists.linuxfoundation.org
> > https://lists.linuxfoundation.org/mailman/listinfo/fuego
> >
> 
> 
> 
> 

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Qiu, Tingting]

Tim

It's convenient to use a common function like as "check-dependencies".
Do you have any idea to define the check rules?
If there is only one "testcase-dependencies.txt" file, how do we discern the difference between the config-check and the  command-check and others?
If there is a list of files, how do we name them?
Looking forward to your prototype.

 -- QiuTingting
-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Friday, September 18, 2020 4:02 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
Cc: fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

Tingting,

I have examined the first patch, and it will work.  But I would like to measure the overhead of the operations that this test is performing.  Already, the time spent in test_pre_check for Functional.LTP is quite long.  I'd like to prevent it from getting too long.

The current mechanisms in Fuego are not very efficient for testing a large number of dependencies.  There may be several program invocations and file operations for each dependency check. So I am thinking about writing an alternate mechanism.

I am considering writing a small python program (which I'll call 'check-dependencies" to perform dependency-checking based on dependencies listed in a file.

With such a program I could pass the dependency file ( case-dependency-list-with-config.txt),
and process them all at once, rather than one at a time.

The program would do the following:
 - read the kernel config file once and keep it in memory
 - read the case-dependency file (or a list of files) specified, and add items to
   the skiplist, if their dependencies are not met

It would be nice to keep an extra file (maybe in json format), that we would place in the log directory, to hold the reason for each skipped item (missing kernel config, wrong architecture, missing command on target).  This information would be quite helpful to diagnose problems with a test (why particular tests were skipped).

BTW - I'd rather call the case-dependency file something like:
"testcase-dependencies.txt", and have syntax in the file to indicate the type of dependency.

Let me know what you think.  I'll try to get a prototype written next week that we can review and discuss.
 -- Tim


> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> 
> Hi,Tim
> 
> Thanks for your reply.
> We modified tst_init_iface() refer to 20200515-164-ga6c8680, and then retested.
> It took some time.
> 
> Some cases(such as ping601) are really unrelated to CONFIG_XFRM_USER(or ip xfrm).
> So we add a file case-dependency-list-with-config.txt, to describe the 
> relationship between testcase and the CONFIG, and add a function skip_tests_with_config_skiplist to add testcases to skiplist.txt.
> refer to 0001-skip-some-cases-based-on-skiplist-with-config.patch
> 
> As the same way, we add case-dependency-list-with-config.txt and a 
> function skip_tests_with_cmd_skiplist, for the relationship between testcases and the cmds.
> refer to 0002-skip-some-cases-based-on-skiplist-with-cmd.patch
> 
> If these patches is ok, we will commit them.
> 
> Thanks,
>   -- QiuTingting
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Friday, August 7, 2020 8:03 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
> Cc: fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 06/12] skip some cases because 
> ONFIG_XFRM_USER is not set
> 
> > -----Original Message-----
> > From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> >
> > Hi,Tim
> >
> > > This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> >
> > We can add wildcard processing in fuego_test.sh, but this is only 
> > used in some cases, such as using skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "^icmp4-uni* ^icmp6-uni*"
> > instead of
> > skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01
> > icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-uni-
> > basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> >     icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > icmp4-uni-basic15
> > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> >     icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > icmp4-uni-basic25
> > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> >     icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > icmp6-uni-basic09
> > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> >     icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > icmp6-uni-basic19
> > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> >     icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 
> > icmp6-uni-basic26" # for net_stress.ipsec_icmp But there are still 
> > some cases(preliminary estimate is at least 80+), the code is not 
> > concise after using wildcards. Becase the name gap is very small 
> > (some cases need to be tested, but some need skip)
> >
> > Therefore, we suggest whether it is possible to write all the cases 
> > that depend on "CONFIG_XFRM_USER=y" into a 
> > CONFIG_XFRM_USER_skiplist.txt (naming rule: ${config 
> > name}_skiplist.txt), and add the
> contents to skiplist.txt during run test.
> > I think this can achieve requirements of skip case, and concise code.
> > See patch :
> > 0006-skip-some-cases-because-ONFIG_XFRM_USER-is-not-set.patch
> 
> The approach is intriguing, but seems a bit complicated.  Also, with 
> this particular dependency, based on my experimentation with dccp01 I don't trust that all these tests are actually dependent on CONFIG_XFRM_USER.
> 
> However, that issue is separate from the issue of how to handle large numbers of dependencies.
> 
> With regards to using wildcards,
> wildcard dependencies only work at the level of fuego_test.sh if Fuego 
> can expand the wildcard to come up with the list of items to match
> - which are individual testcase tags specified in LTP's runtest files 
> (also called 'scenario' files or command files).  I'd rather not have LTP have to go parsing the files in that directory to process wildcards.
> 
> Passing around strings with wildcard patterns in them is quite 
> problematical in shell code, so I don't think it's feasible to try to 
> keep the regexes with wildcards in the skiplist itself (to pass through eventually to runltp), and with that solution, runltp would need to be modified to support wildcards.  Maybe it would work, but it seems error prone.
> 
> So, for now, I think I'd like to withdraw my wildcard suggestion.
> 
> Putting the lists into a file seems like a good idea.  I don't like them cluttering up the fuego_test.sh code with huge lists.
> 
> I'm not sure I want to fully generalize this solution yet.  If we kept 
> this list of files dependent on a config, I think I'd rather do something like modifying skip_if_kconfig_differs as follows:
> 
> function skip_if_kconfig_differs {
>     set -f
>     local KCONFIGS=($1)
>     set +f
>     local TESTS="$2"
> 
>     for cfg in ${KCONFIGS[@]}; do
>         if ! check_kconfig "$cfg"; then
>             for item in "${TESTS}"; do
>                 if [ -f "$item" ]; then
>                     cat "$item" >> ${LOGDIR}/skiplist.txt
>                else
>                     echo "$item" >> ${LOGDIR}/skiplist.txt
>                fi
>            done
>         fi
>     done
> }
> 
> And then call it with:
> 
> skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "$TEST_HOME/CONFIG_XFRM_USER_skiplist.txt"
> 
>  -- Tim
> 
> >
> > Thanks,
> >  -- QiuTingting
> >
> > -----Original Message-----
> > From: Fuego [mailto:fuego-bounces@lists.linuxfoundation.org] On 
> > Behalf Of Qiu, Tingting
> > Sent: Thursday, July 30, 2020 1:49 PM
> > To: Bird, Tim <Tim.Bird@sony.com>
> > Cc: fuego@lists.linuxfoundation.org
> > Subject: Re: [Fuego] [PATCH 06/12] skip some cases because 
> > ONFIG_XFRM_USER is not set
> >
> > Hi,Tim
> >
> > Cases such as ping, tcp, udp and traceroute are all based on the virtual network card.
> > Initializing the virtual network card is implemented by the common script(test_net.sh).
> > The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
> > Therefore, all cases that use virtualized network cards will be affected in the same way.
> > It is reasonable to use the "CONFIG_XFRM_USER=y" to check.
> >
> > The following is my guess:
> > There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
> > when initializing the virtualized network card, using "ip xfrm" is 
> > necessary, because many ipsec-related content need to be tested in the network test set (at least 504 cases).
> >
> > *1:
> > tst_init_iface()
> > {
> >          local type="${1:-lhost}"
> >          local link_num="${2:-0}"
> >          local iface="$(tst_iface $type $link_num)"
> >          tst_resm TINFO "initialize '$type' '$iface' interface"
> >
> >          if [ "$type" = "lhost" ]; then
> >                  ip xfrm policy flush || return $?
> >                  ip xfrm state flush || return $?
> >                  ip link set $iface down || return $?
> >                  ip route flush dev $iface || return $?
> >                  ip addr flush dev $iface || return $?
> >                  ip link set $iface up
> >                  return $?
> >          fi
> >
> >          tst_rhost_run -c "ip xfrm policy flush" || return $?
> >          tst_rhost_run -c "ip xfrm state flush" || return $?
> >          tst_rhost_run -c "ip link set $iface down" || return $?
> >          tst_rhost_run -c "ip route flush dev $iface" || return $?
> >          tst_rhost_run -c "ip addr flush dev $iface" || return $?
> >          tst_rhost_run -c "ip link set $iface up"
> > }
> >
> > Thanks,
> >  -- QiuTingting
> >
> > -----Original Message-----
> > From: Bird, Tim [mailto:Tim.Bird@sony.com]
> > Sent: Thursday, July 9, 2020 6:13 AM
> > To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; 
> > fuego@lists.linuxfoundation.org
> > Subject: RE: [Fuego] [PATCH 06/12] skip some cases because 
> > ONFIG_XFRM_USER is not set
> >
> > This one doesn't look right. See comments below.
> >
> > > -----Original Message-----
> > > From: qiutt
> > >
> > > for m3ulcb, skip cases in  net.features net.multicast net.ipv6 
> > > net.rpc net.rpc_tests net.sctp net_stress.appl 
> > > net_stress.broken_ip net_stress.interface net_stress.ipsec_dccp  
> > > net_stress.ipsec_icmp net_stress.ipsec_tcp  net_stress.ipsec_udp  
> > > net_stress.ipsec_sctp net_stress.multicast net.tcp_cmds 
> > > net.tirpc_tests
> > >
> > > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > > ---
> > >  tests/Functional.LTP/fuego_test.sh | 59
> > > ++++++++++++++++++++++++++++++++++++++
> > >  1 file changed, 59 insertions(+)
> > >
> > > diff --git a/tests/Functional.LTP/fuego_test.sh
> > > b/tests/Functional.LTP/fuego_test.sh
> > > index 156fd39..ab4165d 100755
> > > --- a/tests/Functional.LTP/fuego_test.sh
> > > +++ b/tests/Functional.LTP/fuego_test.sh
> > > @@ -251,6 +251,65 @@ function test_pre_check {
> > >      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
> > >      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> > >
> > > +    # some CONFIGs do not set in m3ulcb
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01
> > > + dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01
> > vlan03"
> > > # for net.features
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602
> > > + sendfile601 traceroute601 dhcpd6" # for net.ipv6
> >
> > Do ping and traceroute depend on CONFIG_XFRM_USER=y?
> > That doesn't seem right.
> >
> > Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set 
> > > + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> > > rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy 
> > > rpc_svcfd_create rpc_clnt_destroy \
> > > +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create 
> > > + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror 
> > > + rpc_clnt_sperrno
> > > rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> > > +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth 
> > > + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> > > rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc 
> > > rpc_clnt_freeres \
> > > +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> > > test_1_to_1_connectx test_1_to_1_events 
> > > test_1_to_1_initmsg_connect \
> > > +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg 
> > > + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> > > test_1_to_1_sendto test_1_to_1_shutdown 
> > > test_1_to_1_socket_bind_listen \
> > > +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort 
> > > + test_assoc_shutdown test_autoclose test_basic test_basic_v6 
> > > + test_connect
> > > test_connectx test_fragments test_fragments_v6 test_getname \
> > > +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 
> > > + test_peeloff
> > > + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> > > test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 
> > > test_tcp_style \
> > > +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for 
> > > + net.sctp
> >
> > A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
> > Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "broken_ip4-version broken_ip4-ihl broken_ip4-totlen 
> > > + broken_ip4-fragment
> > > broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr 
> > > broken_ip6-dstaddr \
> > > +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change 
> > > + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel 
> > > + if4-
> > > route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> > > +    if6-addr-addlarge if6-route-adddel if6-route-addlarge 
> > > + if6-mtu-change" # for net_stress.interface
> >
> > Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01
> > > + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> > > dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09
> > > dccp4_ipsec10 \
> > > +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14
> > > + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> > > dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22
> > > dccp4_ipsec23 \
> > > +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27
> > > + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> > > dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35
> > > dccp4_ipsec36 \
> > > +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04
> > > + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> > > dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12
> > > dccp6_ipsec13 \
> > > +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17
> > > + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> > > dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25
> > > dccp6_ipsec26 \
> > > +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30
> > > + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> > > dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp
> >
> > This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" 
> > > + "icmp4-uni-basic01
> > > + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> > > uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 
> > > \
> > > +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > > + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > > + icmp4-uni-basic15
> > > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> > > +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > > + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > > + icmp4-uni-basic25
> > > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> > > +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > > + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > > + icmp6-uni-basic09
> > > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> > > +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > > + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > > + icmp6-uni-basic19
> > > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> > > +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01
> > > + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> > > tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> > > +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15
> > > + tcp4_ipsec16
> > > + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> > > tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> > > +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29
> > > + tcp4_ipsec30
> > > + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> > > tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> > > +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07
> > > + tcp6_ipsec08
> > > + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> > > tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> > > +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21
> > > + tcp6_ipsec22
> > > + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> > > tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> > > +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01
> > > + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> > > udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10
> > > udp4_ipsec11 \
> > > +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15
> > > + udp4_ipsec16
> > > + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> > > udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> > > +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29
> > > + udp4_ipsec30
> > > + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> > > udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> > > +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07
> > > + udp6_ipsec08
> > > + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> > > udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> > > +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21
> > > + udp6_ipsec22
> > > + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> > > udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> > > +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01
> > > + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> > > sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09
> > > sctp4_ipsec10 \
> > > +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14
> > > + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18
> > > + sctp4_ipsec19
> > > sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> > > +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27
> > > + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31
> > > + sctp4_ipsec32
> > > sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> > > +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04
> > > + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08
> > > + sctp6_ipsec09
> > > sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> > > +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17
> > > + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21
> > > + sctp6_ipsec22
> > > sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> > > +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30
> > > + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34
> > > + sctp6_ipsec35
> > > sctp6_ipsec36" # for net_stress.ipsec_sctp
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "mcast4-group-single-socket mcast4-group-multiple-socket 
> > > + mcast4-group-same-group
> > > mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> > > +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03
> > > + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06
> > > + mcast6-group-
> > > single-socket mcast6-group-multiple-socket mcast6-group-same-group 
> > > \
> > > +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02
> > > + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> > > queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for 
> > > net_stress.multicast
> >
> > A lot of these lines are very long, and the dependency seems a bit sketch.
> >
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> > > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > > + "tirpc_rpcb_getaddr tirpc_rpcb_getmaps tirpc_authdes_create
> > > tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create 
> > > \
> > > +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno 
> > > + tirpc_clnt_perror tirpc_svcerr_noproc tirpc_svcerr_noprog 
> > > + tirpc_svcerr_progvers
> > > tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> > > +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset 
> > > + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call 
> > > + tirpc_clnt_tp_create
> > > tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> > > +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg 
> > > + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp 
> > > + tirpc_clnt_create
> > > tirpc_clnt_create_timed tirpc_svc_create \
> > > +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy"
> > > + # for net.tirpc_tests
> > > +
> > >      echo "Tests skipped depending on the architecture"
> > >      if [ "$ARCHITECTURE" != "s390" ]; then
> > >          skip_tests "sbrk03" # Only works in 32bit on s390 series 
> > > system
> > > --
> > > 2.11.0
> >
> > I have too many questions about this one.  I did NOT apply it.
> >  -- Tim
> >
> >
> >
> >
> >
> > _______________________________________________
> > Fuego mailing list
> > Fuego@lists.linuxfoundation.org
> > https://lists.linuxfoundation.org/mailman/listinfo/fuego
> >
> 
> 
> 
> 

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Bird, Tim]

TingTing,

I just wanted to let you know that I'm still working through this issue.  It's
still on my action list.  I ran into some unrelated problems running LTP
in my lab, and that took some of my attention away from this issue.

I appreciate the suggestions of alternate mechanisms for doing skiplist
processing for LTP.  I want to play around with LTP a bit more, and
familiarize myself with the different dependencies and set of tests
in the latest LTP tree, before making a decision.

The overall skiplist calculation is starting to get long, and taking
enough time that it overwhelms the test duration (for some 
specs).  I'd like to start thinking about possibly caching the
results of the skiplist calculation - but that raises a number of
issues.

More below...

> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> 
> Hi,Tim
> 
> Cases such as ping, tcp, udp and traceroute are all based on the virtual network card.
> Initializing the virtual network card is implemented by the common script(test_net.sh).
> The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
> Therefore, all cases that use virtualized network cards will be affected in the same way.
> It is reasonable to use the "CONFIG_XFRM_USER=y" to check.

It looks like it's possible to use either a two-node setup or a virtual network
card, but I haven't dug into the LTP test setup to trace exactly how the system
distinguishes between these two setups.  In the case of a two-node setup,
the test would not be dependent on CONFIG_XFRM_USER (from what I can see,
but I'm still analyzing this).  Currently, I don't see how the call to 
'ip xfrm' would be avoided in the two-node setup, but maybe the script
works OK even if those commands fail.  In any event, as it is currently
written the Fuego LTP test only uses the single-node setup, so the dependency
on CONFIG_XFRM_USER might be OK.

Sorry this has been so slow - I've been tied up with other issues this last week.
I continue to look at this.
 -- Tim

> 
> The following is my guess:
> There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
> when initializing the virtualized network card, using "ip xfrm" is necessary,
> because many ipsec-related content need to be tested in the network test set (at least 504 cases).
> 
> *1:
> tst_init_iface()
> {
>          local type="${1:-lhost}"
>          local link_num="${2:-0}"
>          local iface="$(tst_iface $type $link_num)"
>          tst_resm TINFO "initialize '$type' '$iface' interface"
> 
>          if [ "$type" = "lhost" ]; then
>                  ip xfrm policy flush || return $?
>                  ip xfrm state flush || return $?
>                  ip link set $iface down || return $?
>                  ip route flush dev $iface || return $?
>                  ip addr flush dev $iface || return $?
>                  ip link set $iface up
>                  return $?
>          fi
> 
>          tst_rhost_run -c "ip xfrm policy flush" || return $?
>          tst_rhost_run -c "ip xfrm state flush" || return $?
>          tst_rhost_run -c "ip link set $iface down" || return $?
>          tst_rhost_run -c "ip route flush dev $iface" || return $?
>          tst_rhost_run -c "ip addr flush dev $iface" || return $?
>          tst_rhost_run -c "ip link set $iface up"
> }
> 
> Thanks,
>  -- QiuTingting
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Thursday, July 9, 2020 6:13 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set
> 
> This one doesn't look right. See comments below.
> 
> > -----Original Message-----
> > From: qiutt
> >
> > for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc
> > net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip
> > net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp
> > net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp
> > net_stress.multicast net.tcp_cmds net.tirpc_tests
> >
> > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > ---
> >  tests/Functional.LTP/fuego_test.sh | 59
> > ++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 59 insertions(+)
> >
> > diff --git a/tests/Functional.LTP/fuego_test.sh
> > b/tests/Functional.LTP/fuego_test.sh
> > index 156fd39..ab4165d 100755
> > --- a/tests/Functional.LTP/fuego_test.sh
> > +++ b/tests/Functional.LTP/fuego_test.sh
> > @@ -251,6 +251,65 @@ function test_pre_check {
> >      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
> >      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> >
> > +    # some CONFIGs do not set in m3ulcb
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01
> vlan03"
> > # for net.features
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602
> > + sendfile601 traceroute601 dhcpd6" # for net.ipv6
> 
> Do ping and traceroute depend on CONFIG_XFRM_USER=y?
> That doesn't seem right.
> 
> Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set
> > + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> > rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create
> > rpc_clnt_destroy \
> > +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create
> > + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror
> > + rpc_clnt_sperrno
> > rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> > +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth
> > + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> > rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc
> > rpc_clnt_freeres \
> > +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> > test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect \
> > +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg
> > + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> > test_1_to_1_sendto test_1_to_1_shutdown test_1_to_1_socket_bind_listen
> > \
> > +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort
> > + test_assoc_shutdown test_autoclose test_basic test_basic_v6
> > + test_connect
> > test_connectx test_fragments test_fragments_v6 test_getname \
> > +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff
> > + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> > test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style \
> > +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for
> > + net.sctp
> 
> A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
> Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "broken_ip4-version
> > + broken_ip4-ihl broken_ip4-totlen broken_ip4-fragment
> > broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr
> > broken_ip6-dstaddr \
> > +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change
> > + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> > route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> > +    if6-addr-addlarge if6-route-adddel if6-route-addlarge
> > + if6-mtu-change" # for net_stress.interface
> 
> Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01
> > + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> > dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 dccp4_ipsec10
> > \
> > +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14
> > + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> > dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 dccp4_ipsec23
> > \
> > +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27
> > + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> > dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 dccp4_ipsec36
> > \
> > +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04
> > + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> > dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 dccp6_ipsec13
> > \
> > +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17
> > + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> > dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 dccp6_ipsec26
> > \
> > +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30
> > + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> > dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp
> 
> This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01
> > + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> > uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> > +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > + icmp4-uni-basic15
> > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> > +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > + icmp4-uni-basic25
> > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> > +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > + icmp6-uni-basic09
> > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> > +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > + icmp6-uni-basic19
> > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> > +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01
> > + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> > tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> > +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 tcp4_ipsec16
> > + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> > tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> > +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 tcp4_ipsec30
> > + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> > tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> > +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 tcp6_ipsec08
> > + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> > tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> > +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 tcp6_ipsec22
> > + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> > tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> > +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01
> > + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> > udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10
> > udp4_ipsec11 \
> > +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 udp4_ipsec16
> > + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> > udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> > +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 udp4_ipsec30
> > + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> > udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> > +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 udp6_ipsec08
> > + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> > udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> > +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 udp6_ipsec22
> > + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> > udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> > +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01
> > + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> > sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 sctp4_ipsec10
> > \
> > +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14
> > + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18
> > + sctp4_ipsec19
> > sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> > +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27
> > + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31
> > + sctp4_ipsec32
> > sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> > +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04
> > + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08
> > + sctp6_ipsec09
> > sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> > +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17
> > + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21
> > + sctp6_ipsec22
> > sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> > +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30
> > + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34
> > + sctp6_ipsec35
> > sctp6_ipsec36" # for net_stress.ipsec_sctp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "mcast4-group-single-socket mcast4-group-multiple-socket
> > + mcast4-group-same-group
> > mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> > +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03
> > + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 mcast6-group-
> > single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> > +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02
> > + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> > queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for
> > net_stress.multicast
> 
> A lot of these lines are very long, and the dependency seems a bit sketch.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tirpc_rpcb_getaddr
> > + tirpc_rpcb_getmaps tirpc_authdes_create
> > tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> > +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror
> > + tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> > tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> > +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset
> > + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call
> > + tirpc_clnt_tp_create
> > tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> > +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg
> > + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp
> > + tirpc_clnt_create
> > tirpc_clnt_create_timed tirpc_svc_create \
> > +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" #
> > + for net.tirpc_tests
> > +
> >      echo "Tests skipped depending on the architecture"
> >      if [ "$ARCHITECTURE" != "s390" ]; then
> >          skip_tests "sbrk03" # Only works in 32bit on s390 series
> > system
> > --
> > 2.11.0
> 
> I have too many questions about this one.  I did NOT apply it.
>  -- Tim
> 
> 
> 
> 

Re: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set

[Bird, Tim]

> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> Sent: Wednesday, July 29, 2020 11:49 PM
> 
> Hi,Tim
> 
> Cases such as ping, tcp, udp and traceroute are all based on the virtual network card.
> Initializing the virtual network card is implemented by the common script(test_net.sh).
> The code is as follows(*1), using "ip xfrm" to build the ipsec tunnel, and the support of "ip xfrm" depends on "CONFIG_XFRM_USER=y".
> Therefore, all cases that use virtualized network cards will be affected in the same way.
> It is reasonable to use the "CONFIG_XFRM_USER=y" to check.

I found some problems with this approach.  I have a board where the kernel has configuration
CONFIG_XFRM_USER=m, and the module xfrm_user.ko is loaded.

I specifically tested dccp01 in that configuration, and the test worked.  So the method
of using 'skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 ...' is not correct.
It will skip the test even though my machine has xfrm_user capabilities.

Also, I don't think that xfrm_user capabilities are actually required for "dccp01".  I disabled
the xfrm_user capabilities on my machine, by doing
   rmmod xfrm_user ; \ 
   cd /usr/lib/{kernel_version}/kernel/network/xfrm ; \
   mv xfrm_user.ko saved_xfrm_user.ko

(renaming the module to prevent autoloading it), and the dccp01 test still ran fine
on that machine.

Here is the result I got for dccp01, with xfrm_user capability disabled on my machine:
-----------
dccp01 1 TINFO: initialize 'lhost' 'ltp_ns_veth2' interface
dccp01 1 TINFO: add local addr 10.0.0.2/24
dccp01 1 TINFO: add local addr fd00:1:1:1::2/64
dccp01 1 TINFO: initialize 'rhost' 'ltp_ns_veth1' interface
dccp01 1 TINFO: add remote addr 10.0.0.1/24
dccp01 1 TINFO: add remote addr fd00:1:1:1::1/64
dccp01 1 TINFO: Network config (local -- remote):
dccp01 1 TINFO: ltp_ns_veth2 -- ltp_ns_veth1
dccp01 1 TINFO: 10.0.0.2/24 -- 10.0.0.1/24
dccp01 1 TINFO: fd00:1:1:1::2/64 -- fd00:1:1:1::1/64
dccp01 1 TINFO: timeout per run is 0h 5m 0s
dccp01 1 TINFO: run UDP
dccp01 1 TINFO: run server 'netstress -T udp -D ltp_ns_veth1 -R 500000 -B /home/fuego/fuego.Functional.LTP/tmp/ltp-NXKGsF5Jee/LTP_dccp01.MRd75dTRu1'
dccp01 1 TINFO: run client 'netstress -l -T udp -H 10.0.0.1 -D ltp_ns_veth2 -a 2 -r 10000 -d tst_netload.res -g 53124'
dccp01 1 TPASS: netstress passed, time spent '1385' ms
dccp01 2 TINFO: compare UDP/DCCP performance
dccp01 2 TINFO: run server 'netstress -T dccp -D ltp_ns_veth1 -R 500000 -B /home/fuego/fuego.Functional.LTP/tmp/ltp-NXKGsF5Jee/LTP_dccp01.MRd75dTRu1'
dccp01 2 TINFO: run client 'netstress -l -T dccp -H 10.0.0.1 -D ltp_ns_veth2 -a 2 -r 10000 -d tst_netload.res -g 42547'
dccp01 2 TPASS: netstress passed, time spent '1822' ms
dccp01 2 TPASS: DCCP performance -24 % in range -100 ... 100 %
dccp01 3 TINFO: compare UDP/UDP-Lite performance
dccp01 3 TINFO: run server 'netstress -T udp_lite -D ltp_ns_veth1 -R 500000 -B /home/fuego/fuego.Functional.LTP/tmp/ltp-NXKGsF5Jee/LTP_dccp01.MRd75dTRu1'
dccp01 3 TINFO: run client 'netstress -l -T udp_lite -H 10.0.0.1 -D ltp_ns_veth2 -a 2 -r 10000 -d tst_netload.res -g 39262'
dccp01 3 TPASS: netstress passed, time spent '1392' ms
dccp01 3 TPASS: UDP-Lite performance -1 % in range -100 ... 100 %

Summary:
passed   5
failed   0
skipped  0
warnings 0
incrementing stop
---------

I'm not sure how many of the other tests that you have identified as being dependent 
on CONFIG_XFRM_USER actually are.  Please see if you can identify some other attribute
of your system that is preventing these tests from working on your m3ulcb distro/board.

Also, I found that the net_tst.sh code from the latest LTP git repository has the usage of
'ip xfrm' slightly different than that shown in your code snippet below.  Here is the 
current code:

tst_init_iface()
{
        local type="${1:-lhost}"
        local link_num="${2:-0}"
        local iface="$(tst_iface $type $link_num)"
        tst_res_ TINFO "initialize '$type' '$iface' interface"

        if [ "$type" = "lhost" ]; then
                if ip xfrm state 1>/dev/null 2>&1; then
                        ip xfrm policy flush || return $?
                        ip xfrm state flush || return $?
                fi
                ip link set $iface down || return $?
                ip route flush dev $iface || return $?
                ip addr flush dev $iface || return $?
                ip link set $iface up
                return $?
        fi

        if tst_rhost_run -c "ip xfrm state 1>/dev/null 2>&1"; then
                tst_rhost_run -c "ip xfrm policy flush" || return $?
                tst_rhost_run -c "ip xfrm state flush" || return $?
        fi
        tst_rhost_run -c "ip link set $iface down" || return $?
        tst_rhost_run -c "ip route flush dev $iface" || return $?
        tst_rhost_run -c "ip addr flush dev $iface" || return $?
        tst_rhost_run -c "ip link set $iface up"
}

My LTP git repo is at this commit: 
$ git describe
20200515-164-ga6c8680

Note that in this code, the routine first detects whether the 'ip xfrm state' command works,
before trying to do the policy flush and state flush.  This makes me believe that the 'ip xfrm'
commands are optional for this code to run.  At least on my board they appear to be optional.

Just as an extra aside, I tried this code in fuego_test.sh, as an alternative to testing the kconfig:
    if ! cmd "ip xfrm state >/dev/null 2>&1" ; then
        echo "WARNING: 'ip xfrm' is not supported on the target."
        skip_tests "dccp01"
    fi

This mirrors the code used in tst_net.sh, and successfully skipped dccp01 when the 'ip xfrm state'
command failed on the target.  This technique may be useful for other dependencies.  (It shouldn't
be used here, since it doesn't appear that dccp01 is actually dependent on either
CONFIG_XFRM_USER="y" or "ip xfrm state"

Please try to find another method of handling the dependency for the tests you indicated.

Thanks,
 -- Tim

> 
> The following is my guess:
> There are too many cases which need to initialize the virtual network card to initial virtual network card by the common script.
> when initializing the virtualized network card, using "ip xfrm" is necessary,
> because many ipsec-related content need to be tested in the network test set (at least 504 cases).
> 
> *1:
> tst_init_iface()
> {
>          local type="${1:-lhost}"
>          local link_num="${2:-0}"
>          local iface="$(tst_iface $type $link_num)"
>          tst_resm TINFO "initialize '$type' '$iface' interface"
> 
>          if [ "$type" = "lhost" ]; then
>                  ip xfrm policy flush || return $?
>                  ip xfrm state flush || return $?
>                  ip link set $iface down || return $?
>                  ip route flush dev $iface || return $?
>                  ip addr flush dev $iface || return $?
>                  ip link set $iface up
>                  return $?
>          fi
> 
>          tst_rhost_run -c "ip xfrm policy flush" || return $?
>          tst_rhost_run -c "ip xfrm state flush" || return $?
>          tst_rhost_run -c "ip link set $iface down" || return $?
>          tst_rhost_run -c "ip route flush dev $iface" || return $?
>          tst_rhost_run -c "ip addr flush dev $iface" || return $?
>          tst_rhost_run -c "ip link set $iface up"
> }
> 
> Thanks,
>  -- QiuTingting
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Thursday, July 9, 2020 6:13 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 06/12] skip some cases because ONFIG_XFRM_USER is not set
> 
> This one doesn't look right. See comments below.
> 
> > -----Original Message-----
> > From: qiutt
> >
> > for m3ulcb, skip cases in  net.features net.multicast net.ipv6 net.rpc
> > net.rpc_tests net.sctp net_stress.appl net_stress.broken_ip
> > net_stress.interface net_stress.ipsec_dccp  net_stress.ipsec_icmp
> > net_stress.ipsec_tcp  net_stress.ipsec_udp  net_stress.ipsec_sctp
> > net_stress.multicast net.tcp_cmds net.tirpc_tests
> >
> > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > ---
> >  tests/Functional.LTP/fuego_test.sh | 59
> > ++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 59 insertions(+)
> >
> > diff --git a/tests/Functional.LTP/fuego_test.sh
> > b/tests/Functional.LTP/fuego_test.sh
> > index 156fd39..ab4165d 100755
> > --- a/tests/Functional.LTP/fuego_test.sh
> > +++ b/tests/Functional.LTP/fuego_test.sh
> > @@ -251,6 +251,65 @@ function test_pre_check {
> >      skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
> >      skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
> >
> > +    # some CONFIGs do not set in m3ulcb
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp01 dccp01_ipv6 sctp01 sctp01_ipv6 tcp_fastopen tcp_fastopen6 vlan01
> vlan03"
> > # for net.features
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping601 ping602
> > + sendfile601 traceroute601 dhcpd6" # for net.ipv6
> 
> Do ping and traceroute depend on CONFIG_XFRM_USER=y?
> That doesn't seem right.
> 
> Are ping601 and traceroute601 related to ipsec?  I didn't have time to dig into the tests to see if this was the case or not.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "mc_cmds mc_commo mc_member mc_opts" # for net.multicast
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc01" # for net.rpc
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "rpc_pmap_set
> > + rpc_pmap_unset rpc_pmap_getport rpc_pmap_getmaps
> > rpc_pmap_rmtcall rpc_clnt_broadcast rpc_svc_destroy rpc_svcfd_create
> > rpc_clnt_destroy \
> > +    rpc_clnt_create rpc_clnttcp_create rpc_clntudp_create
> > + rpc_clntudp_bufcreate rpc_clnt_perrno rpc_clnt_perror
> > + rpc_clnt_sperrno
> > rpc_clnt_sperror rpc_svcerr_noproc rpc_svcerr_noprog \
> > +    rpc_svcerr_progvers rpc_svcerr_systemerr rpc_svcerr_auth
> > + rpc_svcerr_weakauth rpc_xprt_register rpc_xprt_unregister
> > rpc_svc_register rpc_registerrpc rpc_clnt_call rpc_callrpc
> > rpc_clnt_freeres \
> > +    rpc_clnt_geterr rpc_clnt_control rpc_svc_getcaller rpc_svc_getargs rpc_svc_sendreply" # for net.rpc_tests
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "test_1_to_1_accept_close test_1_to_1_addrs test_1_to_1_connect
> > test_1_to_1_connectx test_1_to_1_events test_1_to_1_initmsg_connect \
> > +    test_1_to_1_nonblock test_1_to_1_recvfrom test_1_to_1_recvmsg
> > + test_1_to_1_rtoinfo test_1_to_1_send test_1_to_1_sendmsg
> > test_1_to_1_sendto test_1_to_1_shutdown test_1_to_1_socket_bind_listen
> > \
> > +    test_1_to_1_sockopt test_1_to_1_threads test_assoc_abort
> > + test_assoc_shutdown test_autoclose test_basic test_basic_v6
> > + test_connect
> > test_connectx test_fragments test_fragments_v6 test_getname \
> > +    test_getname_v6 test_inaddr_any test_inaddr_any_v6 test_peeloff
> > + test_peeloff_v6 test_recvmsg test_sctp_sendrecvmsg
> > test_sctp_sendrecvmsg_v6 test_sockopt test_sockopt_v6 test_tcp_style \
> > +    test_tcp_style_v6 test_timetolive test_timetolive_v6" # for
> > + net.sctp
> 
> A lot of these tests don't seem to be specific to CONFIG_XFRM_USER configuration.
> Are you just using CONFIG_XFRM_USER as a placeholder for networking config for the m3ulcb board?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ssh4-stress ssh6-stress" # for net_stress.appl
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "broken_ip4-version
> > + broken_ip4-ihl broken_ip4-totlen broken_ip4-fragment
> > broken_ip4-protcol broken_ip4-checksum broken_ip4-dstaddr
> > broken_ip6-dstaddr \
> > +    broken_ip6-nexthdr broken_ip6-plen broken_ip6-version" # for net_stress.broken_ip
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "if4-addr-change
> > + if4-updown if4-addr-adddel if4-addr-addlarge if4-route-adddel if4-
> > route-addlarge if4-mtu-change if6-updown if6-addr-adddel \
> > +    if6-addr-addlarge if6-route-adddel if6-route-addlarge
> > + if6-mtu-change" # for net_stress.interface
> 
> Same issue here.  Is CONFIG_XFRM_USER the best kernel config to check here?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "dccp4_ipsec01
> > + dccp4_ipsec02 dccp4_ipsec03 dccp4_ipsec04 dccp4_ipsec05
> > dccp4_ipsec06 dccp4_ipsec07 dccp4_ipsec08 dccp4_ipsec09 dccp4_ipsec10
> > \
> > +    dccp4_ipsec11 dccp4_ipsec12 dccp4_ipsec13 dccp4_ipsec14
> > + dccp4_ipsec15 dccp4_ipsec16 dccp4_ipsec17 dccp4_ipsec18
> > dccp4_ipsec19 dccp4_ipsec20 dccp4_ipsec21 dccp4_ipsec22 dccp4_ipsec23
> > \
> > +    dccp4_ipsec24 dccp4_ipsec25 dccp4_ipsec26 dccp4_ipsec27
> > + dccp4_ipsec28 dccp4_ipsec29 dccp4_ipsec30 dccp4_ipsec31
> > dccp4_ipsec32 dccp4_ipsec33 dccp4_ipsec34 dccp4_ipsec35 dccp4_ipsec36
> > \
> > +    dccp6_ipsec01 dccp6_ipsec02 dccp6_ipsec03 dccp6_ipsec04
> > + dccp6_ipsec05 dccp6_ipsec06 dccp6_ipsec07 dccp6_ipsec08
> > dccp6_ipsec09 dccp6_ipsec10 dccp6_ipsec11 dccp6_ipsec12 dccp6_ipsec13
> > \
> > +    dccp6_ipsec14 dccp6_ipsec15 dccp6_ipsec16 dccp6_ipsec17
> > + dccp6_ipsec18 dccp6_ipsec19 dccp6_ipsec20 dccp6_ipsec21
> > dccp6_ipsec22 dccp6_ipsec23 dccp6_ipsec24 dccp6_ipsec25 dccp6_ipsec26
> > \
> > +    dccp6_ipsec27 dccp6_ipsec28 dccp6_ipsec29 dccp6_ipsec30
> > + dccp6_ipsec31 dccp6_ipsec32 dccp6_ipsec33 dccp6_ipsec34
> > dccp6_ipsec35 dccp6_ipsec36" # for net_stress.ipsec_dccp
> 
> This is a LOT of dependencies!  Is there any way to use wildcards here, to avoid having to list all of these testcases individually?
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "icmp4-uni-basic01
> > + icmp4-uni-basic02 icmp4-uni-basic03 icmp4-uni-basic04 icmp4-
> > uni-basic05 icmp4-uni-basic06 icmp4-uni-basic07 icmp4-uni-basic08 \
> > +    icmp4-uni-basic09 icmp4-uni-basic10 icmp4-uni-basic11
> > + icmp4-uni-basic12 icmp4-uni-basic13 icmp4-uni-basic14
> > + icmp4-uni-basic15
> > icmp4-uni-basic16 icmp4-uni-basic17 icmp4-uni-basic18 \
> > +    icmp4-uni-basic19 icmp4-uni-basic20 icmp4-uni-basic21
> > + icmp4-uni-basic22 icmp4-uni-basic23 icmp4-uni-basic24
> > + icmp4-uni-basic25
> > icmp4-uni-basic26 icmp6-uni-basic01 icmp6-uni-basic02 \
> > +    icmp6-uni-basic03 icmp6-uni-basic04 icmp6-uni-basic05
> > + icmp6-uni-basic06 icmp6-uni-basic07 icmp6-uni-basic08
> > + icmp6-uni-basic09
> > icmp6-uni-basic10 icmp6-uni-basic11 icmp6-uni-basic12 \
> > +    icmp6-uni-basic13 icmp6-uni-basic14 icmp6-uni-basic15
> > + icmp6-uni-basic16 icmp6-uni-basic17 icmp6-uni-basic18
> > + icmp6-uni-basic19
> > icmp6-uni-basic20 icmp6-uni-basic21 icmp6-uni-basic22 \
> > +    icmp6-uni-basic23 icmp6-uni-basic24 icmp6-uni-basic25 icmp6-uni-basic26" # for net_stress.ipsec_icmp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tcp4_ipsec01
> > + tcp4_ipsec02 tcp4_ipsec03 tcp4_ipsec04 tcp4_ipsec05 tcp4_ipsec06
> > tcp4_ipsec07 tcp4_ipsec08 tcp4_ipsec09 tcp4_ipsec10 tcp4_ipsec11 \
> > +    tcp4_ipsec12 tcp4_ipsec13 tcp4_ipsec14 tcp4_ipsec15 tcp4_ipsec16
> > + tcp4_ipsec17 tcp4_ipsec18 tcp4_ipsec19 tcp4_ipsec20
> > tcp4_ipsec21 tcp4_ipsec22 tcp4_ipsec23 tcp4_ipsec24 tcp4_ipsec25 \
> > +    tcp4_ipsec26 tcp4_ipsec27 tcp4_ipsec28 tcp4_ipsec29 tcp4_ipsec30
> > + tcp4_ipsec31 tcp4_ipsec32 tcp4_ipsec33 tcp4_ipsec34
> > tcp4_ipsec35 tcp4_ipsec36 tcp6_ipsec01 tcp6_ipsec02 tcp6_ipsec03 \
> > +    tcp6_ipsec04 tcp6_ipsec05 tcp6_ipsec06 tcp6_ipsec07 tcp6_ipsec08
> > + tcp6_ipsec09 tcp6_ipsec10 tcp6_ipsec11 tcp6_ipsec12
> > tcp6_ipsec13 tcp6_ipsec14 tcp6_ipsec15 tcp6_ipsec16 tcp6_ipsec17 \
> > +    tcp6_ipsec18 tcp6_ipsec19 tcp6_ipsec20 tcp6_ipsec21 tcp6_ipsec22
> > + tcp6_ipsec23 tcp6_ipsec24 tcp6_ipsec25 tcp6_ipsec26
> > tcp6_ipsec27 tcp6_ipsec28 tcp6_ipsec29 tcp6_ipsec30 tcp6_ipsec31 \
> > +    tcp6_ipsec32 tcp6_ipsec33 tcp6_ipsec34 tcp6_ipsec35 tcp6_ipsec36" # for net_stress.ipsec_tcp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "udp4_ipsec01
> > + udp4_ipsec02 udp4_ipsec03 udp4_ipsec04 udp4_ipsec05
> > udp4_ipsec06 udp4_ipsec07 udp4_ipsec08 udp4_ipsec09 udp4_ipsec10
> > udp4_ipsec11 \
> > +    udp4_ipsec12 udp4_ipsec13 udp4_ipsec14 udp4_ipsec15 udp4_ipsec16
> > + udp4_ipsec17 udp4_ipsec18 udp4_ipsec19 udp4_ipsec20
> > udp4_ipsec21 udp4_ipsec22 udp4_ipsec23 udp4_ipsec24 udp4_ipsec25 \
> > +    udp4_ipsec26 udp4_ipsec27 udp4_ipsec28 udp4_ipsec29 udp4_ipsec30
> > + udp4_ipsec31 udp4_ipsec32 udp4_ipsec33 udp4_ipsec34
> > udp4_ipsec35 udp4_ipsec36 udp6_ipsec01 udp6_ipsec02 udp6_ipsec03 \
> > +    udp6_ipsec04 udp6_ipsec05 udp6_ipsec06 udp6_ipsec07 udp6_ipsec08
> > + udp6_ipsec09 udp6_ipsec10 udp6_ipsec11 udp6_ipsec12
> > udp6_ipsec13 udp6_ipsec14 udp6_ipsec15 udp6_ipsec16 udp6_ipsec17 \
> > +    udp6_ipsec18 udp6_ipsec19 udp6_ipsec20 udp6_ipsec21 udp6_ipsec22
> > + udp6_ipsec23 udp6_ipsec24 udp6_ipsec25 udp6_ipsec26
> > udp6_ipsec27 udp6_ipsec28 udp6_ipsec29 udp6_ipsec30 udp6_ipsec31 \
> > +    udp6_ipsec32 udp6_ipsec33 udp6_ipsec34 udp6_ipsec35 udp6_ipsec36" # for net_stress.ipsec_udp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "sctp4_ipsec01
> > + sctp4_ipsec02 sctp4_ipsec03 sctp4_ipsec04 sctp4_ipsec05
> > sctp4_ipsec06 sctp4_ipsec07 sctp4_ipsec08 sctp4_ipsec09 sctp4_ipsec10
> > \
> > +    sctp4_ipsec11 sctp4_ipsec12 sctp4_ipsec13 sctp4_ipsec14
> > + sctp4_ipsec15 sctp4_ipsec16 sctp4_ipsec17 sctp4_ipsec18
> > + sctp4_ipsec19
> > sctp4_ipsec20 sctp4_ipsec21 sctp4_ipsec22 sctp4_ipsec23 \
> > +    sctp4_ipsec24 sctp4_ipsec25 sctp4_ipsec26 sctp4_ipsec27
> > + sctp4_ipsec28 sctp4_ipsec29 sctp4_ipsec30 sctp4_ipsec31
> > + sctp4_ipsec32
> > sctp4_ipsec33 sctp4_ipsec34 sctp4_ipsec35 sctp4_ipsec36 \
> > +    sctp6_ipsec01 sctp6_ipsec02 sctp6_ipsec03 sctp6_ipsec04
> > + sctp6_ipsec05 sctp6_ipsec06 sctp6_ipsec07 sctp6_ipsec08
> > + sctp6_ipsec09
> > sctp6_ipsec10 sctp6_ipsec11 sctp6_ipsec12 sctp6_ipsec13 \
> > +    sctp6_ipsec14 sctp6_ipsec15 sctp6_ipsec16 sctp6_ipsec17
> > + sctp6_ipsec18 sctp6_ipsec19 sctp6_ipsec20 sctp6_ipsec21
> > + sctp6_ipsec22
> > sctp6_ipsec23 sctp6_ipsec24 sctp6_ipsec25 sctp6_ipsec26 \
> > +    sctp6_ipsec27 sctp6_ipsec28 sctp6_ipsec29 sctp6_ipsec30
> > + sctp6_ipsec31 sctp6_ipsec32 sctp6_ipsec33 sctp6_ipsec34
> > + sctp6_ipsec35
> > sctp6_ipsec36" # for net_stress.ipsec_sctp
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y"
> > + "mcast4-group-single-socket mcast4-group-multiple-socket
> > + mcast4-group-same-group
> > mcast4-group-source-filter mcast4-pktfld01 mcast4-pktfld02 \
> > +    mcast4-queryfld01 mcast4-queryfld02 mcast4-queryfld03
> > + mcast4-queryfld04 mcast4-queryfld05 mcast4-queryfld06 mcast6-group-
> > single-socket mcast6-group-multiple-socket mcast6-group-same-group \
> > +    mcast6-group-source-filter mcast6-pktfld01 mcast6-pktfld02
> > + mcast6-queryfld01 mcast6-queryfld02 mcast6-queryfld03 mcast6-
> > queryfld04 mcast6-queryfld05 mcast6-queryfld06" # for
> > net_stress.multicast
> 
> A lot of these lines are very long, and the dependency seems a bit sketch.
> 
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "ping01 ping02 sendfile iproute traceroute01 dhcpd" # for net.tcp_cmds
> > +    skip_if_kconfig_differs "CONFIG_XFRM_USER=y" "tirpc_rpcb_getaddr
> > + tirpc_rpcb_getmaps tirpc_authdes_create
> > tirpc_authdes_seccreate tirpc_clnt_dg_create tirpc_clnt_vc_create \
> > +    tirpc_bottomlevel_clnt_call tirpc_clnt_perrno tirpc_clnt_perror
> > + tirpc_svcerr_noproc tirpc_svcerr_noprog tirpc_svcerr_progvers
> > tirpc_svcerr_systemerr tirpc_svcerr_weakauth \
> > +    tirpc_clnt_tli_create tirpc_rpcb_set tirpc_rpcb_unset
> > + tirpc_rpcb_rmtcall tirpc_svc_reg tirpc_expertlevel_clnt_call
> > + tirpc_clnt_tp_create
> > tirpc_clnt_tp_create_timed tirpc_svc_tp_create \
> > +    tirpc_interlevel_clnt_call tirpc_clnt_control tirpc_rpc_reg
> > + tirpc_rpc_call tirpc_rpc_broadcast tirpc_rpc_broadcast_exp
> > + tirpc_clnt_create
> > tirpc_clnt_create_timed tirpc_svc_create \
> > +    tirpc_toplevel_clnt_call tirpc_clnt_destroy tirpc_svc_destroy" #
> > + for net.tirpc_tests
> > +
> >      echo "Tests skipped depending on the architecture"
> >      if [ "$ARCHITECTURE" != "s390" ]; then
> >          skip_tests "sbrk03" # Only works in 32bit on s390 series
> > system
> > --
> > 2.11.0
> 
> I have too many questions about this one.  I did NOT apply it.
>  -- Tim
> 
> 
> 
> 

[Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[qiutt]

for aarch64(m3ulcb) skip cases in net.tcp_cmds

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index ab4165d..2254474 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -352,6 +352,10 @@ function test_pre_check {
     if [ "$ARCHITECTURE" != "i386" ]; then
         skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
     fi
+
+    if [ "$ARCHITECTURE" == "aarch64" ]; then
+        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
+    fi
 }
 
 function test_build {
-- 
2.11.0

Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[Bird, Tim]

> -----Original Message-----
> From:  qiutt
> 
> for aarch64(m3ulcb) skip cases in net.tcp_cmds
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
> index ab4165d..2254474 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -352,6 +352,10 @@ function test_pre_check {
>      if [ "$ARCHITECTURE" != "i386" ]; then
>          skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
>      fi
> +
> +    if [ "$ARCHITECTURE" == "aarch64" ]; then
> +        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
> +    fi

This isn't right.  It may be true that netstat -s is not supported on m3ulcb,
but it is supported just fine on other aarch64 platforms.

What is the issue with supporting 'netstat -s' on m3ulcb?

>  }
> 
>  function test_build {
> --
> 2.11.0
> 
> 
> 
> _______________________________________________
> Fuego mailing list
> Fuego@lists.linuxfoundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/fuego

Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[Qiu, Tingting]

Hi,Tim

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Thursday, July 9, 2020 6:20 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support



> -----Original Message-----
> From:  qiutt
> 
> for aarch64(m3ulcb) skip cases in net.tcp_cmds
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh 
> b/tests/Functional.LTP/fuego_test.sh
> index ab4165d..2254474 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -352,6 +352,10 @@ function test_pre_check {
>      if [ "$ARCHITECTURE" != "i386" ]; then
>          skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
>      fi
> +
> +    if [ "$ARCHITECTURE" == "aarch64" ]; then
> +        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
> +    fi

This isn't right.  It may be true that netstat -s is not supported on m3ulcb, but it is supported just fine on other aarch64 platforms.

What is the issue with supporting 'netstat -s' on m3ulcb?
->
The patch was made for m3ulcb, and m3ulcb used the AGL system. 
The "netstat" command used was provided by "busybox"(*1). 
The error of "netstat -s" refer to (*2).
Check the code(aarch64-agl-linux/busybox/1.29.3-r0/busybox-1.29.3/networking/netstat.c), the contents refer to (*3). 
According to the code, you can know that the netstat provided by busybox only supports the parameter "-laentuwxrWp", 
and does not support the parameter "-s".

(*1)m3ulcb:~# which netstat
/bin/netstat
m3ulcb:~# ls -al /bin/netstat
lrwxrwxrwx 1 root root 19 Feb 25  2020 /bin/netstat -> /bin/busybox.nosuid

(*2)m3ulcb:~# netstat -s
netstat: invalid option -- 's'
BusyBox v1.29.3 (2020-01-16 16:59:41 UTC) multi-call binary.
Usage: netstat [-ral] [-tuwx] [-en]

(*3)
//usage:#define netstat_trivial_usage
//usage:       "[-"IF_ROUTE("r")"al] [-tuwx] [-en"IF_FEATURE_NETSTAT_WIDE("W")IF_FEATURE_NETSTAT_PRG("p")"]"
//usage:#define netstat_full_usage "\n\n"
//usage:       "Display networking information\n"
//usage:        IF_ROUTE(
//usage:     "\n        -r      Routing table"
//usage:        )
//usage:     "\n        -a      All sockets"
//usage:     "\n        -l      Listening sockets"
//usage:     "\n                Else: connected sockets"
//usage:     "\n        -t      TCP sockets"
//usage:     "\n        -u      UDP sockets"
//usage:     "\n        -w      Raw sockets"
//usage:     "\n        -x      Unix sockets"
//usage:     "\n                Else: all socket types"
//usage:     "\n        -e      Other/more information"
//usage:     "\n        -n      Don't resolve names"
//usage:        IF_FEATURE_NETSTAT_WIDE(
//usage:     "\n        -W      Wide display"
//usage:        )
//usage:        IF_FEATURE_NETSTAT_PRG(
//usage:     "\n        -p      Show PID/program name for sockets"
//usage:        )

#define NETSTAT_OPTS "laentuwx" \
        IF_ROUTE(               "r") \
        IF_FEATURE_NETSTAT_WIDE("W") \
        IF_FEATURE_NETSTAT_PRG( "p")
...


Thanks,
 -- QiuTingting

>  }
> 
>  function test_build {
> --
> 2.11.0
> 
> 
> 
> _______________________________________________
> Fuego mailing list
> Fuego@lists.linuxfoundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/fuego

Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[Bird, Tim]

> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Thursday, July 9, 2020 6:20 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support
> 
> > -----Original Message-----
> > From:  qiutt
> >
> > for aarch64(m3ulcb) skip cases in net.tcp_cmds
> >
> > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > ---
> >  tests/Functional.LTP/fuego_test.sh | 4 ++++
> >  1 file changed, 4 insertions(+)
> >
> > diff --git a/tests/Functional.LTP/fuego_test.sh
> > b/tests/Functional.LTP/fuego_test.sh
> > index ab4165d..2254474 100755
> > --- a/tests/Functional.LTP/fuego_test.sh
> > +++ b/tests/Functional.LTP/fuego_test.sh
> > @@ -352,6 +352,10 @@ function test_pre_check {
> >      if [ "$ARCHITECTURE" != "i386" ]; then
> >          skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
> >      fi
> > +
> > +    if [ "$ARCHITECTURE" == "aarch64" ]; then
> > +        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
> > +    fi
> 
> This isn't right.  It may be true that netstat -s is not supported on m3ulcb, but it is supported just fine on other aarch64 platforms.
> 
> What is the issue with supporting 'netstat -s' on m3ulcb?
> ->
> The patch was made for m3ulcb, and m3ulcb used the AGL system.
> The "netstat" command used was provided by "busybox"(*1).
> The error of "netstat -s" refer to (*2).
> Check the code(aarch64-agl-linux/busybox/1.29.3-r0/busybox-1.29.3/networking/netstat.c), the contents refer to (*3).
> According to the code, you can know that the netstat provided by busybox only supports the parameter "-laentuwxrWp",
> and does not support the parameter "-s".
> 
> (*1)m3ulcb:~# which netstat
> /bin/netstat
> m3ulcb:~# ls -al /bin/netstat
> lrwxrwxrwx 1 root root 19 Feb 25  2020 /bin/netstat -> /bin/busybox.nosuid
> 
> (*2)m3ulcb:~# netstat -s
> netstat: invalid option -- 's'
> BusyBox v1.29.3 (2020-01-16 16:59:41 UTC) multi-call binary.
> Usage: netstat [-ral] [-tuwx] [-en]
> 
> (*3)
> //usage:#define netstat_trivial_usage
> //usage:       "[-"IF_ROUTE("r")"al] [-tuwx] [-en"IF_FEATURE_NETSTAT_WIDE("W")IF_FEATURE_NETSTAT_PRG("p")"]"
> //usage:#define netstat_full_usage "\n\n"
> //usage:       "Display networking information\n"
> //usage:        IF_ROUTE(
> //usage:     "\n        -r      Routing table"
> //usage:        )
> //usage:     "\n        -a      All sockets"
> //usage:     "\n        -l      Listening sockets"
> //usage:     "\n                Else: connected sockets"
> //usage:     "\n        -t      TCP sockets"
> //usage:     "\n        -u      UDP sockets"
> //usage:     "\n        -w      Raw sockets"
> //usage:     "\n        -x      Unix sockets"
> //usage:     "\n                Else: all socket types"
> //usage:     "\n        -e      Other/more information"
> //usage:     "\n        -n      Don't resolve names"
> //usage:        IF_FEATURE_NETSTAT_WIDE(
> //usage:     "\n        -W      Wide display"
> //usage:        )
> //usage:        IF_FEATURE_NETSTAT_PRG(
> //usage:     "\n        -p      Show PID/program name for sockets"
> //usage:        )
> 
> #define NETSTAT_OPTS "laentuwx" \
>         IF_ROUTE(               "r") \
>         IF_FEATURE_NETSTAT_WIDE("W") \
>         IF_FEATURE_NETSTAT_PRG( "p")
> ...
> 

I see.  I was able to reproduce a problem running the netstat01 sub-test of 'net.tcp_cmds'
test in LTP, on a machine that only had busybox netstat.

This should not be dependent on the machine architecture, which is unrelated, but rather
on whether the machine supports 'netstat -s'.  But that's actually one of the functions
of the netstat01 test in LTP (to test 'netstat -s' support).  In this case, the test is reporting
correctly that the m3ulcb board doesn't support that option.  So the test is correct.
Other Fuego users of the test may wish to see that problem reported.  It is not appropriate
to mask out this correct bug report by adding to a skiplist that applies to everyone.

I'm not sure exactly what you are trying to accomplish (maybe hiding this bug
so that it doesn't show up for this board, because your organization has decided
to ignore it?).  However, if you want to skip the test, then
I would recommend adding this to a board-specific LTP skiplist.

In your m3ulcb.board file, you should add:
FUNCTIONAL_LTP_BOARD_SKIPLIST="netstat01"

Let me know if this does not solve your problem.
 -- Tim

Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[Qiu, Tingting]

Tim

Thanks for your suggestion.
Using a board-specific LTP skiplist is a good idear.

Thanks,
  -- QiuTingting

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Friday, August 7, 2020 7:28 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
Cc: fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support



> -----Original Message-----
> From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Thursday, July 9, 2020 6:20 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; 
> fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not 
> support
> 
> > -----Original Message-----
> > From:  qiutt
> >
> > for aarch64(m3ulcb) skip cases in net.tcp_cmds
> >
> > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > ---
> >  tests/Functional.LTP/fuego_test.sh | 4 ++++
> >  1 file changed, 4 insertions(+)
> >
> > diff --git a/tests/Functional.LTP/fuego_test.sh
> > b/tests/Functional.LTP/fuego_test.sh
> > index ab4165d..2254474 100755
> > --- a/tests/Functional.LTP/fuego_test.sh
> > +++ b/tests/Functional.LTP/fuego_test.sh
> > @@ -352,6 +352,10 @@ function test_pre_check {
> >      if [ "$ARCHITECTURE" != "i386" ]; then
> >          skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
> >      fi
> > +
> > +    if [ "$ARCHITECTURE" == "aarch64" ]; then
> > +        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
> > +    fi
> 
> This isn't right.  It may be true that netstat -s is not supported on m3ulcb, but it is supported just fine on other aarch64 platforms.
> 
> What is the issue with supporting 'netstat -s' on m3ulcb?
> ->
> The patch was made for m3ulcb, and m3ulcb used the AGL system.
> The "netstat" command used was provided by "busybox"(*1).
> The error of "netstat -s" refer to (*2).
> Check the code(aarch64-agl-linux/busybox/1.29.3-r0/busybox-1.29.3/networking/netstat.c), the contents refer to (*3).
> According to the code, you can know that the netstat provided by 
> busybox only supports the parameter "-laentuwxrWp", and does not support the parameter "-s".
> 
> (*1)m3ulcb:~# which netstat
> /bin/netstat
> m3ulcb:~# ls -al /bin/netstat
> lrwxrwxrwx 1 root root 19 Feb 25  2020 /bin/netstat -> 
> /bin/busybox.nosuid
> 
> (*2)m3ulcb:~# netstat -s
> netstat: invalid option -- 's'
> BusyBox v1.29.3 (2020-01-16 16:59:41 UTC) multi-call binary.
> Usage: netstat [-ral] [-tuwx] [-en]
> 
> (*3)
> //usage:#define netstat_trivial_usage
> //usage:       "[-"IF_ROUTE("r")"al] [-tuwx] [-en"IF_FEATURE_NETSTAT_WIDE("W")IF_FEATURE_NETSTAT_PRG("p")"]"
> //usage:#define netstat_full_usage "\n\n"
> //usage:       "Display networking information\n"
> //usage:        IF_ROUTE(
> //usage:     "\n        -r      Routing table"
> //usage:        )
> //usage:     "\n        -a      All sockets"
> //usage:     "\n        -l      Listening sockets"
> //usage:     "\n                Else: connected sockets"
> //usage:     "\n        -t      TCP sockets"
> //usage:     "\n        -u      UDP sockets"
> //usage:     "\n        -w      Raw sockets"
> //usage:     "\n        -x      Unix sockets"
> //usage:     "\n                Else: all socket types"
> //usage:     "\n        -e      Other/more information"
> //usage:     "\n        -n      Don't resolve names"
> //usage:        IF_FEATURE_NETSTAT_WIDE(
> //usage:     "\n        -W      Wide display"
> //usage:        )
> //usage:        IF_FEATURE_NETSTAT_PRG(
> //usage:     "\n        -p      Show PID/program name for sockets"
> //usage:        )
> 
> #define NETSTAT_OPTS "laentuwx" \
>         IF_ROUTE(               "r") \
>         IF_FEATURE_NETSTAT_WIDE("W") \
>         IF_FEATURE_NETSTAT_PRG( "p")
> ...
> 

I see.  I was able to reproduce a problem running the netstat01 sub-test of 'net.tcp_cmds'
test in LTP, on a machine that only had busybox netstat.

This should not be dependent on the machine architecture, which is unrelated, but rather on whether the machine supports 'netstat -s'.  But that's actually one of the functions of the netstat01 test in LTP (to test 'netstat -s' support).  In this case, the test is reporting correctly that the m3ulcb board doesn't support that option.  So the test is correct.
Other Fuego users of the test may wish to see that problem reported.  It is not appropriate to mask out this correct bug report by adding to a skiplist that applies to everyone.

I'm not sure exactly what you are trying to accomplish (maybe hiding this bug so that it doesn't show up for this board, because your organization has decided to ignore it?).  However, if you want to skip the test, then I would recommend adding this to a board-specific LTP skiplist.

In your m3ulcb.board file, you should add:
FUNCTIONAL_LTP_BOARD_SKIPLIST="netstat01"

Let me know if this does not solve your problem.
 -- Tim

Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[daniel.sangorrin]

Hi Tim, Qiu

I am guessing this work is being done for the old 2018 LTP tarball. However, I think that LTP has changed quite a few things since then (I need to catchup on that). I think it is a good time to update the test to use the latest LTP and the new features that were added, what do you think? 

Thanks,
Daniel


> -----Original Message-----
> From: Fuego <fuego-bounces@lists.linuxfoundation.org> On Behalf Of Qiu, Tingting
> Sent: Friday, August 7, 2020 10:42 AM
> To: Bird, Tim <Tim.Bird@sony.com>
> Cc: fuego@lists.linuxfoundation.org
> Subject: Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support
> 
> Tim
> 
> Thanks for your suggestion.
> Using a board-specific LTP skiplist is a good idear.
> 
> Thanks,
>   -- QiuTingting
> 
> -----Original Message-----
> From: Bird, Tim [mailto:Tim.Bird@sony.com]
> Sent: Friday, August 7, 2020 7:28 AM
> To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
> Cc: fuego@lists.linuxfoundation.org
> Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support
> 
> 
> 
> > -----Original Message-----
> > From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> >
> > -----Original Message-----
> > From: Bird, Tim [mailto:Tim.Bird@sony.com]
> > Sent: Thursday, July 9, 2020 6:20 AM
> > To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>;
> > fuego@lists.linuxfoundation.org
> > Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not
> > support
> >
> > > -----Original Message-----
> > > From:  qiutt
> > >
> > > for aarch64(m3ulcb) skip cases in net.tcp_cmds
> > >
> > > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > > ---
> > >  tests/Functional.LTP/fuego_test.sh | 4 ++++
> > >  1 file changed, 4 insertions(+)
> > >
> > > diff --git a/tests/Functional.LTP/fuego_test.sh
> > > b/tests/Functional.LTP/fuego_test.sh
> > > index ab4165d..2254474 100755
> > > --- a/tests/Functional.LTP/fuego_test.sh
> > > +++ b/tests/Functional.LTP/fuego_test.sh
> > > @@ -352,6 +352,10 @@ function test_pre_check {
> > >      if [ "$ARCHITECTURE" != "i386" ]; then
> > >          skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
> > >      fi
> > > +
> > > +    if [ "$ARCHITECTURE" == "aarch64" ]; then
> > > +        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
> > > +    fi
> >
> > This isn't right.  It may be true that netstat -s is not supported on m3ulcb, but it is supported just fine on other aarch64 platforms.
> >
> > What is the issue with supporting 'netstat -s' on m3ulcb?
> > ->
> > The patch was made for m3ulcb, and m3ulcb used the AGL system.
> > The "netstat" command used was provided by "busybox"(*1).
> > The error of "netstat -s" refer to (*2).
> > Check the code(aarch64-agl-linux/busybox/1.29.3-r0/busybox-1.29.3/networking/netstat.c), the contents refer to (*3).
> > According to the code, you can know that the netstat provided by
> > busybox only supports the parameter "-laentuwxrWp", and does not support the parameter "-s".
> >
> > (*1)m3ulcb:~# which netstat
> > /bin/netstat
> > m3ulcb:~# ls -al /bin/netstat
> > lrwxrwxrwx 1 root root 19 Feb 25  2020 /bin/netstat ->
> > /bin/busybox.nosuid
> >
> > (*2)m3ulcb:~# netstat -s
> > netstat: invalid option -- 's'
> > BusyBox v1.29.3 (2020-01-16 16:59:41 UTC) multi-call binary.
> > Usage: netstat [-ral] [-tuwx] [-en]
> >
> > (*3)
> > //usage:#define netstat_trivial_usage
> > //usage:       "[-"IF_ROUTE("r")"al] [-tuwx] [-en"IF_FEATURE_NETSTAT_WIDE("W")IF_FEATURE_NETSTAT_PRG("p")"]"
> > //usage:#define netstat_full_usage "\n\n"
> > //usage:       "Display networking information\n"
> > //usage:        IF_ROUTE(
> > //usage:     "\n        -r      Routing table"
> > //usage:        )
> > //usage:     "\n        -a      All sockets"
> > //usage:     "\n        -l      Listening sockets"
> > //usage:     "\n                Else: connected sockets"
> > //usage:     "\n        -t      TCP sockets"
> > //usage:     "\n        -u      UDP sockets"
> > //usage:     "\n        -w      Raw sockets"
> > //usage:     "\n        -x      Unix sockets"
> > //usage:     "\n                Else: all socket types"
> > //usage:     "\n        -e      Other/more information"
> > //usage:     "\n        -n      Don't resolve names"
> > //usage:        IF_FEATURE_NETSTAT_WIDE(
> > //usage:     "\n        -W      Wide display"
> > //usage:        )
> > //usage:        IF_FEATURE_NETSTAT_PRG(
> > //usage:     "\n        -p      Show PID/program name for sockets"
> > //usage:        )
> >
> > #define NETSTAT_OPTS "laentuwx" \
> >         IF_ROUTE(               "r") \
> >         IF_FEATURE_NETSTAT_WIDE("W") \
> >         IF_FEATURE_NETSTAT_PRG( "p")
> > ...
> >
> 
> I see.  I was able to reproduce a problem running the netstat01 sub-test of 'net.tcp_cmds'
> test in LTP, on a machine that only had busybox netstat.
> 
> This should not be dependent on the machine architecture, which is unrelated, but rather on whether the machine supports 'netstat -s'.
> But that's actually one of the functions of the netstat01 test in LTP (to test 'netstat -s' support).  In this case, the test is reporting
> correctly that the m3ulcb board doesn't support that option.  So the test is correct.
> Other Fuego users of the test may wish to see that problem reported.  It is not appropriate to mask out this correct bug report by
> adding to a skiplist that applies to everyone.
> 
> I'm not sure exactly what you are trying to accomplish (maybe hiding this bug so that it doesn't show up for this board, because your
> organization has decided to ignore it?).  However, if you want to skip the test, then I would recommend adding this to a board-specific
> LTP skiplist.
> 
> In your m3ulcb.board file, you should add:
> FUNCTIONAL_LTP_BOARD_SKIPLIST="netstat01"
> 
> Let me know if this does not solve your problem.
>  -- Tim
> 
> 
> 
> 
> 
> _______________________________________________
> Fuego mailing list
> Fuego@lists.linuxfoundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/fuego

Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[Bird, Tim]

> -----Original Message-----
> From: daniel.sangorrin@toshiba.co.jp > 
> Hi Tim, Qiu
> 
> I am guessing this work is being done for the old 2018 LTP tarball. However, I think that LTP has changed quite a few things since then (I
> need to catchup on that). I think it is a good time to update the test to use the latest LTP and the new features that were added, what do
> you think?

I'm OK with this.  We could do this:

diff --git a/tests/Functional.LTP/spec.json b/tests/Functional.LTP/spec.json
index 7c409ff..388097c 100644
--- a/tests/Functional.LTP/spec.json
+++ b/tests/Functional.LTP/spec.json
@@ -75,6 +75,12 @@
             "tests": "cve",
             "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
             "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "latest": {
+            "gitrepo": "https://github.com/linux-test-project/ltp.git",
+            "tests": "syscalls",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
         }
-    }
+   }
 }

But that would only do one test from the latest (and it wouldn't be tied to a release of LTP).

Having been lurking on the LTP list for some time now, I have a couple of opinions:
 1. LTP top of tree moves and has bugs during development, so I would definitely want to
   specify a specific release commit, or use a snapshot release tarball
    - my preference is a snapshot tarball.  In general, I like avoiding network dependencies on
       external projects at test runtime, if possible.
 2. LTP has gotten a lot better over time, from what I can see.  I was initially hesitant to move forward
    to a new version, but I think that hesitation was misfounded.  I think Fuego is missing out on
    test improvements by not moving forward as LTP is improved.  It will cause churn in the testcase
    list produced between different versions of LTP, but I think that's a lesser problem than I expected.
    No one appears to be comparing LTP results to each other anyway.

So - sounds good to me!

Note that LTP just had a release this week, and I put a few new patches into Fuego related to that
release.  Now might be a good time to add a more current test option for LTP, and make it default.

> > -----Original Message-----
> > From: Fuego <fuego-bounces@lists.linuxfoundation.org> On Behalf Of Qiu, Tingting
> > Sent: Friday, August 7, 2020 10:42 AM
> > To: Bird, Tim <Tim.Bird@sony.com>
> > Cc: fuego@lists.linuxfoundation.org
> > Subject: Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support
> >
> > Tim
> >
> > Thanks for your suggestion.
> > Using a board-specific LTP skiplist is a good idear.
> >
> > Thanks,
> >   -- QiuTingting
> >
> > -----Original Message-----
> > From: Bird, Tim [mailto:Tim.Bird@sony.com]
> > Sent: Friday, August 7, 2020 7:28 AM
> > To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
> > Cc: fuego@lists.linuxfoundation.org
> > Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support
> >
> >
> >
> > > -----Original Message-----
> > > From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> > >
> > > -----Original Message-----
> > > From: Bird, Tim [mailto:Tim.Bird@sony.com]
> > > Sent: Thursday, July 9, 2020 6:20 AM
> > > To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>;
> > > fuego@lists.linuxfoundation.org
> > > Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not
> > > support
> > >
> > > > -----Original Message-----
> > > > From:  qiutt
> > > >
> > > > for aarch64(m3ulcb) skip cases in net.tcp_cmds
> > > >
> > > > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > > > ---
> > > >  tests/Functional.LTP/fuego_test.sh | 4 ++++
> > > >  1 file changed, 4 insertions(+)
> > > >
> > > > diff --git a/tests/Functional.LTP/fuego_test.sh
> > > > b/tests/Functional.LTP/fuego_test.sh
> > > > index ab4165d..2254474 100755
> > > > --- a/tests/Functional.LTP/fuego_test.sh
> > > > +++ b/tests/Functional.LTP/fuego_test.sh
> > > > @@ -352,6 +352,10 @@ function test_pre_check {
> > > >      if [ "$ARCHITECTURE" != "i386" ]; then
> > > >          skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
> > > >      fi
> > > > +
> > > > +    if [ "$ARCHITECTURE" == "aarch64" ]; then
> > > > +        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
> > > > +    fi
> > >
> > > This isn't right.  It may be true that netstat -s is not supported on m3ulcb, but it is supported just fine on other aarch64 platforms.
> > >
> > > What is the issue with supporting 'netstat -s' on m3ulcb?
> > > ->
> > > The patch was made for m3ulcb, and m3ulcb used the AGL system.
> > > The "netstat" command used was provided by "busybox"(*1).
> > > The error of "netstat -s" refer to (*2).
> > > Check the code(aarch64-agl-linux/busybox/1.29.3-r0/busybox-1.29.3/networking/netstat.c), the contents refer to (*3).
> > > According to the code, you can know that the netstat provided by
> > > busybox only supports the parameter "-laentuwxrWp", and does not support the parameter "-s".
> > >
> > > (*1)m3ulcb:~# which netstat
> > > /bin/netstat
> > > m3ulcb:~# ls -al /bin/netstat
> > > lrwxrwxrwx 1 root root 19 Feb 25  2020 /bin/netstat ->
> > > /bin/busybox.nosuid
> > >
> > > (*2)m3ulcb:~# netstat -s
> > > netstat: invalid option -- 's'
> > > BusyBox v1.29.3 (2020-01-16 16:59:41 UTC) multi-call binary.
> > > Usage: netstat [-ral] [-tuwx] [-en]
> > >
> > > (*3)
> > > //usage:#define netstat_trivial_usage
> > > //usage:       "[-"IF_ROUTE("r")"al] [-tuwx] [-en"IF_FEATURE_NETSTAT_WIDE("W")IF_FEATURE_NETSTAT_PRG("p")"]"
> > > //usage:#define netstat_full_usage "\n\n"
> > > //usage:       "Display networking information\n"
> > > //usage:        IF_ROUTE(
> > > //usage:     "\n        -r      Routing table"
> > > //usage:        )
> > > //usage:     "\n        -a      All sockets"
> > > //usage:     "\n        -l      Listening sockets"
> > > //usage:     "\n                Else: connected sockets"
> > > //usage:     "\n        -t      TCP sockets"
> > > //usage:     "\n        -u      UDP sockets"
> > > //usage:     "\n        -w      Raw sockets"
> > > //usage:     "\n        -x      Unix sockets"
> > > //usage:     "\n                Else: all socket types"
> > > //usage:     "\n        -e      Other/more information"
> > > //usage:     "\n        -n      Don't resolve names"
> > > //usage:        IF_FEATURE_NETSTAT_WIDE(
> > > //usage:     "\n        -W      Wide display"
> > > //usage:        )
> > > //usage:        IF_FEATURE_NETSTAT_PRG(
> > > //usage:     "\n        -p      Show PID/program name for sockets"
> > > //usage:        )
> > >
> > > #define NETSTAT_OPTS "laentuwx" \
> > >         IF_ROUTE(               "r") \
> > >         IF_FEATURE_NETSTAT_WIDE("W") \
> > >         IF_FEATURE_NETSTAT_PRG( "p")
> > > ...
> > >
> >
> > I see.  I was able to reproduce a problem running the netstat01 sub-test of 'net.tcp_cmds'
> > test in LTP, on a machine that only had busybox netstat.
> >
> > This should not be dependent on the machine architecture, which is unrelated, but rather on whether the machine supports 'netstat -s'.
> > But that's actually one of the functions of the netstat01 test in LTP (to test 'netstat -s' support).  In this case, the test is reporting
> > correctly that the m3ulcb board doesn't support that option.  So the test is correct.
> > Other Fuego users of the test may wish to see that problem reported.  It is not appropriate to mask out this correct bug report by
> > adding to a skiplist that applies to everyone.
> >
> > I'm not sure exactly what you are trying to accomplish (maybe hiding this bug so that it doesn't show up for this board, because your
> > organization has decided to ignore it?).  However, if you want to skip the test, then I would recommend adding this to a board-specific
> > LTP skiplist.
> >
> > In your m3ulcb.board file, you should add:
> > FUNCTIONAL_LTP_BOARD_SKIPLIST="netstat01"
> >
> > Let me know if this does not solve your problem.
> >  -- Tim
> >
> >
> >
> >
> >
> > _______________________________________________
> > Fuego mailing list
> > Fuego@lists.linuxfoundation.org
> > https://lists.linuxfoundation.org/mailman/listinfo/fuego

Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support

[Qiu, Tingting]

Hi, Daniel Tim

Sorry for the late reply because of the vacation.

I also think the latest LTP is better.
But, I'm afraid there is no time to test the latest LTP recently.

Regards,
 --Qiu Tingting

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Friday, October 2, 2020 6:35 AM
To: daniel.sangorrin@toshiba.co.jp; Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
Cc: fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' not support



> -----Original Message-----
> From: daniel.sangorrin@toshiba.co.jp > Hi Tim, Qiu
> 
> I am guessing this work is being done for the old 2018 LTP tarball. 
> However, I think that LTP has changed quite a few things since then (I 
> need to catchup on that). I think it is a good time to update the test to use the latest LTP and the new features that were added, what do you think?

I'm OK with this.  We could do this:

diff --git a/tests/Functional.LTP/spec.json b/tests/Functional.LTP/spec.json index 7c409ff..388097c 100644
--- a/tests/Functional.LTP/spec.json
+++ b/tests/Functional.LTP/spec.json
@@ -75,6 +75,12 @@
             "tests": "cve",
             "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
             "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
+        },
+        "latest": {
+            "gitrepo": "https://github.com/linux-test-project/ltp.git",
+            "tests": "syscalls",
+            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
+            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": 
+ "skiplist.txt"}
         }
-    }
+   }
 }

But that would only do one test from the latest (and it wouldn't be tied to a release of LTP).

Having been lurking on the LTP list for some time now, I have a couple of opinions:
 1. LTP top of tree moves and has bugs during development, so I would definitely want to
   specify a specific release commit, or use a snapshot release tarball
    - my preference is a snapshot tarball.  In general, I like avoiding network dependencies on
       external projects at test runtime, if possible.
 2. LTP has gotten a lot better over time, from what I can see.  I was initially hesitant to move forward
    to a new version, but I think that hesitation was misfounded.  I think Fuego is missing out on
    test improvements by not moving forward as LTP is improved.  It will cause churn in the testcase
    list produced between different versions of LTP, but I think that's a lesser problem than I expected.
    No one appears to be comparing LTP results to each other anyway.

So - sounds good to me!

Note that LTP just had a release this week, and I put a few new patches into Fuego related to that release.  Now might be a good time to add a more current test option for LTP, and make it default.

> > -----Original Message-----
> > From: Fuego <fuego-bounces@lists.linuxfoundation.org> On Behalf Of 
> > Qiu, Tingting
> > Sent: Friday, August 7, 2020 10:42 AM
> > To: Bird, Tim <Tim.Bird@sony.com>
> > Cc: fuego@lists.linuxfoundation.org
> > Subject: Re: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' 
> > not support
> >
> > Tim
> >
> > Thanks for your suggestion.
> > Using a board-specific LTP skiplist is a good idear.
> >
> > Thanks,
> >   -- QiuTingting
> >
> > -----Original Message-----
> > From: Bird, Tim [mailto:Tim.Bird@sony.com]
> > Sent: Friday, August 7, 2020 7:28 AM
> > To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>
> > Cc: fuego@lists.linuxfoundation.org
> > Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' 
> > not support
> >
> >
> >
> > > -----Original Message-----
> > > From: Qiu, Tingting <qiutt@cn.fujitsu.com>
> > >
> > > -----Original Message-----
> > > From: Bird, Tim [mailto:Tim.Bird@sony.com]
> > > Sent: Thursday, July 9, 2020 6:20 AM
> > > To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; 
> > > fuego@lists.linuxfoundation.org
> > > Subject: RE: [Fuego] [PATCH 07/12] skip cases because 'netstat -s' 
> > > not support
> > >
> > > > -----Original Message-----
> > > > From:  qiutt
> > > >
> > > > for aarch64(m3ulcb) skip cases in net.tcp_cmds
> > > >
> > > > Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> > > > ---
> > > >  tests/Functional.LTP/fuego_test.sh | 4 ++++
> > > >  1 file changed, 4 insertions(+)
> > > >
> > > > diff --git a/tests/Functional.LTP/fuego_test.sh
> > > > b/tests/Functional.LTP/fuego_test.sh
> > > > index ab4165d..2254474 100755
> > > > --- a/tests/Functional.LTP/fuego_test.sh
> > > > +++ b/tests/Functional.LTP/fuego_test.sh
> > > > @@ -352,6 +352,10 @@ function test_pre_check {
> > > >      if [ "$ARCHITECTURE" != "i386" ]; then
> > > >          skip_tests "modify_ldt01 modify_ldt02 modify_ldt03" # Only work on i386
> > > >      fi
> > > > +
> > > > +    if [ "$ARCHITECTURE" == "aarch64" ]; then
> > > > +        skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
> > > > +    fi
> > >
> > > This isn't right.  It may be true that netstat -s is not supported on m3ulcb, but it is supported just fine on other aarch64 platforms.
> > >
> > > What is the issue with supporting 'netstat -s' on m3ulcb?
> > > ->
> > > The patch was made for m3ulcb, and m3ulcb used the AGL system.
> > > The "netstat" command used was provided by "busybox"(*1).
> > > The error of "netstat -s" refer to (*2).
> > > Check the code(aarch64-agl-linux/busybox/1.29.3-r0/busybox-1.29.3/networking/netstat.c), the contents refer to (*3).
> > > According to the code, you can know that the netstat provided by 
> > > busybox only supports the parameter "-laentuwxrWp", and does not support the parameter "-s".
> > >
> > > (*1)m3ulcb:~# which netstat
> > > /bin/netstat
> > > m3ulcb:~# ls -al /bin/netstat
> > > lrwxrwxrwx 1 root root 19 Feb 25  2020 /bin/netstat -> 
> > > /bin/busybox.nosuid
> > >
> > > (*2)m3ulcb:~# netstat -s
> > > netstat: invalid option -- 's'
> > > BusyBox v1.29.3 (2020-01-16 16:59:41 UTC) multi-call binary.
> > > Usage: netstat [-ral] [-tuwx] [-en]
> > >
> > > (*3)
> > > //usage:#define netstat_trivial_usage
> > > //usage:       "[-"IF_ROUTE("r")"al] [-tuwx] [-en"IF_FEATURE_NETSTAT_WIDE("W")IF_FEATURE_NETSTAT_PRG("p")"]"
> > > //usage:#define netstat_full_usage "\n\n"
> > > //usage:       "Display networking information\n"
> > > //usage:        IF_ROUTE(
> > > //usage:     "\n        -r      Routing table"
> > > //usage:        )
> > > //usage:     "\n        -a      All sockets"
> > > //usage:     "\n        -l      Listening sockets"
> > > //usage:     "\n                Else: connected sockets"
> > > //usage:     "\n        -t      TCP sockets"
> > > //usage:     "\n        -u      UDP sockets"
> > > //usage:     "\n        -w      Raw sockets"
> > > //usage:     "\n        -x      Unix sockets"
> > > //usage:     "\n                Else: all socket types"
> > > //usage:     "\n        -e      Other/more information"
> > > //usage:     "\n        -n      Don't resolve names"
> > > //usage:        IF_FEATURE_NETSTAT_WIDE(
> > > //usage:     "\n        -W      Wide display"
> > > //usage:        )
> > > //usage:        IF_FEATURE_NETSTAT_PRG(
> > > //usage:     "\n        -p      Show PID/program name for sockets"
> > > //usage:        )
> > >
> > > #define NETSTAT_OPTS "laentuwx" \
> > >         IF_ROUTE(               "r") \
> > >         IF_FEATURE_NETSTAT_WIDE("W") \
> > >         IF_FEATURE_NETSTAT_PRG( "p") ...
> > >
> >
> > I see.  I was able to reproduce a problem running the netstat01 sub-test of 'net.tcp_cmds'
> > test in LTP, on a machine that only had busybox netstat.
> >
> > This should not be dependent on the machine architecture, which is unrelated, but rather on whether the machine supports 'netstat -s'.
> > But that's actually one of the functions of the netstat01 test in 
> > LTP (to test 'netstat -s' support).  In this case, the test is reporting correctly that the m3ulcb board doesn't support that option.  So the test is correct.
> > Other Fuego users of the test may wish to see that problem reported.  
> > It is not appropriate to mask out this correct bug report by adding to a skiplist that applies to everyone.
> >
> > I'm not sure exactly what you are trying to accomplish (maybe hiding 
> > this bug so that it doesn't show up for this board, because your 
> > organization has decided to ignore it?).  However, if you want to skip the test, then I would recommend adding this to a board-specific LTP skiplist.
> >
> > In your m3ulcb.board file, you should add:
> > FUNCTIONAL_LTP_BOARD_SKIPLIST="netstat01"
> >
> > Let me know if this does not solve your problem.
> >  -- Tim
> >
> >
> >
> >
> >
> > _______________________________________________
> > Fuego mailing list
> > Fuego@lists.linuxfoundation.org
> > https://lists.linuxfoundation.org/mailman/listinfo/fuego

[Fuego] [PATCH 08/12] skip cases because MULTIPORT and LIMIT not set

[qiutt]

for m3ulcb, skip cases in net.tcp_cmds
and MULTIPORT and LIMIT not set basis:
 CONFIG_NETFILTER_XT_MATCH_MULTIPORT is not set
 CONFIG_NETFILTER_XT_MATCH_LIMIT is not set

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 2254474..9963678 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -355,6 +355,7 @@ function test_pre_check {
 
     if [ "$ARCHITECTURE" == "aarch64" ]; then
         skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
+        skip_tests "iptables" # for net.tcp_cmds  CONFIG_NETFILTER_XT_MATCH_MULTIPORT is not set, CONFIG_NETFILTER_XT_MATCH_LIMIT is not set
     fi
 }
 
-- 
2.11.0

[Fuego] [PATCH 09/12] skip cases because label rule does not support

[qiutt]

for m3ulcb, skip cases in smack

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 9963678..2ac7caf 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -356,6 +356,7 @@ function test_pre_check {
     if [ "$ARCHITECTURE" == "aarch64" ]; then
         skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
         skip_tests "iptables" # for net.tcp_cmds  CONFIG_NETFILTER_XT_MATCH_MULTIPORT is not set, CONFIG_NETFILTER_XT_MATCH_LIMIT is not set
+        skip_tests "smack_file_access" # for smack, label rule does not support
     fi
 }
 
-- 
2.11.0

Re: [Fuego] [PATCH 09/12] skip cases because label rule does not support

[Bird, Tim]

> -----Original Message-----
> From: qiutt
> 
> for m3ulcb, skip cases in smack
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
> index 9963678..2ac7caf 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -356,6 +356,7 @@ function test_pre_check {
>      if [ "$ARCHITECTURE" == "aarch64" ]; then
>          skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
>          skip_tests "iptables" # for net.tcp_cmds  CONFIG_NETFILTER_XT_MATCH_MULTIPORT is not set,
> CONFIG_NETFILTER_XT_MATCH_LIMIT is not set
> +        skip_tests "smack_file_access" # for smack, label rule does not support

You can't use ARCHITECTURE == aarch64 as a proxy for testing for specific
features of m3ulcb (particularly smack - which is completely independent
of architecture and board).

>      fi
>  }
> 
> --
> 2.11.0
Not accepted.  Find another way to avoid running this.  Maybe based on a
kernel config variable?
 -- Tim

Re: [Fuego] [PATCH 09/12] skip cases because label rule does not support

[Qiu, Tingting]

[-- Attachment #1: Type: text/plain, Size: 1937 bytes --]

Hi,Tim

For patch08, use CONFIG configuration item as skip check condition.
See patch: 0008-skip-cases-because-MULTIPORT-and-LIMIT-not-set.patch

For patch09, "smack_file_access" failed because of the label rule setting problem in the system environment of the board, which has nothing to do with the case and architecture. 
The original skip condition is not suitable. 
Do you have any better suggestions to skip this case?

Thanks,
 -- QiuTingting

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Thursday, July 9, 2020 6:23 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 09/12] skip cases because label rule does not support



> -----Original Message-----
> From: qiutt
> 
> for m3ulcb, skip cases in smack
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/fuego_test.sh | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/tests/Functional.LTP/fuego_test.sh 
> b/tests/Functional.LTP/fuego_test.sh
> index 9963678..2ac7caf 100755
> --- a/tests/Functional.LTP/fuego_test.sh
> +++ b/tests/Functional.LTP/fuego_test.sh
> @@ -356,6 +356,7 @@ function test_pre_check {
>      if [ "$ARCHITECTURE" == "aarch64" ]; then
>          skip_tests "netstat" # for net.tcp_cmds, 'netstat -s' not support
>          skip_tests "iptables" # for net.tcp_cmds  
> CONFIG_NETFILTER_XT_MATCH_MULTIPORT is not set, 
> CONFIG_NETFILTER_XT_MATCH_LIMIT is not set
> +        skip_tests "smack_file_access" # for smack, label rule does 
> + not support

You can't use ARCHITECTURE == aarch64 as a proxy for testing for specific features of m3ulcb (particularly smack - which is completely independent of architecture and board).

>      fi
>  }
> 
> --
> 2.11.0
Not accepted.  Find another way to avoid running this.  Maybe based on a kernel config variable?
 -- Tim






[-- Attachment #2: 0008-skip-cases-because-MULTIPORT-and-LIMIT-not-set.patch --]
[-- Type: application/octet-stream, Size: 1274 bytes --]

From 6eae7eeaccd7c3287f4e70fc96a259f5a8d16c3d Mon Sep 17 00:00:00 2001
From: qiutt <qiutt@cn.fujitsu.com>
Date: Thu, 30 Jul 2020 14:17:39 +0800
Subject: [PATCH 2/2] skip cases because MULTIPORT and LIMIT not set

for m3ulcb, skip cases in net.tcp_cmds
if the following configs not set:
 CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
 CONFIG_NETFILTER_XT_MATCH_LIMIT=y

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 88dea7e..d5cbeaa 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -265,6 +265,7 @@ function test_pre_check {
     skip_if_kconfig_differs "CONFIG_CHECKPOINT_RESTORE=y" "kcmp01 kcmp02 kcmp03"
     skip_if_kconfig_differs "CONFIG_SWAP=y" "swapoff01 swapoff02 swapon01 swapon02 swapon03"
     skip_if_kconfig_differs "CONFIG_SGETMASK_SYSCALL=y" "ssetmask01"
+    skip_if_kconfig_differs "CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y CONFIG_NETFILTER_XT_MATCH_LIMIT=y" "iptables" # for net.tcp_cmds
 
     echo "Tests skipped depending on the config skiplist file"
     skip_tests_with_config_skiplist "CONFIG_XFRM_USER=y" "CONFIG_XFRM_USER_skiplist.txt"
-- 
2.11.0

[Fuego] [PATCH 10/12] add smackfsdir for smack cases

[qiutt]

for aarch64(m3ulcb), add "smackfsdir=/sys/fs/smackfs" for cases in smack

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh                     | 10 ++++++++++
 .../Functional.LTP/ltp_network_smack_set_ambient.patch | 10 ++++++++++
 tests/Functional.LTP/ltp_network_smack_set_cipso.patch | 10 ++++++++++
 .../Functional.LTP/ltp_network_smack_set_direct.patch  | 10 ++++++++++
 tests/Functional.LTP/ltp_network_smack_set_doi.patch   | 10 ++++++++++
 tests/Functional.LTP/ltp_network_smack_set_load.patch  | 10 ++++++++++
 .../ltp_network_smack_set_netlabel.patch               | 10 ++++++++++
 .../Functional.LTP/ltp_network_smack_set_onlycap.patch | 18 ++++++++++++++++++
 8 files changed, 88 insertions(+)
 create mode 100644 tests/Functional.LTP/ltp_network_smack_set_ambient.patch
 create mode 100644 tests/Functional.LTP/ltp_network_smack_set_cipso.patch
 create mode 100644 tests/Functional.LTP/ltp_network_smack_set_direct.patch
 create mode 100644 tests/Functional.LTP/ltp_network_smack_set_doi.patch
 create mode 100644 tests/Functional.LTP/ltp_network_smack_set_load.patch
 create mode 100644 tests/Functional.LTP/ltp_network_smack_set_netlabel.patch
 create mode 100644 tests/Functional.LTP/ltp_network_smack_set_onlycap.patch

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 2ac7caf..b097d22 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -364,6 +364,16 @@ function test_build {
     if [[ "$FUNCTIONAL_LTP_PHASES" == *build* ]] ; then
         echo "Building LTP"
         # Build the LTP tests
+            if [ "$ARCHITECTURE" == "aarch64" ]; then
+                patch -p0 < ${TEST_HOME}/ltp_network_smack_set_ambient.patch
+                patch -p0 < ${TEST_HOME}/ltp_network_smack_set_cipso.patch
+                patch -p0 < ${TEST_HOME}/ltp_network_smack_set_direct.patch
+                patch -p0 < ${TEST_HOME}/ltp_network_smack_set_doi.patch
+                patch -p0 < ${TEST_HOME}/ltp_network_smack_set_load.patch
+                patch -p0 < ${TEST_HOME}/ltp_network_smack_set_netlabel.patch
+                patch -p0 < ${TEST_HOME}/ltp_network_smack_set_onlycap.patch
+            fi
+
         make autotools
         ./configure CC="${CC}" AR="${AR}" RANLIB="${RANLIB}" LDFLAGS="$LDFLAGS" SYSROOT="${SDKROOT}" \
             --with-open-posix-testsuite --with-realtime-testsuite --without-perl --without-python --target=$PREFIX --host=$PREFIX \
diff --git a/tests/Functional.LTP/ltp_network_smack_set_ambient.patch b/tests/Functional.LTP/ltp_network_smack_set_ambient.patch
new file mode 100644
index 0000000..a0295a9
--- /dev/null
+++ b/tests/Functional.LTP/ltp_network_smack_set_ambient.patch
@@ -0,0 +1,10 @@
+--- testcases/kernel/security/smack/smack_set_ambient.sh	2020-06-24 13:56:22.109262738 +0800
++++ testcases/kernel/security/smack/smack_set_ambient.sh	2020-06-24 13:58:29.560200819 +0800
+@@ -12,6 +12,7 @@
+ 
+ export TCID=smack_set_ambient
+ export TST_TOTAL=1
++export smackfsdir=/sys/fs/smackfs
+ 
+ . test.sh
+ 
diff --git a/tests/Functional.LTP/ltp_network_smack_set_cipso.patch b/tests/Functional.LTP/ltp_network_smack_set_cipso.patch
new file mode 100644
index 0000000..f1d1e7e
--- /dev/null
+++ b/tests/Functional.LTP/ltp_network_smack_set_cipso.patch
@@ -0,0 +1,10 @@
+--- testcases/kernel/security/smack/smack_set_cipso.sh-bak	2020-06-24 13:56:22.069261816 +0800
++++ testcases/kernel/security/smack/smack_set_cipso.sh	2020-06-24 14:02:10.717299076 +0800
+@@ -16,6 +16,7 @@
+ 
+ export TCID=smack_set_cipso
+ export TST_TOTAL=1
++export smackfsdir=/sys/fs/smackfs
+ 
+ . test.sh
+ 
diff --git a/tests/Functional.LTP/ltp_network_smack_set_direct.patch b/tests/Functional.LTP/ltp_network_smack_set_direct.patch
new file mode 100644
index 0000000..0533e7d
--- /dev/null
+++ b/tests/Functional.LTP/ltp_network_smack_set_direct.patch
@@ -0,0 +1,10 @@
+--- testcases/kernel/security/smack/smack_set_direct.sh-bak	2020-06-24 13:56:22.069261816 +0800
++++ testcases/kernel/security/smack/smack_set_direct.sh	2020-06-24 14:01:42.064638556 +0800
+@@ -12,6 +12,7 @@
+ 
+ export TCID=smack_set_direct
+ export TST_TOTAL=1
++export smackfsdir=/sys/fs/smackfs
+ 
+ . test.sh
+ 
diff --git a/tests/Functional.LTP/ltp_network_smack_set_doi.patch b/tests/Functional.LTP/ltp_network_smack_set_doi.patch
new file mode 100644
index 0000000..e9f77ac
--- /dev/null
+++ b/tests/Functional.LTP/ltp_network_smack_set_doi.patch
@@ -0,0 +1,10 @@
+--- testcases/kernel/security/smack/smack_set_doi.sh-bak	2020-06-24 13:56:22.069261816 +0800
++++ testcases/kernel/security/smack/smack_set_doi.sh	2020-06-24 14:01:25.848264726 +0800
+@@ -12,6 +12,7 @@
+ 
+ export TCID=smack_set_doi
+ export TST_TOTAL=1
++export smackfsdir=/sys/fs/smackfs
+ 
+ . test.sh
+ 
diff --git a/tests/Functional.LTP/ltp_network_smack_set_load.patch b/tests/Functional.LTP/ltp_network_smack_set_load.patch
new file mode 100644
index 0000000..9b34af6
--- /dev/null
+++ b/tests/Functional.LTP/ltp_network_smack_set_load.patch
@@ -0,0 +1,10 @@
+--- testcases/kernel/security/smack/smack_set_load.sh-bak	2020-06-24 13:56:22.069261816 +0800
++++ testcases/kernel/security/smack/smack_set_load.sh	        2020-06-24 14:02:55.374328539 +0800
+@@ -16,6 +16,7 @@
+ 
+ export TCID=smack_set_load
+ export TST_TOTAL=1
++export smackfsdir=/sys/fs/smackfs
+ 
+ . test.sh
+ 
diff --git a/tests/Functional.LTP/ltp_network_smack_set_netlabel.patch b/tests/Functional.LTP/ltp_network_smack_set_netlabel.patch
new file mode 100644
index 0000000..4f485d2
--- /dev/null
+++ b/tests/Functional.LTP/ltp_network_smack_set_netlabel.patch
@@ -0,0 +1,10 @@
+--- testcases/kernel/security/smack/smack_set_netlabel.sh-bak	2020-06-24 13:56:22.069261816 +0800
++++ testcases/kernel/security/smack/smack_set_netlabel.sh	2020-06-24 14:02:25.393637404 +0800
+@@ -12,6 +12,7 @@
+ 
+ export TCID=smack_set_netlabel
+ export TST_TOTAL=1
++export smackfsdir=/sys/fs/smackfs
+ 
+ . test.sh
+ 
diff --git a/tests/Functional.LTP/ltp_network_smack_set_onlycap.patch b/tests/Functional.LTP/ltp_network_smack_set_onlycap.patch
new file mode 100644
index 0000000..150eb6b
--- /dev/null
+++ b/tests/Functional.LTP/ltp_network_smack_set_onlycap.patch
@@ -0,0 +1,18 @@
+--- testcases/kernel/security/smack/smack_set_onlycap.sh-bak	2020-06-24 13:56:22.069261816 +0800
++++ testcases/kernel/security/smack/smack_set_onlycap.sh	2020-06-24 14:44:42.816131711 +0800
+@@ -12,6 +12,7 @@
+ 
+ export TCID=smack_set_onlycap
+ export TST_TOTAL=1
++export smackfsdir=/sys/fs/smackfs
+ 
+ . test.sh
+ 
+@@ -23,6 +24,7 @@
+ echo "$my_label" 2>/dev/null > "$smackfsdir/onlycap"
+ 
+ label=$(cat "$smackfsdir/onlycap" 2>/dev/null)
++label=`echo $label | sed -e 's/$[ \t]*//g'`
+ if [ "$label" != "$my_label" ]; then
+ 	tst_brkm TFAIL "The smack label reported for \"$smackfsdir/onlycap\" "
+ 		       "is \"$label\", not the expected \"$my_label\"."
-- 
2.11.0

[Fuego] [PATCH 11/12] skip cases when LHOST_IFACES not defined

[qiutt]

for m3ulcb, skip in6_02 in net.ipv6lib, code modification refer to
latest ltp
 https://github.com/linux-test-project/ltp/commit/95c40a1cdfb35b4de69761599db37a14ea91af1d

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh    |  1 +
 tests/Functional.LTP/ltp-in6_02.patch | 51 +++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 tests/Functional.LTP/ltp-in6_02.patch

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index b097d22..25af0f5 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -372,6 +372,7 @@ function test_build {
                 patch -p0 < ${TEST_HOME}/ltp_network_smack_set_load.patch
                 patch -p0 < ${TEST_HOME}/ltp_network_smack_set_netlabel.patch
                 patch -p0 < ${TEST_HOME}/ltp_network_smack_set_onlycap.patch
+                patch -p0 < ${TEST_HOME}/ltp-in6_02.patch
             fi
 
         make autotools
diff --git a/tests/Functional.LTP/ltp-in6_02.patch b/tests/Functional.LTP/ltp-in6_02.patch
new file mode 100644
index 0000000..c9311a6
--- /dev/null
+++ b/tests/Functional.LTP/ltp-in6_02.patch
@@ -0,0 +1,51 @@
+--- testcases/network/lib6/in6_02.c-bak	2020-04-26 16:04:59.105983746 +0800
++++ testcases/network/lib6/in6_02.c	2020-06-28 15:58:09.700717063 +0800
+@@ -34,7 +34,7 @@
+ 	int nonzero;
+ } n2i[] = {
+ 	{ "lo", 1 },
+-	{ "eth0", 1 },
++	{ NULL, 1 },
+ 	{ "hoser75", 0 },
+ 	{ "6", 0 },
+ };
+@@ -80,6 +80,11 @@
+ 	char ifname[IF_NAMESIZE], *pifn;
+ 
+ 	for (i = 0; i < N2I_COUNT; ++i) {
++		if (n2i[i].name == NULL) {
++			tst_resm(TCONF, "LHOST_IFACES not defined or invalid, skip testing it");
++			return;
++		}
++
+ 		TEST(if_nametoindex(n2i[i].name));
+ 		if (!TEST_RETURN != !n2i[i].nonzero) {
+ 			tst_resm(TFAIL, "if_nametoindex(\"%s\") %ld "
+@@ -256,22 +261,18 @@
+ {
+ 	TEST_PAUSE;
+ 
+-	tst_resm(TINFO, "get interface name from LHOST_IFACES var");
+-
+ 	char *ifnames = getenv("LHOST_IFACES");
+ 
+-	if (!ifnames) {
+-		tst_resm(TWARN, "LHOST_IFACES not defined, default to eth0");
++	if (!ifnames)
+ 		return;
+-	}
+ 
+ 	static char name[256];
++        int ret;
+ 
+-	sscanf(ifnames, "%255s", name);
+-
+-	if (!strcmp(name, n2i[1].name))
++	ret = sscanf(ifnames, "%255s", name);
++	if (ret == -1)
+ 		return;
+ 
+-	tst_resm(TINFO, "change default 'eth0' name to '%s'", name);
++	tst_resm(TINFO, "get interface name from LHOST_IFACES: '%s'", name);
+ 	n2i[1].name = name;
+ }
-- 
2.11.0

[Fuego] [PATCH 12/12] add hostname and ip to /etc/hosts

[qiutt]

for m3ulcb, getaddrinfo_01 in net.ipv6lib, when the hostname and ip are
not in
/etc/hosts, add them as following
 127.0.0.1   m3ulcb
 ::1         m3ulcb

Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
---
 tests/Functional.LTP/fuego_test.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/Functional.LTP/fuego_test.sh b/tests/Functional.LTP/fuego_test.sh
index 25af0f5..c5dce55 100755
--- a/tests/Functional.LTP/fuego_test.sh
+++ b/tests/Functional.LTP/fuego_test.sh
@@ -474,6 +474,11 @@ function test_deploy {
 function test_run {
     if [[ "$FUNCTIONAL_LTP_PHASES" == *run* ]] ; then
         echo "Running LTP"
+        #for net.ipv6_lib(getaddrinfo_01), when the hostname and ip are not in etc/hosts, add them(e.g. 127.0.0.1   m3ulcb)
+        if [ $FUNCTIONAL_LTP_BOARD_NAME ]; then
+            cmd "if ! grep -q '127.0.0.1.*$FUNCTIONAL_LTP_BOARD_NAME' /etc/hosts; then sed -i -r '\$a\127.0.0.1        $FUNCTIONAL_LTP_BOARD_NAME' /etc/hosts;fi; \
+                 if ! grep -q '::1.*$FUNCTIONAL_LTP_BOARD_NAME' /etc/hosts; then sed -i -r '\$a\::1       $FUNCTIONAL_LTP_BOARD_NAME' /etc/hosts; fi;"
+        fi
 
         # we patch them here because of the different phases
         cmd "sed -i 's/^fork13 fork13 -i 1000000/fork13 fork13 -i 100000/' $LTP_DESTDIR/runtest/syscalls"
-- 
2.11.0

Re: [Fuego] [PATCH 01/12] add network testcase to spec

[Bird, Tim]

> -----Original Message-----
> From: Fuego <fuego-bounces@lists.linuxfoundation.org> On Behalf Of qiutt
> 
> include
>  net.features net.ipv6 net.ipv6_lib net.multicast
>  net.nfs net.rpc net.rpc_tests net.sctp
>  net_stress.appl net_stress.broken_ip net_stress.interface
>  net_stress.ipsec_dccp net_stress.ipsec_icmp net_stress.ipsec_sctp
>  net_stress.ipsec_tcp net_stress.ipsec_udp net_stress.multicast
>  net_stress.route net.tcp_cmds net.tirpc_tests network_commands smack
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/spec.json | 111 +++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 111 insertions(+)
> 
> diff --git a/tests/Functional.LTP/spec.json b/tests/Functional.LTP/spec.json
> index d23edc1..8da77b0 100644
> --- a/tests/Functional.LTP/spec.json
> +++ b/tests/Functional.LTP/spec.json
> @@ -73,6 +73,117 @@
>              "tests": "cve",
>              "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
>              "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.ipv6": {
> +            "tests": "net.ipv6",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.rpc_tests": {
> +            "tests": "net.rpc_tests",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.features": {
> +            "tests": "net.features",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.ipv6_lib": {
> +            "tests": "net.ipv6_lib",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "BOARD_NAME": "m3ulcb"
> +        },
> +        "net.multicast": {
> +            "tests": "net.multicast",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.nfs": {
> +            "tests": "net.nfs",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.rpc": {
> +            "tests": "net.rpc",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.sctp": {
> +            "tests": "net.sctp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.appl": {
> +            "tests": "net_stress.appl",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.broken_ip": {
> +            "tests": "net_stress.broken_ip",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.interface": {
> +            "tests": "net_stress.interface",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_dccp": {
> +            "tests": "net_stress.ipsec_dccp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_icmp": {
> +            "tests": "net_stress.ipsec_icmp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_sctp": {
> +            "tests": "net_stress.ipsec_sctp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_tcp": {
> +            "tests": "net_stress.ipsec_tcp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_udp": {
> +            "tests": "net_stress.ipsec_udp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.multicast": {
> +            "tests": "net_stress.multicast",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.route": {
> +            "tests": "net_stress.route",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },

Is there a reason all these individual stress tests are broken out into their
own specs?

> +        "net.tcp_cmds": {
> +            "tests": "net.tcp_cmds ",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.tirpc_tests": {
> +            "tests": "net.tirpc_tests",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "network_commands": {
> +            "tests": "network_commands",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "smack": {
> +            "tests": "smack",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
>          }
>      }
>  }
> --
> 2.11.0

This seems like a lot of new specs.  Do you actually run the networking
tests separately, or are they usually run together?

Thanks,
 -- Tim

Re: [Fuego] [PATCH 01/12] add network testcase to spec

[Qiu, Tingting]

Sorry for reply late.

> Is there a reason all these individual stress tests are broken out into their own specs?
> This seems like a lot of new specs.  Do you actually run the networking tests separately, or are they usually run together?
In LTP's test suites, all these individual stress tests are broken out into their own suites.
All the suites can be tested in one spec, but you must modify the spec file when you want to test some of them.
When one spec has one suite, you can choose jobs for testing some of them.

Thanks,
 -- QiuTingting

-----Original Message-----
From: Bird, Tim [mailto:Tim.Bird@sony.com] 
Sent: Thursday, July 9, 2020 5:01 AM
To: Qiu, Tingting/仇 婷婷 <qiutt@cn.fujitsu.com>; fuego@lists.linuxfoundation.org
Subject: RE: [Fuego] [PATCH 01/12] add network testcase to spec



> -----Original Message-----
> From: Fuego <fuego-bounces@lists.linuxfoundation.org> On Behalf Of 
> qiutt
> 
> include
>  net.features net.ipv6 net.ipv6_lib net.multicast  net.nfs net.rpc 
> net.rpc_tests net.sctp  net_stress.appl net_stress.broken_ip 
> net_stress.interface  net_stress.ipsec_dccp net_stress.ipsec_icmp 
> net_stress.ipsec_sctp  net_stress.ipsec_tcp net_stress.ipsec_udp 
> net_stress.multicast  net_stress.route net.tcp_cmds net.tirpc_tests 
> network_commands smack
> 
> Signed-off-by: qiutt <qiutt@cn.fujitsu.com>
> ---
>  tests/Functional.LTP/spec.json | 111 
> +++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 111 insertions(+)
> 
> diff --git a/tests/Functional.LTP/spec.json 
> b/tests/Functional.LTP/spec.json index d23edc1..8da77b0 100644
> --- a/tests/Functional.LTP/spec.json
> +++ b/tests/Functional.LTP/spec.json
> @@ -73,6 +73,117 @@
>              "tests": "cve",
>              "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
>              "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": 
> "skiplist.txt"}
> +        },
> +        "net.ipv6": {
> +            "tests": "net.ipv6",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.rpc_tests": {
> +            "tests": "net.rpc_tests",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.features": {
> +            "tests": "net.features",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.ipv6_lib": {
> +            "tests": "net.ipv6_lib",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "BOARD_NAME": "m3ulcb"
> +        },
> +        "net.multicast": {
> +            "tests": "net.multicast",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.nfs": {
> +            "tests": "net.nfs",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.rpc": {
> +            "tests": "net.rpc",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.sctp": {
> +            "tests": "net.sctp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.appl": {
> +            "tests": "net_stress.appl",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.broken_ip": {
> +            "tests": "net_stress.broken_ip",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.interface": {
> +            "tests": "net_stress.interface",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_dccp": {
> +            "tests": "net_stress.ipsec_dccp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_icmp": {
> +            "tests": "net_stress.ipsec_icmp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_sctp": {
> +            "tests": "net_stress.ipsec_sctp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_tcp": {
> +            "tests": "net_stress.ipsec_tcp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.ipsec_udp": {
> +            "tests": "net_stress.ipsec_udp",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.multicast": {
> +            "tests": "net_stress.multicast",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net_stress.route": {
> +            "tests": "net_stress.route",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },

Is there a reason all these individual stress tests are broken out into their own specs?

> +        "net.tcp_cmds": {
> +            "tests": "net.tcp_cmds ",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "net.tirpc_tests": {
> +            "tests": "net.tirpc_tests",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "network_commands": {
> +            "tests": "network_commands",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"}
> +        },
> +        "smack": {
> +            "tests": "smack",
> +            "extra_success_links": {"xlsx": "results.xlsx", "skiplist": "skiplist.txt"},
> +            "extra_fail_links": {"xlsx": "results.xlsx", "skiplist": 
> + "skiplist.txt"}
>          }
>      }
>  }
> --
> 2.11.0

This seems like a lot of new specs.  Do you actually run the networking tests separately, or are they usually run together?

Thanks,
 -- Tim