Re: [PATCH mptcp-next 3/9] selftests: mptcp: join: option to execute specific tests

[Paolo Abeni <pabeni@redhat.com>]

On Wed, 2022-02-09 at 16:36 -0800, Mat Martineau wrote:
> On Wed, 9 Feb 2022, Matthieu Baerts wrote:
> 
> > Often, it is needed to run one specific test.
> > 
> > There are options to run subgroups of tests but when only one fails, no
> > need to run all the subgroup. So far, the solution was to edit the
> > script to comment the tests that are not needed but that's not ideal.
> > 
> > Now, it is possible to run one specific test by giving the ID of the
> > tests that are going to be validated, e.g.
> > 
> >  ./mptcp_join.sh 36 37
> > 
> > This is cleaner and saves time.
> > 
> > Technically, the reset* functions now return 0 if the test can be
> > executed. This naturally creates sections per test in the code which is
> > also helpful to understand what a test is exactly doing.
> > 
> 
> My sense is that this is primarily for development purposes since the test 
> numbers may shift as tests are added or modified. Is it worth the churn 
> when we have the subgroup functionality and could comment out chunks of 
> the test script that are not useful for running specific tests in a loop?
> 
> I'm not set against this change, but hoping to discuss it some more to 
> both understand the benefits (I assume for running CI in a loop?) and to 
> see if anyone has bright ideas for avoiding the whitespace churn.
> 
> ("Include a bash fork in our selftest code that implements 'goto'" seems 
> like overkill :) )

Perhapas something alike the following ?!? Not a complete solution:
- completely not tested
- we will first need to add some wrapper for iptables or even beffer
for ip netns exec, to be more generic
- possibly more 'skip_test && return' needed, but not many, I guess.

I hope should be enough demonstrate the idea...
---
diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh
b/tools/testing/selftests/net/mptcp/mptcp_join.sh
index 29623d7ad415..4199d7404f10 100755
--- a/tools/testing/selftests/net/mptcp/mptcp_join.sh
+++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh
@@ -113,6 +113,7 @@ reset()
 
 reset_with_cookies()
 {
+	skip_test && return
 	reset
 
 	for netns in "$ns1" "$ns2";do
@@ -125,6 +126,7 @@ reset_with_add_addr_timeout()
 	local ip="${1:-4}"
 	local tables
 
+	skip_test && return
 	tables="iptables"
 	if [ $ip -eq 6 ]; then
 		tables="ip6tables"
@@ -145,6 +147,7 @@ reset_with_checksum()
 	local ns1_enable=$1
 	local ns2_enable=$2
 
+	skip_test && return
 	reset
 
 	ip netns exec $ns1 sysctl -q
net.mptcp.checksum_enabled=$ns1_enable
@@ -156,6 +159,7 @@ reset_with_allow_join_id0()
 	local ns1_enable=$1
 	local ns2_enable=$2
 
+	skip_test && return
 	reset
 
 	ip netns exec $ns1 sysctl -q
net.mptcp.allow_join_initial_addr_port=$ns1_enable
@@ -295,6 +299,7 @@ pm_nl_set_limits()
 	local addrs=$2
 	local subflows=$3
 
+	skip_test && return
 	if [ $ip_mptcp -eq 1 ]; then
 		ip -n $ns mptcp limits set add_addr_accepted $addrs
subflows $subflows
 	else
@@ -312,6 +317,7 @@ pm_nl_add_endpoint()
 	local id
 	local nr=2
 
+	skip_test && return
 	for p in $@
 	do
 		if [ $p = "flags" ]; then
@@ -689,6 +695,7 @@ run_tests()
 	lret=0
 	oldin=""
 
+	skip_test && return
 	# create the input file for the failure test when
 	# the first failure test run
 	if [ "$test_linkfail" -ne 0 -a -z "$cinfail" ]; then
@@ -732,6 +739,7 @@ chk_csum_nr()
 	local count
 	local dump_stats
 
+	skip_test && return
 	if [ ! -z "$msg" ]; then
 		printf "%02u" "$TEST_COUNT"
 	else
@@ -767,6 +775,7 @@ chk_fail_nr()
 	local count
 	local dump_stats
 
+	skip_test && return
 	printf "%-39s %s" " " "ftx"
 	count=`ip netns exec $ns1 nstat -as | grep MPTcpExtMPFailTx |
awk '{print $2}'`
 	[ -z "$count" ] && count=0
@@ -801,6 +810,7 @@ chk_join_nr()
 	local count
 	local dump_stats
 
+	skip_test && return
 	printf "%02u %-36s %s" "$TEST_COUNT" "$msg" "syn"
 	count=`ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinSynRx
| awk '{print $2}'`
 	[ -z "$count" ] && count=0
@@ -855,6 +865,7 @@ chk_stale_nr()
 	local stale_nr
 	local recover_nr
 
+	skip_test && return
 	printf "%-39s %-18s" " " "stale"
 	stale_nr=`ip netns exec $ns nstat -as | grep
MPTcpExtSubflowStale | awk '{print $2}'`
 	[ -z "$stale_nr" ] && stale_nr=0
@@ -893,6 +904,7 @@ chk_add_nr()
 	local count
 	local dump_stats
 
+	skip_test && return
 	printf "%-39s %s" " " "add"
 	count=`ip netns exec $ns2 nstat -as | grep MPTcpExtAddAddr |
awk '{print $2}'`
 	[ -z "$count" ] && count=0
@@ -1008,6 +1020,7 @@ chk_rm_nr()
 	local addr_ns
 	local subflow_ns
 
+	skip_test && return
 	if [ -z $invert ]; then
 		addr_ns=$ns1
 		subflow_ns=$ns2
@@ -1048,6 +1061,7 @@ chk_prio_nr()
 	local count
 	local dump_stats
 
+	skip_test && return
 	printf "%-39s %s" " " "ptx"
 	count=`ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioTx |
awk '{print $2}'`
 	[ -z "$count" ] && count=0
@@ -1084,6 +1098,7 @@ chk_link_usage()
 	local tx_rate=$((tx_link * 100 / $tx_total))
 	local tolerance=5
 
+	skip_test && return
 	printf "%-39s %-18s" " " "link usage"
 	if [ $tx_rate -lt $((expected_rate - $tolerance)) -o \
 	     $tx_rate -gt $((expected_rate + $tolerance)) ]; then