[MPTCP][PATCH v6 mptcp-next 0/5] MP_FAIL support

[MPTCP][PATCH v6 mptcp-next 0/5] MP_FAIL support

[Geliang Tang]

From: Geliang Tang <geliangtang@xiaomi.com>

v6: only rebased patch 1
 - move the struct member 'fail_seq' behind 'ext_copy'.
 - define OPTION_MPTCP_FAIL to BIT(12), BIT(11) is used by DSS
 - move the MP_FAIL writing code at the beginning of mptcp_write_options,
   and add the 'unlikely' tag.
 - tag: export/20210728T080904

v5:
 - patch 1, change "ret = true;" to "return true;"
 - patch 3, in the single-subflow case, send MP_FAIL and receive the
   echo, then temporarily handled by reset.

v4:
 - just deal with the multiple subflows case, put the single subflow
   case into the new 'infinite mapping' part.

v3:
 - respond with MP_FAIL
 - add single subflow check
 - add infinite mapping sending and receiving
 - export/20210626T054902

v2:
 - MP_FAIL logic:
   * Peer B send a DSS to peer A, and the data has been modify by the
  middleboxes, then peer A detects the bad checksum.
   * In the multiple subflows case, peer A sends MP_FAIL+RST back to peer B,
  and peer A discards the data following the bad data sequence number. Peer
  B receives this MP_FAIL+RST, and close this subflow.
   * In the single subflow case, using the simple implementation, peer A
  sends MP_FAIL back to peer B, and peer A fallback to a regular TCP. Peer
  B receives this MP_FAIL, and fallback to a regular TCP.

Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/52

Geliang Tang (5):
  mptcp: MP_FAIL suboption sending
  mptcp: MP_FAIL suboption receiving
  mptcp: send out MP_FAIL when data checksum fails
  mptcp: add the mibs for MP_FAIL
  selftests: mptcp: add MP_FAIL mibs check

 include/net/mptcp.h                           |  5 +-
 net/mptcp/mib.c                               |  2 +
 net/mptcp/mib.h                               |  2 +
 net/mptcp/options.c                           | 78 ++++++++++++++++++-
 net/mptcp/pm.c                                | 20 +++++
 net/mptcp/protocol.h                          | 20 +++++
 net/mptcp/subflow.c                           | 18 +++++
 .../testing/selftests/net/mptcp/mptcp_join.sh | 38 +++++++++
 8 files changed, 178 insertions(+), 5 deletions(-)

-- 
2.31.1

[MPTCP][PATCH v6 mptcp-next 1/5] mptcp: MP_FAIL suboption sending

[Geliang Tang]

From: Geliang Tang <geliangtang@xiaomi.com>

This patch added the MP_FAIL suboption sending support.

Add a new flag named send_mp_fail in struct mptcp_subflow_context. If
this flag is set, send out MP_FAIL suboption.

Add a new member fail_seq in struct mptcp_out_options to save the data
sequence number to put into the MP_FAIL suboption.

An MP_FAIL option could be included in a RST or on the subflow-level
ACK.

Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
---
 include/net/mptcp.h  |  5 +++-
 net/mptcp/options.c  | 61 +++++++++++++++++++++++++++++++++++++++++---
 net/mptcp/protocol.h |  3 +++
 3 files changed, 64 insertions(+), 5 deletions(-)

diff --git a/include/net/mptcp.h b/include/net/mptcp.h
index 3236010afa29..6026bbefbffd 100644
--- a/include/net/mptcp.h
+++ b/include/net/mptcp.h
@@ -74,7 +74,10 @@ struct mptcp_out_options {
 			struct mptcp_addr_info addr;
 			u64 ahmac;
 		};
-		struct mptcp_ext ext_copy;
+		struct {
+			struct mptcp_ext ext_copy;
+			u64 fail_seq;
+		};
 		struct {
 			u32 nonce;
 			u32 token;
diff --git a/net/mptcp/options.c b/net/mptcp/options.c
index 753d6ac43bff..2b15063c8009 100644
--- a/net/mptcp/options.c
+++ b/net/mptcp/options.c
@@ -763,7 +763,7 @@ static bool mptcp_established_options_mp_prio(struct sock *sk,
 	return true;
 }
 
-static noinline void mptcp_established_options_rst(struct sock *sk, struct sk_buff *skb,
+static noinline bool mptcp_established_options_rst(struct sock *sk, struct sk_buff *skb,
 						   unsigned int *size,
 						   unsigned int remaining,
 						   struct mptcp_out_options *opts)
@@ -771,12 +771,36 @@ static noinline void mptcp_established_options_rst(struct sock *sk, struct sk_bu
 	const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 
 	if (remaining < TCPOLEN_MPTCP_RST)
-		return;
+		return false;
 
 	*size = TCPOLEN_MPTCP_RST;
 	opts->suboptions |= OPTION_MPTCP_RST;
 	opts->reset_transient = subflow->reset_transient;
 	opts->reset_reason = subflow->reset_reason;
+
+	return true;
+}
+
+static bool mptcp_established_options_mp_fail(struct sock *sk,
+					      unsigned int *size,
+					      unsigned int remaining,
+					      struct mptcp_out_options *opts)
+{
+	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
+
+	if (!subflow->send_mp_fail)
+		return false;
+
+	if (remaining < TCPOLEN_MPTCP_FAIL)
+		return false;
+
+	*size = TCPOLEN_MPTCP_FAIL;
+	opts->suboptions |= OPTION_MPTCP_FAIL;
+	opts->fail_seq = subflow->map_seq;
+
+	pr_debug("MP_FAIL fail_seq=%llu", opts->fail_seq);
+
+	return true;
 }
 
 bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
@@ -795,15 +819,30 @@ bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
 		return false;
 
 	if (unlikely(skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST)) {
-		mptcp_established_options_rst(sk, skb, size, remaining, opts);
+		if (mptcp_established_options_mp_fail(sk, &opt_size, remaining, opts)) {
+			*size += opt_size;
+			remaining -= opt_size;
+		}
+		if (mptcp_established_options_rst(sk, skb, &opt_size, remaining, opts)) {
+			*size += opt_size;
+			remaining -= opt_size;
+		}
 		return true;
 	}
 
 	snd_data_fin = mptcp_data_fin_enabled(msk);
 	if (mptcp_established_options_mp(sk, skb, snd_data_fin, &opt_size, remaining, opts))
 		ret = true;
-	else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, remaining, opts))
+	else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, remaining, opts)) {
 		ret = true;
+		if (opts->ext_copy.use_ack) {
+			if (mptcp_established_options_mp_fail(sk, &opt_size, remaining, opts)) {
+				*size += opt_size;
+				remaining -= opt_size;
+				return true;
+			}
+		}
+	}
 
 	/* we reserved enough space for the above options, and exceeding the
 	 * TCP option space would be fatal
@@ -1210,6 +1249,20 @@ static u16 mptcp_make_csum(const struct mptcp_ext *mpext)
 void mptcp_write_options(__be32 *ptr, const struct tcp_sock *tp,
 			 struct mptcp_out_options *opts)
 {
+	if (unlikely(OPTION_MPTCP_FAIL & opts->suboptions)) {
+		const struct sock *ssk = (const struct sock *)tp;
+		struct mptcp_subflow_context *subflow;
+
+		subflow = mptcp_subflow_ctx(ssk);
+		subflow->send_mp_fail = 0;
+
+		*ptr++ = mptcp_option(MPTCPOPT_MP_FAIL,
+				      TCPOLEN_MPTCP_FAIL,
+				      0, 0);
+		put_unaligned_be64(opts->fail_seq, ptr);
+		ptr += 2;
+	}
+
 	/* RST is mutually exclusive with everything else */
 	if (unlikely(OPTION_MPTCP_RST & opts->suboptions)) {
 		*ptr++ = mptcp_option(MPTCPOPT_RST,
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index e8a36ff52af6..b389fec18c89 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -27,6 +27,7 @@
 #define OPTION_MPTCP_PRIO	BIT(9)
 #define OPTION_MPTCP_RST	BIT(10)
 #define OPTION_MPTCP_DSS	BIT(11)
+#define OPTION_MPTCP_FAIL	BIT(12)
 
 /* MPTCP option subtypes */
 #define MPTCPOPT_MP_CAPABLE	0
@@ -68,6 +69,7 @@
 #define TCPOLEN_MPTCP_PRIO_ALIGN	4
 #define TCPOLEN_MPTCP_FASTCLOSE		12
 #define TCPOLEN_MPTCP_RST		4
+#define TCPOLEN_MPTCP_FAIL		12
 
 #define TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM	(TCPOLEN_MPTCP_DSS_CHECKSUM + TCPOLEN_MPTCP_MPC_ACK_DATA)
 
@@ -429,6 +431,7 @@ struct mptcp_subflow_context {
 		mpc_map : 1,
 		backup : 1,
 		send_mp_prio : 1,
+		send_mp_fail : 1,
 		rx_eof : 1,
 		can_ack : 1,        /* only after processing the remote a key */
 		disposable : 1,	    /* ctx can be free at ulp release time */
-- 
2.31.1

[MPTCP][PATCH v6 mptcp-next 2/5] mptcp: MP_FAIL suboption receiving

[Geliang Tang]

From: Geliang Tang <geliangtang@xiaomi.com>

This patch added handling for receiving MP_FAIL suboption.

Add a new members mp_fail and fail_seq in struct mptcp_options_received.
When MP_FAIL suboption is received, set mp_fail to 1 and save the sequence
number to fail_seq.

Then invoke mptcp_pm_mp_fail_received to deal with the MP_FAIL suboption.

Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
---
 net/mptcp/options.c  | 16 ++++++++++++++++
 net/mptcp/pm.c       |  5 +++++
 net/mptcp/protocol.h |  3 +++
 3 files changed, 24 insertions(+)

diff --git a/net/mptcp/options.c b/net/mptcp/options.c
index 2b15063c8009..cd9ec4acf127 100644
--- a/net/mptcp/options.c
+++ b/net/mptcp/options.c
@@ -336,6 +336,16 @@ static void mptcp_parse_option(const struct sk_buff *skb,
 		mp_opt->reset_reason = *ptr;
 		break;
 
+	case MPTCPOPT_MP_FAIL:
+		if (opsize != TCPOLEN_MPTCP_FAIL)
+			break;
+
+		ptr += 2;
+		mp_opt->mp_fail = 1;
+		mp_opt->fail_seq = get_unaligned_be64(ptr);
+		pr_debug("MP_FAIL: data_seq=%llu", mp_opt->fail_seq);
+		break;
+
 	default:
 		break;
 	}
@@ -364,6 +374,7 @@ void mptcp_get_options(const struct sock *sk,
 	mp_opt->reset = 0;
 	mp_opt->csum_reqd = READ_ONCE(msk->csum_enabled);
 	mp_opt->deny_join_id0 = 0;
+	mp_opt->mp_fail = 0;
 
 	length = (th->doff * 4) - sizeof(struct tcphdr);
 	ptr = (const unsigned char *)(th + 1);
@@ -1147,6 +1158,11 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
 		mp_opt.mp_prio = 0;
 	}
 
+	if (mp_opt.mp_fail) {
+		mptcp_pm_mp_fail_received(sk, mp_opt.fail_seq);
+		mp_opt.mp_fail = 0;
+	}
+
 	if (mp_opt.reset) {
 		subflow->reset_seen = 1;
 		subflow->reset_reason = mp_opt.reset_reason;
diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index da0c4c925350..6ab386ff3294 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -249,6 +249,11 @@ void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup)
 	mptcp_event(MPTCP_EVENT_SUB_PRIORITY, mptcp_sk(subflow->conn), sk, GFP_ATOMIC);
 }
 
+void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
+{
+	pr_debug("fail_seq=%llu", fail_seq);
+}
+
 /* path manager helpers */
 
 bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, struct sk_buff *skb,
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index b389fec18c89..09d0e9406ea9 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -140,6 +140,7 @@ struct mptcp_options_received {
 		add_addr : 1,
 		rm_addr : 1,
 		mp_prio : 1,
+		mp_fail : 1,
 		echo : 1,
 		csum_reqd : 1,
 		backup : 1,
@@ -161,6 +162,7 @@ struct mptcp_options_received {
 	u64	ahmac;
 	u8	reset_reason:4;
 	u8	reset_transient:1;
+	u64	fail_seq;
 };
 
 static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field)
@@ -727,6 +729,7 @@ void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup);
 int mptcp_pm_nl_mp_prio_send_ack(struct mptcp_sock *msk,
 				 struct mptcp_addr_info *addr,
 				 u8 bkup);
+void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq);
 void mptcp_pm_free_anno_list(struct mptcp_sock *msk);
 bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk);
 struct mptcp_pm_add_entry *
-- 
2.31.1

[MPTCP][PATCH v6 mptcp-next 3/5] mptcp: send out MP_FAIL when data checksum fails

[Geliang Tang]

From: Geliang Tang <geliangtang@xiaomi.com>

When a bad checksum is detected, set the send_mp_fail flag to send out
the MP_FAIL option.

Add a new function mptcp_has_another_subflow() to check whether there's
only a single subflow.

When multiple subflows are in use, close the affected subflow with a RST
that includes an MP_FAIL option and discard the data with the bad
checksum.

Set the sk_state of the subsocket to TCP_CLOSE, then the flag
MPTCP_WORK_CLOSE_SUBFLOW will be set in subflow_sched_work_if_closed,
and the subflow will be closed.

When a single subfow is in use, send back an MP_FAIL option on the
subflow-level ACK. And the receiver of this MP_FAIL respond with an
MP_FAIL in the reverse direction.

Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
---
 net/mptcp/pm.c       | 14 ++++++++++++++
 net/mptcp/protocol.h | 14 ++++++++++++++
 net/mptcp/subflow.c  | 17 +++++++++++++++++
 3 files changed, 45 insertions(+)

diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index 6ab386ff3294..c2df5cc28ba1 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -251,7 +251,21 @@ void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup)
 
 void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
 {
+	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
+
 	pr_debug("fail_seq=%llu", fail_seq);
+
+	if (!mptcp_has_another_subflow(sk)) {
+		if (!subflow->mp_fail_expect_echo) {
+			subflow->send_mp_fail = 1;
+		} else {
+			subflow->mp_fail_expect_echo = 0;
+			/* TODO the single-subflow case is temporarily
+			 * handled by reset.
+			 */
+			mptcp_subflow_reset(sk);
+		}
+	}
 }
 
 /* path manager helpers */
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index 09d0e9406ea9..c46011318f65 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -434,6 +434,7 @@ struct mptcp_subflow_context {
 		backup : 1,
 		send_mp_prio : 1,
 		send_mp_fail : 1,
+		mp_fail_expect_echo : 1,
 		rx_eof : 1,
 		can_ack : 1,        /* only after processing the remote a key */
 		disposable : 1,	    /* ctx can be free at ulp release time */
@@ -615,6 +616,19 @@ static inline void mptcp_subflow_tcp_fallback(struct sock *sk,
 	inet_csk(sk)->icsk_af_ops = ctx->icsk_af_ops;
 }
 
+static inline bool mptcp_has_another_subflow(struct sock *ssk)
+{
+	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk), *tmp;
+	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
+
+	mptcp_for_each_subflow(msk, tmp) {
+		if (tmp != subflow)
+			return true;
+	}
+
+	return false;
+}
+
 void __init mptcp_proto_init(void);
 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
 int __init mptcp_proto_v6_init(void);
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index 1151926d335b..a69839520472 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -910,6 +910,7 @@ static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *
 	csum = csum_partial(&header, sizeof(header), subflow->map_data_csum);
 	if (unlikely(csum_fold(csum))) {
 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR);
+		subflow->send_mp_fail = 1;
 		return subflow->mp_join ? MAPPING_INVALID : MAPPING_DUMMY;
 	}
 
@@ -1157,6 +1158,22 @@ static bool subflow_check_data_avail(struct sock *ssk)
 
 fallback:
 	/* RFC 8684 section 3.7. */
+	if (subflow->send_mp_fail) {
+		if (mptcp_has_another_subflow(ssk)) {
+			ssk->sk_err = EBADMSG;
+			tcp_set_state(ssk, TCP_CLOSE);
+			subflow->reset_transient = 0;
+			subflow->reset_reason = MPTCP_RST_EMIDDLEBOX;
+			tcp_send_active_reset(ssk, GFP_ATOMIC);
+			while ((skb = skb_peek(&ssk->sk_receive_queue)))
+				sk_eat_skb(ssk, skb);
+		} else {
+			subflow->mp_fail_expect_echo = 1;
+		}
+		WRITE_ONCE(subflow->data_avail, 0);
+		return true;
+	}
+
 	if (subflow->mp_join || subflow->fully_established) {
 		/* fatal protocol error, close the socket.
 		 * subflow_error_report() will introduce the appropriate barriers
-- 
2.31.1

[MPTCP][PATCH v6 mptcp-next 4/5] mptcp: add the mibs for MP_FAIL

[Geliang Tang]

From: Geliang Tang <geliangtang@xiaomi.com>

This patch added the mibs for MP_FAIL: MPTCP_MIB_MPFAILTX and
MPTCP_MIB_MPFAILRX.

Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
---
 net/mptcp/mib.c     | 2 ++
 net/mptcp/mib.h     | 2 ++
 net/mptcp/options.c | 1 +
 net/mptcp/pm.c      | 1 +
 net/mptcp/subflow.c | 1 +
 5 files changed, 7 insertions(+)

diff --git a/net/mptcp/mib.c b/net/mptcp/mib.c
index 3a7c4e7b2d79..b21ff9be04c6 100644
--- a/net/mptcp/mib.c
+++ b/net/mptcp/mib.c
@@ -44,6 +44,8 @@ static const struct snmp_mib mptcp_snmp_list[] = {
 	SNMP_MIB_ITEM("RmSubflow", MPTCP_MIB_RMSUBFLOW),
 	SNMP_MIB_ITEM("MPPrioTx", MPTCP_MIB_MPPRIOTX),
 	SNMP_MIB_ITEM("MPPrioRx", MPTCP_MIB_MPPRIORX),
+	SNMP_MIB_ITEM("MPFailTx", MPTCP_MIB_MPFAILTX),
+	SNMP_MIB_ITEM("MPFailRx", MPTCP_MIB_MPFAILRX),
 	SNMP_MIB_ITEM("RcvPruned", MPTCP_MIB_RCVPRUNED),
 	SNMP_MIB_ITEM("SubflowStale", MPTCP_MIB_SUBFLOWSTALE),
 	SNMP_MIB_ITEM("SubflowRecover", MPTCP_MIB_SUBFLOWRECOVER),
diff --git a/net/mptcp/mib.h b/net/mptcp/mib.h
index 8ec16c991aac..ecd3d8b117e0 100644
--- a/net/mptcp/mib.h
+++ b/net/mptcp/mib.h
@@ -37,6 +37,8 @@ enum linux_mptcp_mib_field {
 	MPTCP_MIB_RMSUBFLOW,		/* Remove a subflow */
 	MPTCP_MIB_MPPRIOTX,		/* Transmit a MP_PRIO */
 	MPTCP_MIB_MPPRIORX,		/* Received a MP_PRIO */
+	MPTCP_MIB_MPFAILTX,		/* Transmit a MP_FAIL */
+	MPTCP_MIB_MPFAILRX,		/* Received a MP_FAIL */
 	MPTCP_MIB_RCVPRUNED,		/* Incoming packet dropped due to memory limit */
 	MPTCP_MIB_SUBFLOWSTALE,		/* Subflows entered 'stale' status */
 	MPTCP_MIB_SUBFLOWRECOVER,	/* Subflows returned to active status after being stale */
diff --git a/net/mptcp/options.c b/net/mptcp/options.c
index cd9ec4acf127..8b899c308b83 100644
--- a/net/mptcp/options.c
+++ b/net/mptcp/options.c
@@ -1160,6 +1160,7 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
 
 	if (mp_opt.mp_fail) {
 		mptcp_pm_mp_fail_received(sk, mp_opt.fail_seq);
+		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILRX);
 		mp_opt.mp_fail = 0;
 	}
 
diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index c2df5cc28ba1..43530d3a78e9 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -258,6 +258,7 @@ void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
 	if (!mptcp_has_another_subflow(sk)) {
 		if (!subflow->mp_fail_expect_echo) {
 			subflow->send_mp_fail = 1;
+			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILTX);
 		} else {
 			subflow->mp_fail_expect_echo = 0;
 			/* TODO the single-subflow case is temporarily
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index a69839520472..c25b1d961206 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -911,6 +911,7 @@ static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *
 	if (unlikely(csum_fold(csum))) {
 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR);
 		subflow->send_mp_fail = 1;
+		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPFAILTX);
 		return subflow->mp_join ? MAPPING_INVALID : MAPPING_DUMMY;
 	}
 
-- 
2.31.1

[MPTCP][PATCH v6 mptcp-next 5/5] selftests: mptcp: add MP_FAIL mibs check

[Geliang Tang]

From: Geliang Tang <geliangtang@xiaomi.com>

This patch added a function chk_fail_nr to check the mibs for MP_FAIL.

Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
---
 .../testing/selftests/net/mptcp/mptcp_join.sh | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh
index 937e861e9490..551fcce7c2f2 100755
--- a/tools/testing/selftests/net/mptcp/mptcp_join.sh
+++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh
@@ -566,6 +566,43 @@ chk_csum_nr()
 	fi
 }
 
+chk_fail_nr()
+{
+	local mp_fail_nr_tx=$1
+	local mp_fail_nr_rx=$2
+	local count
+	local dump_stats
+
+	printf "%-39s %s" " " "ftx"
+	count=`ip netns exec $ns1 nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}'`
+	[ -z "$count" ] && count=0
+	if [ "$count" != "$mp_fail_nr_tx" ]; then
+		echo "[fail] got $count MP_FAIL[s] TX expected $mp_fail_nr_tx"
+		ret=1
+		dump_stats=1
+	else
+		echo -n "[ ok ]"
+	fi
+
+	echo -n " - frx   "
+	count=`ip netns exec $ns2 nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}'`
+	[ -z "$count" ] && count=0
+	if [ "$count" != "$mp_fail_nr_rx" ]; then
+		echo "[fail] got $count MP_FAIL[s] RX expected $mp_fail_nr_rx"
+		ret=1
+		dump_stats=1
+	else
+		echo "[ ok ]"
+	fi
+
+	if [ "${dump_stats}" = 1 ]; then
+		echo Server ns stats
+		ip netns exec $ns1 nstat -as | grep MPTcp
+		echo Client ns stats
+		ip netns exec $ns2 nstat -as | grep MPTcp
+	fi
+}
+
 chk_join_nr()
 {
 	local msg="$1"
@@ -615,6 +652,7 @@ chk_join_nr()
 	fi
 	if [ $checksum -eq 1 ]; then
 		chk_csum_nr
+		chk_fail_nr 0 0
 	fi
 }
 
-- 
2.31.1

Re: [MPTCP][PATCH v6 mptcp-next 3/5] mptcp: send out MP_FAIL when data checksum fails

[Mat Martineau]

On Wed, 28 Jul 2021, Geliang Tang wrote:

> From: Geliang Tang <geliangtang@xiaomi.com>
>
> When a bad checksum is detected, set the send_mp_fail flag to send out
> the MP_FAIL option.
>
> Add a new function mptcp_has_another_subflow() to check whether there's
> only a single subflow.
>
> When multiple subflows are in use, close the affected subflow with a RST
> that includes an MP_FAIL option and discard the data with the bad
> checksum.
>

Thanks for the test code! I do see in wireshark that the multiple subflow 
case sends a TCP RST with both MP_FAIL and MP_TCPRST options when the 
checksum fails on one subflow.

> Set the sk_state of the subsocket to TCP_CLOSE, then the flag
> MPTCP_WORK_CLOSE_SUBFLOW will be set in subflow_sched_work_if_closed,
> and the subflow will be closed.
>
> When a single subfow is in use, send back an MP_FAIL option on the
> subflow-level ACK. And the receiver of this MP_FAIL respond with an
> MP_FAIL in the reverse direction.
>

The single subflow case has some unexpected behavior:

1. The checksum failure is detected, a packet is sent with MP_FAIL. 
However, the packet also has data payload and no DSS option.

2. The peer receives MP_FAIL, and echoes back. But it sends two TCP RST + 
MP_FAIL packets back-to-back.

I'll upload a pcap to 
https://github.com/multipath-tcp/mptcp_net-next/issues/52

I think the right temporary behavior (before implementing infinite 
mappings) for single subflow checksum failure is to do what the RFC says 
for non-contiguous data: "In the rare case that the data is not contiguous 
(which could happen when there is only one subflow but it is 
retransmitting data from a subflow that has recently been uncleanly 
closed), the receiver MUST close the subflow with a RST with MP_FAIL." So, 
in step 1 above the peer that detected the bad checksum would still send 
MP_FAIL but with the RST flag. And then the echo would not be needed 
because the path would already be disconnected by the RST.

What do you think?


Mat


> Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
> ---
> net/mptcp/pm.c       | 14 ++++++++++++++
> net/mptcp/protocol.h | 14 ++++++++++++++
> net/mptcp/subflow.c  | 17 +++++++++++++++++
> 3 files changed, 45 insertions(+)
>
> diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
> index 6ab386ff3294..c2df5cc28ba1 100644
> --- a/net/mptcp/pm.c
> +++ b/net/mptcp/pm.c
> @@ -251,7 +251,21 @@ void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup)
>
> void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
> {
> +	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
> +
> 	pr_debug("fail_seq=%llu", fail_seq);
> +
> +	if (!mptcp_has_another_subflow(sk)) {
> +		if (!subflow->mp_fail_expect_echo) {
> +			subflow->send_mp_fail = 1;
> +		} else {
> +			subflow->mp_fail_expect_echo = 0;
> +			/* TODO the single-subflow case is temporarily
> +			 * handled by reset.
> +			 */
> +			mptcp_subflow_reset(sk);
> +		}
> +	}
> }
>
> /* path manager helpers */
> diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
> index 09d0e9406ea9..c46011318f65 100644
> --- a/net/mptcp/protocol.h
> +++ b/net/mptcp/protocol.h
> @@ -434,6 +434,7 @@ struct mptcp_subflow_context {
> 		backup : 1,
> 		send_mp_prio : 1,
> 		send_mp_fail : 1,
> +		mp_fail_expect_echo : 1,
> 		rx_eof : 1,
> 		can_ack : 1,        /* only after processing the remote a key */
> 		disposable : 1,	    /* ctx can be free at ulp release time */
> @@ -615,6 +616,19 @@ static inline void mptcp_subflow_tcp_fallback(struct sock *sk,
> 	inet_csk(sk)->icsk_af_ops = ctx->icsk_af_ops;
> }
>
> +static inline bool mptcp_has_another_subflow(struct sock *ssk)
> +{
> +	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk), *tmp;
> +	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
> +
> +	mptcp_for_each_subflow(msk, tmp) {
> +		if (tmp != subflow)
> +			return true;
> +	}
> +
> +	return false;
> +}
> +
> void __init mptcp_proto_init(void);
> #if IS_ENABLED(CONFIG_MPTCP_IPV6)
> int __init mptcp_proto_v6_init(void);
> diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
> index 1151926d335b..a69839520472 100644
> --- a/net/mptcp/subflow.c
> +++ b/net/mptcp/subflow.c
> @@ -910,6 +910,7 @@ static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *
> 	csum = csum_partial(&header, sizeof(header), subflow->map_data_csum);
> 	if (unlikely(csum_fold(csum))) {
> 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR);
> +		subflow->send_mp_fail = 1;
> 		return subflow->mp_join ? MAPPING_INVALID : MAPPING_DUMMY;
> 	}
>
> @@ -1157,6 +1158,22 @@ static bool subflow_check_data_avail(struct sock *ssk)
>
> fallback:
> 	/* RFC 8684 section 3.7. */
> +	if (subflow->send_mp_fail) {
> +		if (mptcp_has_another_subflow(ssk)) {
> +			ssk->sk_err = EBADMSG;
> +			tcp_set_state(ssk, TCP_CLOSE);
> +			subflow->reset_transient = 0;
> +			subflow->reset_reason = MPTCP_RST_EMIDDLEBOX;
> +			tcp_send_active_reset(ssk, GFP_ATOMIC);
> +			while ((skb = skb_peek(&ssk->sk_receive_queue)))
> +				sk_eat_skb(ssk, skb);
> +		} else {
> +			subflow->mp_fail_expect_echo = 1;
> +		}
> +		WRITE_ONCE(subflow->data_avail, 0);
> +		return true;
> +	}
> +
> 	if (subflow->mp_join || subflow->fully_established) {
> 		/* fatal protocol error, close the socket.
> 		 * subflow_error_report() will introduce the appropriate barriers
> -- 
> 2.31.1
>
>
>

--
Mat Martineau
Intel

Re: [MPTCP][PATCH v6 mptcp-next 2/5] mptcp: MP_FAIL suboption receiving

[Paolo Abeni]

On Wed, 2021-07-28 at 17:35 +0800, Geliang Tang wrote:
> From: Geliang Tang <geliangtang@xiaomi.com>
> 
> This patch added handling for receiving MP_FAIL suboption.
> 
> Add a new members mp_fail and fail_seq in struct mptcp_options_received.
> When MP_FAIL suboption is received, set mp_fail to 1 and save the sequence
> number to fail_seq.
> 
> Then invoke mptcp_pm_mp_fail_received to deal with the MP_FAIL suboption.
> 
> Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
> ---
>  net/mptcp/options.c  | 16 ++++++++++++++++
>  net/mptcp/pm.c       |  5 +++++
>  net/mptcp/protocol.h |  3 +++
>  3 files changed, 24 insertions(+)
> 
> diff --git a/net/mptcp/options.c b/net/mptcp/options.c
> index 2b15063c8009..cd9ec4acf127 100644
> --- a/net/mptcp/options.c
> +++ b/net/mptcp/options.c
> @@ -336,6 +336,16 @@ static void mptcp_parse_option(const struct sk_buff *skb,
>  		mp_opt->reset_reason = *ptr;
>  		break;
>  
> +	case MPTCPOPT_MP_FAIL:
> +		if (opsize != TCPOLEN_MPTCP_FAIL)
> +			break;
> +
> +		ptr += 2;
> +		mp_opt->mp_fail = 1;
> +		mp_opt->fail_seq = get_unaligned_be64(ptr);
> +		pr_debug("MP_FAIL: data_seq=%llu", mp_opt->fail_seq);
> +		break;
> +
>  	default:
>  		break;
>  	}
> @@ -364,6 +374,7 @@ void mptcp_get_options(const struct sock *sk,
>  	mp_opt->reset = 0;
>  	mp_opt->csum_reqd = READ_ONCE(msk->csum_enabled);
>  	mp_opt->deny_join_id0 = 0;
> +	mp_opt->mp_fail = 0;
>  
>  	length = (th->doff * 4) - sizeof(struct tcphdr);
>  	ptr = (const unsigned char *)(th + 1);
> @@ -1147,6 +1158,11 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
>  		mp_opt.mp_prio = 0;
>  	}
>  
> +	if (mp_opt.mp_fail) {
> +		mptcp_pm_mp_fail_received(sk, mp_opt.fail_seq);
> +		mp_opt.mp_fail = 0;
> +	}
> +

Side note not specifically related to this patch: usually we get a
single MPTCP subopt per packet: a DSS. So we could optimize this code
path with something alike:

	if (unlikely(any subopt other than dss is present))
		// go checking all of them individually

To do the above we likely need to wrap all the 'mp_capable',
'fastclose', 'rm_addr' flags in a single bitmask. e.v. using a union.

/P

Re: [MPTCP][PATCH v6 mptcp-next 2/5] mptcp: MP_FAIL suboption receiving

[Geliang Tang]

Hi Paolo,

Paolo Abeni <pabeni@redhat.com> 于2021年7月28日周三 下午6:36写道：

>
> On Wed, 2021-07-28 at 17:35 +0800, Geliang Tang wrote:
> > From: Geliang Tang <geliangtang@xiaomi.com>
> >
> > This patch added handling for receiving MP_FAIL suboption.
> >
> > Add a new members mp_fail and fail_seq in struct mptcp_options_received.
> > When MP_FAIL suboption is received, set mp_fail to 1 and save the sequence
> > number to fail_seq.
> >
> > Then invoke mptcp_pm_mp_fail_received to deal with the MP_FAIL suboption.
> >
> > Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
> > ---
> >  net/mptcp/options.c  | 16 ++++++++++++++++
> >  net/mptcp/pm.c       |  5 +++++
> >  net/mptcp/protocol.h |  3 +++
> >  3 files changed, 24 insertions(+)
> >
> > diff --git a/net/mptcp/options.c b/net/mptcp/options.c
> > index 2b15063c8009..cd9ec4acf127 100644
> > --- a/net/mptcp/options.c
> > +++ b/net/mptcp/options.c
> > @@ -336,6 +336,16 @@ static void mptcp_parse_option(const struct sk_buff *skb,
> >               mp_opt->reset_reason = *ptr;
> >               break;
> >
> > +     case MPTCPOPT_MP_FAIL:
> > +             if (opsize != TCPOLEN_MPTCP_FAIL)
> > +                     break;
> > +
> > +             ptr += 2;
> > +             mp_opt->mp_fail = 1;
> > +             mp_opt->fail_seq = get_unaligned_be64(ptr);
> > +             pr_debug("MP_FAIL: data_seq=%llu", mp_opt->fail_seq);
> > +             break;
> > +
> >       default:
> >               break;
> >       }
> > @@ -364,6 +374,7 @@ void mptcp_get_options(const struct sock *sk,
> >       mp_opt->reset = 0;
> >       mp_opt->csum_reqd = READ_ONCE(msk->csum_enabled);
> >       mp_opt->deny_join_id0 = 0;
> > +     mp_opt->mp_fail = 0;
> >
> >       length = (th->doff * 4) - sizeof(struct tcphdr);
> >       ptr = (const unsigned char *)(th + 1);
> > @@ -1147,6 +1158,11 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
> >               mp_opt.mp_prio = 0;
> >       }
> >
> > +     if (mp_opt.mp_fail) {
> > +             mptcp_pm_mp_fail_received(sk, mp_opt.fail_seq);
> > +             mp_opt.mp_fail = 0;
> > +     }
> > +
>
> Side note not specifically related to this patch: usually we get a
> single MPTCP subopt per packet: a DSS. So we could optimize this code
> path with something alike:
>
>         if (unlikely(any subopt other than dss is present))
>                 // go checking all of them individually

How about simply doing it like this:

       if (unlikely(mp_opt.fastclose || mp_opt.add_addr || mp_opt.rm_addr ||
                    mp_opt.mp_prio || mp_opt.mp_fail || mp_opt.reset)) {
                  // go checking all of them individually
       }

I just sent out a patch for this named "mptcp: use unlikely for non-DSS
suboptions" to ML. Please review it.

Thanks,
-Geliang


>
> To do the above we likely need to wrap all the 'mp_capable',
> 'fastclose', 'rm_addr' flags in a single bitmask. e.v. using a union.
>
> /P
>

Re: [MPTCP][PATCH v6 mptcp-next 1/5] mptcp: MP_FAIL suboption sending

[Paolo Abeni]

Hello,

On Wed, 2021-07-28 at 17:35 +0800, Geliang Tang wrote:
> From: Geliang Tang <geliangtang@xiaomi.com>
> 
> This patch added the MP_FAIL suboption sending support.
> 
> Add a new flag named send_mp_fail in struct mptcp_subflow_context. If
> this flag is set, send out MP_FAIL suboption.
> 
> Add a new member fail_seq in struct mptcp_out_options to save the data
> sequence number to put into the MP_FAIL suboption.
> 
> An MP_FAIL option could be included in a RST or on the subflow-level
> ACK.
> 
> Signed-off-by: Geliang Tang <geliangtang@xiaomi.com>
> ---
>  include/net/mptcp.h  |  5 +++-
>  net/mptcp/options.c  | 61 +++++++++++++++++++++++++++++++++++++++++---
>  net/mptcp/protocol.h |  3 +++
>  3 files changed, 64 insertions(+), 5 deletions(-)
> 
> diff --git a/include/net/mptcp.h b/include/net/mptcp.h
> index 3236010afa29..6026bbefbffd 100644
> --- a/include/net/mptcp.h
> +++ b/include/net/mptcp.h
> @@ -74,7 +74,10 @@ struct mptcp_out_options {
>  			struct mptcp_addr_info addr;
>  			u64 ahmac;
>  		};
> -		struct mptcp_ext ext_copy;
> +		struct {
> +			struct mptcp_ext ext_copy;
> +			u64 fail_seq;
> +		};
>  		struct {
>  			u32 nonce;
>  			u32 token;
> diff --git a/net/mptcp/options.c b/net/mptcp/options.c
> index 753d6ac43bff..2b15063c8009 100644
> --- a/net/mptcp/options.c
> +++ b/net/mptcp/options.c
> @@ -763,7 +763,7 @@ static bool mptcp_established_options_mp_prio(struct sock *sk,
>  	return true;
>  }
>  
> -static noinline void mptcp_established_options_rst(struct sock *sk, struct sk_buff *skb,
> +static noinline bool mptcp_established_options_rst(struct sock *sk, struct sk_buff *skb,
>  						   unsigned int *size,
>  						   unsigned int remaining,
>  						   struct mptcp_out_options *opts)
> @@ -771,12 +771,36 @@ static noinline void mptcp_established_options_rst(struct sock *sk, struct sk_bu
>  	const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
>  
>  	if (remaining < TCPOLEN_MPTCP_RST)
> -		return;
> +		return false;
>  
>  	*size = TCPOLEN_MPTCP_RST;
>  	opts->suboptions |= OPTION_MPTCP_RST;
>  	opts->reset_transient = subflow->reset_transient;
>  	opts->reset_reason = subflow->reset_reason;
> +
> +	return true;
> +}
> +
> +static bool mptcp_established_options_mp_fail(struct sock *sk,
> +					      unsigned int *size,
> +					      unsigned int remaining,
> +					      struct mptcp_out_options *opts)
> +{
> +	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
> +
> +	if (!subflow->send_mp_fail)
> +		return false;
> +
> +	if (remaining < TCPOLEN_MPTCP_FAIL)
> +		return false;
> +
> +	*size = TCPOLEN_MPTCP_FAIL;
> +	opts->suboptions |= OPTION_MPTCP_FAIL;
> +	opts->fail_seq = subflow->map_seq;
> +
> +	pr_debug("MP_FAIL fail_seq=%llu", opts->fail_seq);
> +
> +	return true;
>  }
>  
>  bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
> @@ -795,15 +819,30 @@ bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
>  		return false;
>  
>  	if (unlikely(skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST)) {
> -		mptcp_established_options_rst(sk, skb, size, remaining, opts);
> +		if (mptcp_established_options_mp_fail(sk, &opt_size, remaining, opts)) {
> +			*size += opt_size;
> +			remaining -= opt_size;
> +		}
> +		if (mptcp_established_options_rst(sk, skb, &opt_size, remaining, opts)) {
> +			*size += opt_size;
> +			remaining -= opt_size;
> +		}
>  		return true;
>  	}
>  
>  	snd_data_fin = mptcp_data_fin_enabled(msk);
>  	if (mptcp_established_options_mp(sk, skb, snd_data_fin, &opt_size, remaining, opts))
>  		ret = true;
> -	else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, remaining, opts))
> +	else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, remaining, opts)) {
>  		ret = true;
> +		if (opts->ext_copy.use_ack) {

I *think* we could drop this check as the RFC says:

"""
   it will send back an MP_FAIL option on
   the subflow-level ACK,
"""

And to me subflow-level ACK really means TCP ack, that is any packet on
a TCP-established subflow will do.

Anyhow I think we can adjust the above with a squash-to patch after
this series is merged, as it already went through several iterations
and overall it LGTM.

While at that a few 'unlikely()' annotation will help ;)

/P

Re: [MPTCP][PATCH v6 mptcp-next 0/5] MP_FAIL support

[Paolo Abeni]

On Wed, 2021-07-28 at 17:35 +0800, Geliang Tang wrote:
> From: Geliang Tang <geliangtang@xiaomi.com>
> 
> v6: only rebased patch 1
>  - move the struct member 'fail_seq' behind 'ext_copy'.
>  - define OPTION_MPTCP_FAIL to BIT(12), BIT(11) is used by DSS
>  - move the MP_FAIL writing code at the beginning of mptcp_write_options,
>    and add the 'unlikely' tag.
>  - tag: export/20210728T080904
> 
> v5:
>  - patch 1, change "ret = true;" to "return true;"
>  - patch 3, in the single-subflow case, send MP_FAIL and receive the
>    echo, then temporarily handled by reset.
> 
> v4:
>  - just deal with the multiple subflows case, put the single subflow
>    case into the new 'infinite mapping' part.
> 
> v3:
>  - respond with MP_FAIL
>  - add single subflow check
>  - add infinite mapping sending and receiving
>  - export/20210626T054902
> 
> v2:
>  - MP_FAIL logic:
>    * Peer B send a DSS to peer A, and the data has been modify by the
>   middleboxes, then peer A detects the bad checksum.
>    * In the multiple subflows case, peer A sends MP_FAIL+RST back to peer B,
>   and peer A discards the data following the bad data sequence number. Peer
>   B receives this MP_FAIL+RST, and close this subflow.
>    * In the single subflow case, using the simple implementation, peer A
>   sends MP_FAIL back to peer B, and peer A fallback to a regular TCP. Peer
>   B receives this MP_FAIL, and fallback to a regular TCP.
> 
> Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/52
> 
> Geliang Tang (5):
>   mptcp: MP_FAIL suboption sending
>   mptcp: MP_FAIL suboption receiving
>   mptcp: send out MP_FAIL when data checksum fails
>   mptcp: add the mibs for MP_FAIL
>   selftests: mptcp: add MP_FAIL mibs check
> 
>  include/net/mptcp.h                           |  5 +-
>  net/mptcp/mib.c                               |  2 +
>  net/mptcp/mib.h                               |  2 +
>  net/mptcp/options.c                           | 78 ++++++++++++++++++-
>  net/mptcp/pm.c                                | 20 +++++
>  net/mptcp/protocol.h                          | 20 +++++
>  net/mptcp/subflow.c                           | 18 +++++
>  .../testing/selftests/net/mptcp/mptcp_join.sh | 38 +++++++++
>  8 files changed, 178 insertions(+), 5 deletions(-)

I had a couple of minor comments, but they could be addressed with
squash-to or even completely unrelated changes.

Overall LGTM, thanks!

/P