[PATCH v4 5/6] net/mlx5: support mark flow action

[Nelio Laranjeiro <nelio.laranjeiro@6wind.com>]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 78 ++++++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_prm.h  | 70 ++++++++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.c | 12 ++++++-
 drivers/net/mlx5/mlx5_rxtx.h |  3 +-
 4 files changed, 160 insertions(+), 3 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 1ec0ef5..01f7a77 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -50,6 +50,7 @@
 #include <rte_malloc.h>
 
 #include "mlx5.h"
+#include "mlx5_prm.h"
 
 static int
 mlx5_flow_create_eth(const struct rte_flow_item *item, void *data);
@@ -81,6 +82,7 @@ struct rte_flow {
 	struct ibv_exp_wq *wq; /**< Verbs work queue. */
 	struct ibv_cq *cq; /**< Verbs completion queue. */
 	struct rxq *rxq; /**< Pointer to the queue, NULL if drop queue. */
+	uint32_t mark:1; /**< Set if the flow is marked. */
 };
 
 /** Static initializer for items. */
@@ -119,6 +121,7 @@ struct mlx5_flow_items {
 static const enum rte_flow_action_type valid_actions[] = {
 	RTE_FLOW_ACTION_TYPE_DROP,
 	RTE_FLOW_ACTION_TYPE_QUEUE,
+	RTE_FLOW_ACTION_TYPE_MARK,
 	RTE_FLOW_ACTION_TYPE_END,
 };
 
@@ -246,7 +249,9 @@ struct mlx5_flow {
 struct mlx5_flow_action {
 	uint32_t queue:1; /**< Target is a receive queue. */
 	uint32_t drop:1; /**< Target is a drop queue. */
+	uint32_t mark:1; /**< Mark is present in the flow. */
 	uint32_t queue_id; /**< Identifier of the queue. */
+	uint32_t mark_id; /**< Mark identifier. */
 };
 
 /**
@@ -341,6 +346,7 @@ priv_flow_validate(struct priv *priv,
 	struct mlx5_flow_action action = {
 		.queue = 0,
 		.drop = 0,
+		.mark = 0,
 	};
 
 	(void)priv;
@@ -427,10 +433,26 @@ priv_flow_validate(struct priv *priv,
 			if (!queue || (queue->index > (priv->rxqs_n - 1)))
 				goto exit_action_not_supported;
 			action.queue = 1;
+		} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
+			const struct rte_flow_action_mark *mark =
+				(const struct rte_flow_action_mark *)
+				actions->conf;
+
+			if (mark && (mark->id >= MLX5_FLOW_MARK_MAX)) {
+				rte_flow_error_set(error, ENOTSUP,
+						   RTE_FLOW_ERROR_TYPE_ACTION,
+						   actions,
+						   "mark must be between 0"
+						   " and 16777199");
+				return -rte_errno;
+			}
+			action.mark = 1;
 		} else {
 			goto exit_action_not_supported;
 		}
 	}
+	if (action.mark && !flow->ibv_attr)
+		flow->offset += sizeof(struct ibv_exp_flow_spec_action_tag);
 	if (!action.queue && !action.drop) {
 		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 				   NULL, "no valid action");
@@ -745,6 +767,30 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item, void *data)
 }
 
 /**
+ * Convert mark/flag action to Verbs specification.
+ *
+ * @param flow
+ *   Pointer to MLX5 flow structure.
+ * @param mark_id
+ *   Mark identifier.
+ */
+static int
+mlx5_flow_create_flag_mark(struct mlx5_flow *flow, uint32_t mark_id)
+{
+	struct ibv_exp_flow_spec_action_tag *tag;
+	unsigned int size = sizeof(struct ibv_exp_flow_spec_action_tag);
+
+	tag = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
+	*tag = (struct ibv_exp_flow_spec_action_tag){
+		.type = IBV_EXP_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+		.tag_id = mlx5_flow_mark_set(mark_id),
+	};
+	++flow->ibv_attr->num_of_specs;
+	return 0;
+}
+
+/**
  * Complete flow rule creation.
  *
  * @param priv
@@ -800,8 +846,10 @@ priv_flow_create_action_queue(struct priv *priv,
 		rxq = container_of((*priv->rxqs)[action->queue_id],
 				   struct rxq_ctrl, rxq);
 		rte_flow->rxq = &rxq->rxq;
+		rxq->rxq.mark |= action->mark;
 		rte_flow->wq = rxq->wq;
 	}
+	rte_flow->mark = action->mark;
 	rte_flow->ibv_attr = ibv_attr;
 	rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
 		priv->ctx,
@@ -917,6 +965,8 @@ priv_flow_create(struct priv *priv,
 	action = (struct mlx5_flow_action){
 		.queue = 0,
 		.drop = 0,
+		.mark = 0,
+		.mark_id = MLX5_FLOW_MARK_DEFAULT,
 	};
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 		if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
@@ -928,6 +978,14 @@ priv_flow_create(struct priv *priv,
 				 actions->conf)->index;
 		} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 			action.drop = 1;
+		} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
+			const struct rte_flow_action_mark *mark =
+				(const struct rte_flow_action_mark *)
+				actions->conf;
+
+			if (mark)
+				action.mark_id = mark->id;
+			action.mark = 1;
 		} else {
 			rte_flow_error_set(error, ENOTSUP,
 					   RTE_FLOW_ERROR_TYPE_ACTION,
@@ -935,6 +993,10 @@ priv_flow_create(struct priv *priv,
 			goto exit;
 		}
 	}
+	if (action.mark) {
+		mlx5_flow_create_flag_mark(&flow, action.mark_id);
+		flow.offset += sizeof(struct ibv_exp_flow_spec_action_tag);
+	}
 	rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
 						 &action, error);
 	return rte_flow;
@@ -993,6 +1055,18 @@ priv_flow_destroy(struct priv *priv,
 		claim_zero(ibv_exp_destroy_wq(flow->wq));
 	if (!flow->rxq && flow->cq)
 		claim_zero(ibv_destroy_cq(flow->cq));
+	if (flow->mark) {
+		struct rte_flow *tmp;
+		uint32_t mark_n = 0;
+
+		for (tmp = LIST_FIRST(&priv->flows);
+		     tmp;
+		     tmp = LIST_NEXT(tmp, next)) {
+			if ((flow->rxq == tmp->rxq) && tmp->mark)
+				++mark_n;
+		}
+		flow->rxq->mark = !!mark_n;
+	}
 	rte_free(flow->ibv_attr);
 	DEBUG("Flow destroyed %p", (void *)flow);
 	rte_free(flow);
@@ -1072,6 +1146,8 @@ priv_flow_stop(struct priv *priv)
 	     flow = LIST_NEXT(flow, next)) {
 		claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
 		flow->ibv_flow = NULL;
+		if (flow->mark)
+			flow->rxq->mark = 0;
 		DEBUG("Flow %p removed", (void *)flow);
 	}
 }
@@ -1101,6 +1177,8 @@ priv_flow_start(struct priv *priv)
 			return rte_errno;
 		}
 		DEBUG("Flow %p applied", (void *)flow);
+		if (flow->rxq)
+			flow->rxq->mark |= flow->mark;
 	}
 	return 0;
 }
diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h
index 9cd9fdf..d9bb332 100644
--- a/drivers/net/mlx5/mlx5_prm.h
+++ b/drivers/net/mlx5/mlx5_prm.h
@@ -34,6 +34,8 @@
 #ifndef RTE_PMD_MLX5_PRM_H_
 #define RTE_PMD_MLX5_PRM_H_
 
+#include <assert.h>
+
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
 #ifdef PEDANTIC
@@ -106,6 +108,15 @@
 /* Outer UDP header and checksum OK. */
 #define MLX5_CQE_RX_OUTER_TCP_UDP_CSUM_OK (1u << 6)
 
+/* INVALID is used by packets matching no flow rules. */
+#define MLX5_FLOW_MARK_INVALID 0
+
+/* Maximum allowed value to mark a packet. */
+#define MLX5_FLOW_MARK_MAX 0xfffff0
+
+/* Default mark value used when none is provided. */
+#define MLX5_FLOW_MARK_DEFAULT 0xffffff
+
 /* Subset of struct mlx5_wqe_eth_seg. */
 struct mlx5_wqe_eth_seg_small {
 	uint32_t rsvd0;
@@ -183,10 +194,67 @@ struct mlx5_cqe {
 	uint8_t rsvd2[12];
 	uint32_t byte_cnt;
 	uint64_t timestamp;
-	uint8_t rsvd3[4];
+	uint32_t sop_drop_qpn;
 	uint16_t wqe_counter;
 	uint8_t rsvd4;
 	uint8_t op_own;
 };
 
+/**
+ * Convert a user mark to flow mark.
+ *
+ * @param val
+ *   Mark value to convert.
+ *
+ * @return
+ *   Converted mark value.
+ */
+static inline uint32_t
+mlx5_flow_mark_set(uint32_t val)
+{
+	uint32_t ret;
+
+	/*
+	 * Add one to the user value to differentiate un-marked flows from
+	 * marked flows.
+	 */
+	++val;
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+	/*
+	 * Mark is 24 bits (minus reserved values) but is stored on a 32 bit
+	 * word, byte-swapped by the kernel on little-endian systems. In this
+	 * case, left-shifting the resulting big-endian value ensures the
+	 * least significant 24 bits are retained when converting it back.
+	 */
+	ret = rte_cpu_to_be_32(val) >> 8;
+#else
+	ret = val;
+#endif
+	assert(ret <= MLX5_FLOW_MARK_MAX);
+	return ret;
+}
+
+/**
+ * Convert a mark to user mark.
+ *
+ * @param val
+ *   Mark value to convert.
+ *
+ * @return
+ *   Converted mark value.
+ */
+static inline uint32_t
+mlx5_flow_mark_get(uint32_t val)
+{
+	/*
+	 * Subtract one from the retrieved value. It was added by
+	 * mlx5_flow_mark_set() to distinguish unmarked flows.
+	 */
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+	return (val >> 8) - 1;
+#else
+	return val - 1;
+#endif
+}
+
 #endif /* RTE_PMD_MLX5_PRM_H_ */
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 6f86ded..8f0b4a6 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -113,7 +113,7 @@ static inline int
 check_cqe_seen(volatile struct mlx5_cqe *cqe)
 {
 	static const uint8_t magic[] = "seen";
-	volatile uint8_t (*buf)[sizeof(cqe->rsvd3)] = &cqe->rsvd3;
+	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
 	int ret = 1;
 	unsigned int i;
 
@@ -1357,6 +1357,16 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				pkt->hash.rss = rss_hash_res;
 				pkt->ol_flags = PKT_RX_RSS_HASH;
 			}
+			if (rxq->mark &&
+			    ((cqe->sop_drop_qpn !=
+			      htonl(MLX5_FLOW_MARK_INVALID)) ||
+			     (cqe->sop_drop_qpn !=
+			      htonl(MLX5_FLOW_MARK_DEFAULT)))) {
+				pkt->hash.fdir.hi =
+					mlx5_flow_mark_get(cqe->sop_drop_qpn);
+				pkt->ol_flags &= ~PKT_RX_RSS_HASH;
+				pkt->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
+			}
 			if (rxq->csum | rxq->csum_l2tun | rxq->vlan_strip |
 			    rxq->crc_present) {
 				if (rxq->csum) {
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e244c48..302ca49 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -114,7 +114,8 @@ struct rxq {
 	unsigned int elts_n:4; /* Log 2 of Mbufs. */
 	unsigned int port_id:8;
 	unsigned int rss_hash:1; /* RSS hash result is enabled. */
-	unsigned int :9; /* Remaining bits. */
+	unsigned int mark:1; /* Marked flow available on the queue. */
+	unsigned int :8; /* Remaining bits. */
 	volatile uint32_t *rq_db;
 	volatile uint32_t *cq_db;
 	uint16_t rq_ci;
-- 
2.1.4