[dpdk-dev] [PATCH v2 6/8] net/mlx5: make shared counters thread safe

[Suanming Mou <suanmingm@nvidia.com>]

The shared counters save the counter index to three level table. As
three level table supports multiple-thread opertations now, the shared
counters can take advantage of the table to support multiple-thread.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.h            |   7 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 124 +++++++++++++++++++------------------
 drivers/net/mlx5/mlx5_flow_verbs.c |  19 ++----
 3 files changed, 73 insertions(+), 77 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e314668..2598fa2 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -348,10 +348,15 @@ struct flow_counter_stats {
 
 /* Shared counters information for counters. */
 struct mlx5_flow_counter_shared {
-	uint32_t ref_cnt; /**< Reference counter. */
 	uint32_t id; /**< User counter ID. */
 };
 
+/* Shared counter configuration. */
+struct mlx5_shared_counter_conf {
+	struct rte_eth_dev *dev; /* The device shared counter belongs to. */
+	uint32_t id; /* The shared counter ID. */
+};
+
 struct mlx5_flow_counter_pool;
 /* Generic counters information. */
 struct mlx5_flow_counter {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 6226d87..067ef0f 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -4903,36 +4903,10 @@ struct field_modify_info modify_tcp[] = {
 }
 
 /**
- * Search for existed shared counter.
- *
- * @param[in] dev
- *   Pointer to the Ethernet device structure.
- * @param[in] id
- *   The shared counter ID to search.
- *
- * @return
- *   0 if not existed, otherwise shared counter index.
- */
-static uint32_t
-flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	union mlx5_l3t_data data;
-
-	if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data))
-		return 0;
-	return data.dword;
-}
-
-/**
  * Allocate a flow counter.
  *
  * @param[in] dev
  *   Pointer to the Ethernet device structure.
- * @param[in] shared
- *   Indicate if this counter is shared with other flows.
- * @param[in] id
- *   Counter identifier.
  * @param[in] age
  *   Whether the counter was allocated for aging.
  *
@@ -4940,8 +4914,7 @@ struct field_modify_info modify_tcp[] = {
  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
  */
 static uint32_t
-flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
-		      uint32_t age)
+flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_counter_pool *pool = NULL;
@@ -4957,19 +4930,6 @@ struct field_modify_info modify_tcp[] = {
 		rte_errno = ENOTSUP;
 		return 0;
 	}
-	if (shared) {
-		cnt_idx = flow_dv_counter_shared_search(dev, id);
-		if (cnt_idx) {
-			cnt_free = flow_dv_counter_get_by_idx(dev, cnt_idx,
-							      NULL);
-			if (cnt_free->shared_info.ref_cnt + 1 == 0) {
-				rte_errno = E2BIG;
-				return 0;
-			}
-			cnt_free->shared_info.ref_cnt++;
-			return cnt_idx;
-		}
-	}
 	/* Get free counters from container. */
 	rte_spinlock_lock(&cmng->csl[cnt_type]);
 	cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
@@ -5007,16 +4967,6 @@ struct field_modify_info modify_tcp[] = {
 	if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
 				 &cnt_free->bytes))
 		goto err;
-	if (shared) {
-		union mlx5_l3t_data data;
-
-		data.dword = cnt_idx;
-		if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
-			goto err;
-		cnt_free->shared_info.ref_cnt = 1;
-		cnt_free->shared_info.id = id;
-		cnt_idx |= MLX5_CNT_SHARED_OFFSET;
-	}
 	if (!fallback && !priv->sh->cmng.query_thread_on)
 		/* Start the asynchronous batch query by the host thread. */
 		mlx5_set_query_alarm(priv->sh);
@@ -5032,6 +4982,60 @@ struct field_modify_info modify_tcp[] = {
 }
 
 /**
+ * Allocate a shared flow counter.
+ *
+ * @param[in] ctx
+ *   Pointer to the shared counter configuration.
+ * @param[in] data
+ *   Pointer to save the allocated counter index.
+ *
+ * @return
+ *   Index to flow counter on success, 0 otherwise and rte_errno is set.
+ */
+
+static int32_t
+flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
+{
+	struct mlx5_shared_counter_conf *conf = ctx;
+	struct rte_eth_dev *dev = conf->dev;
+	struct mlx5_flow_counter *cnt;
+
+	data->dword = flow_dv_counter_alloc(dev, 0);
+	data->dword |= MLX5_CNT_SHARED_OFFSET;
+	cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
+	cnt->shared_info.id = conf->id;
+	return 0;
+}
+
+/**
+ * Get a shared flow counter.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] id
+ *   Counter identifier.
+ *
+ * @return
+ *   Index to flow counter on success, 0 otherwise and rte_errno is set.
+ */
+static uint32_t
+flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_shared_counter_conf conf = {
+		.dev = dev,
+		.id = id,
+	};
+	union mlx5_l3t_data data = {
+		.dword = 0,
+	};
+
+	mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
+			       flow_dv_counter_alloc_shared_cb, &conf);
+	return data.dword;
+}
+
+/**
  * Get age param from counter index.
  *
  * @param[in] dev
@@ -5110,12 +5114,9 @@ struct field_modify_info modify_tcp[] = {
 		return;
 	cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
 	MLX5_ASSERT(pool);
-	if (IS_SHARED_CNT(counter)) {
-		if (--cnt->shared_info.ref_cnt)
-			return;
-		mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
-				     cnt->shared_info.id);
-	}
+	if (IS_SHARED_CNT(counter) &&
+	    mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
+		return;
 	if (IS_AGE_POOL(pool))
 		flow_dv_counter_remove_from_age(dev, counter, cnt);
 	cnt->pool = pool;
@@ -8271,9 +8272,10 @@ struct field_modify_info modify_tcp[] = {
 	uint32_t counter;
 	struct mlx5_age_param *age_param;
 
-	counter = flow_dv_counter_alloc(dev,
-				count ? count->shared : 0,
-				count ? count->id : 0, !!age);
+	if (count && count->shared)
+		counter = flow_dv_counter_get_shared(dev, count->id);
+	else
+		counter = flow_dv_counter_alloc(dev, !!age);
 	if (!counter || age == NULL)
 		return counter;
 	age_param  = flow_dv_counter_idx_get_age(dev, counter);
@@ -11442,7 +11444,7 @@ struct field_modify_info modify_tcp[] = {
 	uint32_t cnt;
 
 	flow_dv_shared_lock(dev);
-	cnt = flow_dv_counter_alloc(dev, 0, 0, 0);
+	cnt = flow_dv_counter_alloc(dev, 0);
 	flow_dv_shared_unlock(dev);
 	return cnt;
 }
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 5df2209..1fd5972 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -265,15 +265,8 @@
 	int ret;
 
 	if (shared && !mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) &&
-	    data.dword) {
-		cnt = flow_verbs_counter_get_by_idx(dev, data.dword, NULL);
-		if (cnt->shared_info.ref_cnt + 1 == 0) {
-			rte_errno = E2BIG;
-			return 0;
-		}
-		cnt->shared_info.ref_cnt++;
+	    data.dword)
 		return data.dword;
-	}
 	for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 		pool = cmng->pools[pool_idx];
 		if (!pool)
@@ -325,7 +318,6 @@
 		data.dword = cnt_idx;
 		if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
 			return 0;
-		cnt->shared_info.ref_cnt = 1;
 		cnt->shared_info.id = id;
 		cnt_idx |= MLX5_CNT_SHARED_OFFSET;
 	}
@@ -360,12 +352,9 @@
 	struct mlx5_flow_counter_ext *cnt_ext;
 
 	cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool);
-	if (IS_SHARED_CNT(counter)) {
-		if (--cnt->shared_info.ref_cnt)
-			return;
-		mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
-				     cnt->shared_info.id);
-	}
+	if (IS_SHARED_CNT(counter) &&
+	    mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
+		return;
 	cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 	claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs));
-- 
1.8.3.1