[dpdk-dev] [PATCH v4 29/34] net/mlx5: make sample and mirror action thread safe

[Suanming Mou <suanmingm@nvidia.com>]

This commit uses cache list to make sample and mirror action thread
safe.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c |  14 ++
 drivers/net/mlx5/mlx5.h          |   5 +-
 drivers/net/mlx5/mlx5_flow.h     |  28 ++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 396 ++++++++++++++++++++++-----------------
 4 files changed, 260 insertions(+), 183 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 9ab3ba9..5856981 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -248,6 +248,20 @@
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
+	/* Init sample action cache list. */
+	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->sample_action_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_sample_create_cb,
+			     flow_dv_sample_match_cb,
+			     flow_dv_sample_remove_cb);
+	/* Init dest array action cache list. */
+	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->dest_array_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_dest_array_create_cb,
+			     flow_dv_dest_array_match_cb,
+			     flow_dv_dest_array_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 9efa354..ac7a026 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -661,8 +661,9 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *tag_table;
 	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
 	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	uint32_t sample_action_list; /* List of sample actions. */
-	uint32_t dest_array_list; /* List of destination array actions. */
+	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
+	struct mlx5_cache_list dest_array_list;
+	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 500a580..4a8c2bf 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -562,9 +562,12 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *verbs_action; /**< Verbs sample action object. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	union {
+		void *verbs_action; /**< Verbs sample action object. */
+		void **sub_actions; /**< Sample sub-action array. */
+	};
+	uint32_t idx; /** Sample object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint32_t ft_id; /** Flow Table Level */
 	uint32_t ratio;   /** Sample Ratio */
@@ -581,8 +584,8 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
 	void *action; /**< Pointer to the rdma core action. */
@@ -1449,4 +1452,19 @@ struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
 void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
 				 struct mlx5_cache_entry *entry);
 
+int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
+			    struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_sample_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
+			      struct mlx5_cache_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
+				struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_dest_array_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
+				  struct mlx5_cache_entry *entry);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 1866a21..39bd6b2 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -8634,30 +8634,43 @@ struct mlx5_hlist_entry *
 	}
 }
 
-/**
- * Find existing sample resource or create and register a new one.
- *
- * @param[in, out] dev
- *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to sample resource.
- * @parm[in, out] dev_flow
- *   Pointer to the dev_flow.
- * @param[in, out] sample_dv_actions
- *   Pointer to sample actions list.
- * @param[out] error
- *   pointer to error structure.
- *
- * @return
- *   0 on success otherwise -errno and errno is set.
- */
-static int
-flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_sample_resource *resource,
-			 struct mlx5_flow *dev_flow,
-			 void **sample_dv_actions,
-			 struct rte_flow_error *error)
+int
+flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
+			struct mlx5_cache_entry *entry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
+	struct mlx5_flow_dv_sample_resource *cache_resource =
+			container_of(entry, typeof(*cache_resource), entry);
+
+	if (resource->ratio == cache_resource->ratio &&
+	    resource->ft_type == cache_resource->ft_type &&
+	    resource->ft_id == cache_resource->ft_id &&
+	    resource->set_action == cache_resource->set_action &&
+	    !memcmp((void *)&resource->sample_act,
+		    (void *)&cache_resource->sample_act,
+		    sizeof(struct mlx5_flow_sub_actions_list))) {
+		/*
+		 * Existing sample action should release the prepared
+		 * sub-actions reference counter.
+		 */
+		flow_dv_sample_sub_actions_release(dev,
+						&resource->sample_idx);
+		return 0;
+	}
+	return 1;
+}
+
+struct mlx5_cache_entry *
+flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
+			 struct mlx5_cache_entry *entry __rte_unused,
+			 void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
+	void **sample_dv_actions = resource->sub_actions;
 	struct mlx5_flow_dv_sample_resource *cache_resource;
 	struct mlx5dv_dr_flow_sampler_attr sampler_attr;
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -8668,42 +8681,17 @@ struct mlx5_hlist_entry *
 	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
 	uint8_t is_egress = 0;
 	uint8_t is_transfer = 0;
+	struct rte_flow_error *error = ctx->error;
 
-	/* Lookup a matching resource from cache. */
-	ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
-		      idx, cache_resource, next) {
-		if (resource->ratio == cache_resource->ratio &&
-		    resource->ft_type == cache_resource->ft_type &&
-		    resource->ft_id == cache_resource->ft_id &&
-		    resource->set_action == cache_resource->set_action &&
-		    !memcmp((void *)&resource->sample_act,
-			    (void *)&cache_resource->sample_act,
-			    sizeof(struct mlx5_flow_sub_actions_list))) {
-			DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
-				(void *)cache_resource,
-				__atomic_load_n(&cache_resource->refcnt,
-						__ATOMIC_RELAXED));
-			__atomic_fetch_add(&cache_resource->refcnt, 1,
-					   __ATOMIC_RELAXED);
-			dev_flow->handle->dvh.rix_sample = idx;
-			dev_flow->dv.sample_res = cache_resource;
-			/*
-			 * Existing smaple action should release the prepared
-			 * sub-actions reference counter.
-			 */
-			flow_dv_sample_sub_actions_release(dev,
-							&resource->sample_idx);
-			return 0;
-		}
-	}
 	/* Register new sample resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
-				       &dev_flow->handle->dvh.rix_sample);
-	if (!cache_resource)
-		return rte_flow_error_set(error, ENOMEM,
+	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!cache_resource) {
+		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
+		return NULL;
+	}
 	*cache_resource = *resource;
 	/* Create normal path table level */
 	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
@@ -8712,7 +8700,7 @@ struct mlx5_hlist_entry *
 		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
 					is_egress, is_transfer,
-					dev_flow->external, NULL, 0, 0, error);
+					true, NULL, 0, 0, error);
 	if (!tbl) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -8751,15 +8739,8 @@ struct mlx5_hlist_entry *
 					NULL, "cannot create sample action");
 		goto error;
 	}
-	__atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
-	ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
-		     dev_flow->handle->dvh.rix_sample, cache_resource,
-		     next);
-	dev_flow->dv.sample_res = cache_resource;
-	DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
-		(void *)cache_resource,
-		__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
-	return 0;
+	cache_resource->idx = idx;
+	return &cache_resource->entry;
 error:
 	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB &&
 	    cache_resource->default_miss)
@@ -8771,19 +8752,18 @@ struct mlx5_hlist_entry *
 	if (cache_resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
 				cache_resource->normal_path_tbl);
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
-				dev_flow->handle->dvh.rix_sample);
-	dev_flow->handle->dvh.rix_sample = 0;
-	return -rte_errno;
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
+	return NULL;
+
 }
 
 /**
- * Find existing destination array resource or create and register a new one.
+ * Find existing sample resource or create and register a new one.
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
  * @param[in] resource
- *   Pointer to destination array resource.
+ *   Pointer to sample resource.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -8793,56 +8773,86 @@ struct mlx5_hlist_entry *
  *   0 on success otherwise -errno and errno is set.
  */
 static int
-flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_dest_array_resource *resource,
+flow_dv_sample_resource_register(struct rte_eth_dev *dev,
+			 struct mlx5_flow_dv_sample_resource *resource,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
+	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_cache_entry *entry;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_cb_ctx ctx = {
+		.dev = dev,
+		.error = error,
+		.data = resource,
+	};
+
+	entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
+	if (!entry)
+		return -rte_errno;
+	cache_resource = container_of(entry, typeof(*cache_resource), entry);
+	dev_flow->handle->dvh.rix_sample = cache_resource->idx;
+	dev_flow->dv.sample_res = cache_resource;
+	return 0;
+}
+
+int
+flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
+			    struct mlx5_cache_entry *entry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_dest_array_resource *cache_resource =
+			container_of(entry, typeof(*cache_resource), entry);
+	uint32_t idx = 0;
+
+	if (resource->num_of_dest == cache_resource->num_of_dest &&
+	    resource->ft_type == cache_resource->ft_type &&
+	    !memcmp((void *)cache_resource->sample_act,
+		    (void *)resource->sample_act,
+		   (resource->num_of_dest *
+		   sizeof(struct mlx5_flow_sub_actions_list)))) {
+		/*
+		 * Existing sample action should release the prepared
+		 * sub-actions reference counter.
+		 */
+		for (idx = 0; idx < resource->num_of_dest; idx++)
+			flow_dv_sample_sub_actions_release(dev,
+					&resource->sample_idx[idx]);
+		return 0;
+	}
+	return 1;
+}
+
+struct mlx5_cache_entry *
+flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
+			 struct mlx5_cache_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_dv_dest_array_resource *cache_resource;
+	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
 	struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
 	struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	struct mlx5_flow_sub_actions_list *sample_act;
 	struct mlx5dv_dr_domain *domain;
-	uint32_t idx = 0;
+	uint32_t idx = 0, res_idx = 0;
+	struct rte_flow_error *error = ctx->error;
 
-	/* Lookup a matching resource from cache. */
-	ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-		      sh->dest_array_list,
-		      idx, cache_resource, next) {
-		if (resource->num_of_dest == cache_resource->num_of_dest &&
-		    resource->ft_type == cache_resource->ft_type &&
-		    !memcmp((void *)cache_resource->sample_act,
-			    (void *)resource->sample_act,
-			   (resource->num_of_dest *
-			   sizeof(struct mlx5_flow_sub_actions_list)))) {
-			DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
-				(void *)cache_resource,
-				__atomic_load_n(&cache_resource->refcnt,
-						__ATOMIC_RELAXED));
-			__atomic_fetch_add(&cache_resource->refcnt, 1,
-					   __ATOMIC_RELAXED);
-			dev_flow->handle->dvh.rix_dest_array = idx;
-			dev_flow->dv.dest_array_res = cache_resource;
-			/*
-			 * Existing smaple action should release the prepared
-			 * sub-actions reference counter.
-			 */
-			for (idx = 0; idx < resource->num_of_dest; idx++)
-				flow_dv_sample_sub_actions_release(dev,
-						&resource->sample_idx[idx]);
-			return 0;
-		}
-	}
 	/* Register new destination array resource. */
 	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-				       &dev_flow->handle->dvh.rix_dest_array);
-	if (!cache_resource)
-		return rte_flow_error_set(error, ENOMEM,
+					    &res_idx);
+	if (!cache_resource) {
+		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
+		return NULL;
+	}
 	*cache_resource = *resource;
 	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
@@ -8891,18 +8901,10 @@ struct mlx5_hlist_entry *
 				   "cannot create destination array action");
 		goto error;
 	}
-	__atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
-	ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-		     &sh->dest_array_list,
-		     dev_flow->handle->dvh.rix_dest_array, cache_resource,
-		     next);
-	dev_flow->dv.dest_array_res = cache_resource;
-	DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
-		(void *)cache_resource,
-		__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+	cache_resource->idx = res_idx;
 	for (idx = 0; idx < resource->num_of_dest; idx++)
 		mlx5_free(dest_attr[idx]);
-	return 0;
+	return &cache_resource->entry;
 error:
 	for (idx = 0; idx < resource->num_of_dest; idx++) {
 		struct mlx5_flow_sub_actions_idx *act_res =
@@ -8923,10 +8925,47 @@ struct mlx5_hlist_entry *
 			mlx5_free(dest_attr[idx]);
 	}
 
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-				dev_flow->handle->dvh.rix_dest_array);
-	dev_flow->handle->dvh.rix_dest_array = 0;
-	return -rte_errno;
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
+	return NULL;
+}
+
+/**
+ * Find existing destination array resource or create and register a new one.
+ *
+ * @param[in, out] dev
+ *   Pointer to rte_eth_dev structure.
+ * @param[in] resource
+ *   Pointer to destination array resource.
+ * @parm[in, out] dev_flow
+ *   Pointer to the dev_flow.
+ * @param[out] error
+ *   pointer to error structure.
+ *
+ * @return
+ *   0 on success otherwise -errno and errno is set.
+ */
+static int
+flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
+			 struct mlx5_flow_dv_dest_array_resource *resource,
+			 struct mlx5_flow *dev_flow,
+			 struct rte_flow_error *error)
+{
+	struct mlx5_flow_dv_dest_array_resource *cache_resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_cb_ctx ctx = {
+		.dev = dev,
+		.error = error,
+		.data = resource,
+	};
+
+	entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
+	if (!entry)
+		return -rte_errno;
+	cache_resource = container_of(entry, typeof(*cache_resource), entry);
+	dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
+	dev_flow->dv.dest_array_res = cache_resource;
+	return 0;
 }
 
 /**
@@ -9233,8 +9272,8 @@ struct mlx5_hlist_entry *
 						  NULL, "can't create sample "
 						  "action");
 	} else {
-		if (flow_dv_sample_resource_register(dev, res, dev_flow,
-						     sample_actions, error))
+		res->sub_actions = sample_actions;
+		if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  NULL,
@@ -10688,6 +10727,34 @@ struct mlx5_hlist_entry *
 	handle->rix_fate = 0;
 }
 
+void
+flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
+			 struct mlx5_cache_entry *entry)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_dv_sample_resource *cache_resource =
+			container_of(entry, typeof(*cache_resource), entry);
+
+	if (cache_resource->verbs_action)
+		claim_zero(mlx5_glue->destroy_flow_action
+				(cache_resource->verbs_action));
+	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
+		if (cache_resource->default_miss)
+			claim_zero(mlx5_glue->destroy_flow_action
+			  (cache_resource->default_miss));
+	}
+	if (cache_resource->normal_path_tbl)
+		flow_dv_tbl_resource_release(MLX5_SH(dev),
+			cache_resource->normal_path_tbl);
+	flow_dv_sample_sub_actions_release(dev,
+				&cache_resource->sample_idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+			cache_resource->idx);
+	DRV_LOG(DEBUG, "sample resource %p: removed",
+		(void *)cache_resource);
+}
+
 /**
  * Release an sample resource.
  *
@@ -10704,41 +10771,38 @@ struct mlx5_hlist_entry *
 				     struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t idx = handle->dvh.rix_sample;
 	struct mlx5_flow_dv_sample_resource *cache_resource;
 
 	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			 idx);
+			 handle->dvh.rix_sample);
 	if (!cache_resource)
 		return 0;
 	MLX5_ASSERT(cache_resource->verbs_action);
-	DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
-		(void *)cache_resource,
-		__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
-	if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
-			       __ATOMIC_RELAXED) == 0) {
-		if (cache_resource->verbs_action)
-			claim_zero(mlx5_glue->destroy_flow_action
-					(cache_resource->verbs_action));
-		if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
-			if (cache_resource->default_miss)
-				claim_zero(mlx5_glue->destroy_flow_action
-				  (cache_resource->default_miss));
-		}
-		if (cache_resource->normal_path_tbl)
-			flow_dv_tbl_resource_release(MLX5_SH(dev),
-				cache_resource->normal_path_tbl);
+	return mlx5_cache_unregister(&priv->sh->sample_action_list,
+				     &cache_resource->entry);
+}
+
+void
+flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_dv_dest_array_resource *cache_resource =
+			container_of(entry, typeof(*cache_resource), entry);
+	uint32_t i = 0;
+
+	MLX5_ASSERT(cache_resource->action);
+	if (cache_resource->action)
+		claim_zero(mlx5_glue->destroy_flow_action
+					(cache_resource->action));
+	for (; i < cache_resource->num_of_dest; i++)
 		flow_dv_sample_sub_actions_release(dev,
-					&cache_resource->sample_idx);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			     &priv->sh->sample_action_list, idx,
-			     cache_resource, next);
-		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
-		DRV_LOG(DEBUG, "sample resource %p: removed",
-			(void *)cache_resource);
-		return 0;
-	}
-	return 1;
+				&cache_resource->sample_idx[i]);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+			cache_resource->idx);
+	DRV_LOG(DEBUG, "destination array resource %p: removed",
+		(void *)cache_resource);
 }
 
 /**
@@ -10754,38 +10818,18 @@ struct mlx5_hlist_entry *
  */
 static int
 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
-				     struct mlx5_flow_handle *handle)
+				    struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	uint32_t idx = handle->dvh.rix_dest_array;
-	uint32_t i = 0;
+	struct mlx5_flow_dv_dest_array_resource *cache;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			 idx);
-	if (!cache_resource)
-		return 0;
-	MLX5_ASSERT(cache_resource->action);
-	DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
-		(void *)cache_resource,
-		__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
-	if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
-			       __ATOMIC_RELAXED) == 0) {
-		if (cache_resource->action)
-			claim_zero(mlx5_glue->destroy_flow_action
-						(cache_resource->action));
-		for (; i < cache_resource->num_of_dest; i++)
-			flow_dv_sample_sub_actions_release(dev,
-					&cache_resource->sample_idx[i]);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			     &priv->sh->dest_array_list, idx,
-			     cache_resource, next);
-		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
-		DRV_LOG(DEBUG, "destination array resource %p: removed",
-			(void *)cache_resource);
+	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+			       handle->dvh.rix_dest_array);
+	if (!cache)
 		return 0;
-	}
-	return 1;
+	MLX5_ASSERT(cache->action);
+	return mlx5_cache_unregister(&priv->sh->dest_array_list,
+				     &cache->entry);
 }
 
 /**
-- 
1.8.3.1