[dpdk-dev] [PATCH v3 4/5] vdpa/mlx5: workaround guest MR registrations

[Matan Azrad <matan@nvidia.com>]

From: Michael Baum <michaelba@oss.nvidia.com>

Due to kernel issue in direct MKEY creation using the DevX API, this
patch replaces the virtio MR creation to use Verbs API.

Fixes: cc07a42da250 ("vdpa/mlx5: prepare memory regions")
Cc: stable@dpdk.org

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.h     |  8 +++---
 drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 41 +++++++++----------------------
 2 files changed, 16 insertions(+), 33 deletions(-)

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 15212a2b30..22617924ea 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -59,10 +59,10 @@ struct mlx5_vdpa_event_qp {
 
 struct mlx5_vdpa_query_mr {
 	SLIST_ENTRY(mlx5_vdpa_query_mr) next;
-	void *addr;
-	uint64_t length;
-	struct mlx5dv_devx_umem *umem;
-	struct mlx5_devx_obj *mkey;
+	union {
+		struct ibv_mr *mr;
+		struct mlx5_devx_obj *mkey;
+	};
 	int is_indirect;
 };
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index d7707bbd91..b1b9053bff 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -23,9 +23,10 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
 	entry = SLIST_FIRST(&priv->mr_list);
 	while (entry) {
 		next = SLIST_NEXT(entry, next);
-		claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
-		if (!entry->is_indirect)
-			claim_zero(mlx5_glue->devx_umem_dereg(entry->umem));
+		if (entry->is_indirect)
+			claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
+		else
+			claim_zero(mlx5_glue->dereg_mr(entry->mr));
 		SLIST_REMOVE(&priv->mr_list, entry, mlx5_vdpa_query_mr, next);
 		rte_free(entry);
 		entry = next;
@@ -202,7 +203,6 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
 		goto error;
 	}
 	DRV_LOG(DEBUG, "Dump fill Mkey = %u.", priv->null_mr->lkey);
-	memset(&mkey_attr, 0, sizeof(mkey_attr));
 	for (i = 0; i < mem->nregions; i++) {
 		reg = &mem->regions[i];
 		entry = rte_zmalloc(__func__, sizeof(*entry), 0);
@@ -211,28 +211,15 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
 			DRV_LOG(ERR, "Failed to allocate mem entry memory.");
 			goto error;
 		}
-		entry->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
-					 (void *)(uintptr_t)reg->host_user_addr,
-					     reg->size, IBV_ACCESS_LOCAL_WRITE);
-		if (!entry->umem) {
-			DRV_LOG(ERR, "Failed to register Umem by Devx.");
-			ret = -errno;
-			goto error;
-		}
-		mkey_attr.addr = (uintptr_t)(reg->guest_phys_addr);
-		mkey_attr.size = reg->size;
-		mkey_attr.umem_id = entry->umem->umem_id;
-		mkey_attr.pd = priv->cdev->pdn;
-		mkey_attr.pg_access = 1;
-		entry->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx,
-							&mkey_attr);
-		if (!entry->mkey) {
+		entry->mr = mlx5_glue->reg_mr_iova(priv->cdev->pd,
+				       (void *)(uintptr_t)(reg->host_user_addr),
+				       reg->size, reg->guest_phys_addr,
+				       IBV_ACCESS_LOCAL_WRITE);
+		if (!entry->mr) {
 			DRV_LOG(ERR, "Failed to create direct Mkey.");
 			ret = -rte_errno;
 			goto error;
 		}
-		entry->addr = (void *)(uintptr_t)(reg->host_user_addr);
-		entry->length = reg->size;
 		entry->is_indirect = 0;
 		if (i > 0) {
 			uint64_t sadd;
@@ -262,12 +249,13 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
 		for (k = 0; k < reg->size; k += klm_size) {
 			klm_array[klm_index].byte_count = k + klm_size >
 					   reg->size ? reg->size - k : klm_size;
-			klm_array[klm_index].mkey = entry->mkey->id;
+			klm_array[klm_index].mkey = entry->mr->lkey;
 			klm_array[klm_index].address = reg->guest_phys_addr + k;
 			klm_index++;
 		}
 		SLIST_INSERT_HEAD(&priv->mr_list, entry, next);
 	}
+	memset(&mkey_attr, 0, sizeof(mkey_attr));
 	mkey_attr.addr = (uintptr_t)(mem->regions[0].guest_phys_addr);
 	mkey_attr.size = mem_size;
 	mkey_attr.pd = priv->cdev->pdn;
@@ -295,13 +283,8 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
 	priv->gpa_mkey_index = entry->mkey->id;
 	return 0;
 error:
-	if (entry) {
-		if (entry->mkey)
-			mlx5_devx_cmd_destroy(entry->mkey);
-		if (entry->umem)
-			mlx5_glue->devx_umem_dereg(entry->umem);
+	if (entry)
 		rte_free(entry);
-	}
 	mlx5_vdpa_mem_dereg(priv);
 	rte_errno = -ret;
 	return ret;
-- 
2.25.1