[PATCH V2 0/2] vDPA/ifcvf: implement management netlink framework

[PATCH V2 0/2] vDPA/ifcvf: implement management netlink framework

[Zhu Lingshan]

This series implements vDPA management netlink for ifcvf.

Please help review

Thanks!

Changes from V1:
(1)handle adapter related error handling in ifcvf_vdpa_dev_add
(Jason and Michael)
(2)relace vdpa_unregister_device() with vdpa_mgmtdev_unregister()
in ifcvf_remove (Jason)
(3)squash patch 3 to patch 2(Jason)

Zhu Lingshan (2):
  vDPA/ifcvf: introduce get_dev_type() which returns virtio dev id
  vDPA/ifcvf: implement management netlink framework for ifcvf

 drivers/vdpa/ifcvf/ifcvf_base.h |   6 +
 drivers/vdpa/ifcvf/ifcvf_main.c | 188 +++++++++++++++++++++++---------
 2 files changed, 145 insertions(+), 49 deletions(-)

-- 
2.27.0

[PATCH V2 1/2] vDPA/ifcvf: introduce get_dev_type() which returns virtio dev id

[Zhu Lingshan]

This commit introduces a new function get_dev_type() which returns
the virtio device id of a device, to avoid duplicated code.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_main.c | 34 ++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index bc1d59f316d1..5f70ab1283a0 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -442,6 +442,26 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.set_config_cb  = ifcvf_vdpa_set_config_cb,
 };
 
+static u32 get_dev_type(struct pci_dev *pdev)
+{
+	u32 dev_type;
+
+	/* This drirver drives both modern virtio devices and transitional
+	 * devices in modern mode.
+	 * vDPA requires feature bit VIRTIO_F_ACCESS_PLATFORM,
+	 * so legacy devices and transitional devices in legacy
+	 * mode will not work for vDPA, this driver will not
+	 * drive devices with legacy interface.
+	 */
+
+	if (pdev->device < 0x1040)
+		dev_type =  pdev->subsystem_device;
+	else
+		dev_type =  pdev->device - 0x1040;
+
+	return dev_type;
+}
+
 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	struct device *dev = &pdev->dev;
@@ -486,19 +506,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	pci_set_drvdata(pdev, adapter);
 
 	vf = &adapter->vf;
-
-	/* This drirver drives both modern virtio devices and transitional
-	 * devices in modern mode.
-	 * vDPA requires feature bit VIRTIO_F_ACCESS_PLATFORM,
-	 * so legacy devices and transitional devices in legacy
-	 * mode will not work for vDPA, this driver will not
-	 * drive devices with legacy interface.
-	 */
-	if (pdev->device < 0x1040)
-		vf->dev_type =  pdev->subsystem_device;
-	else
-		vf->dev_type =  pdev->device - 0x1040;
-
+	vf->dev_type = get_dev_type(pdev);
 	vf->base = pcim_iomap_table(pdev);
 
 	adapter->pdev = pdev;
-- 
2.27.0

[PATCH V2 2/2] vDPA/ifcvf: implement management netlink framework for ifcvf

[Zhu Lingshan]

This commit implments the management netlink framework for ifcvf,
including register and add / remove a device

It works with iprouter2:
[root@localhost lszhu]# vdpa mgmtdev show -jp
{
    "mgmtdev": {
        "pci/0000:01:00.5": {
            "supported_classes": [ "net" ]
        },
        "pci/0000:01:00.6": {
            "supported_classes": [ "net" ]
        }
    }
}

[root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.5 name vdpa0
[root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.6 name vdpa1

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.h |   6 ++
 drivers/vdpa/ifcvf/ifcvf_main.c | 154 ++++++++++++++++++++++++--------
 2 files changed, 124 insertions(+), 36 deletions(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index ded1b1b5fb13..e5251fcbb200 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -104,6 +104,12 @@ struct ifcvf_lm_cfg {
 	struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUE_PAIRS];
 };
 
+struct ifcvf_vdpa_mgmt_dev {
+	struct vdpa_mgmt_dev mdev;
+	struct ifcvf_adapter *adapter;
+	struct pci_dev *pdev;
+};
+
 int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
 int ifcvf_start_hw(struct ifcvf_hw *hw);
 void ifcvf_stop_hw(struct ifcvf_hw *hw);
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index 5f70ab1283a0..c72d9b36e4a0 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -218,7 +218,7 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
 	int ret;
 
 	vf  = vdpa_to_vf(vdpa_dev);
-	adapter = dev_get_drvdata(vdpa_dev->dev.parent);
+	adapter = vdpa_to_adapter(vdpa_dev);
 	status_old = ifcvf_get_status(vf);
 
 	if (status_old == status)
@@ -442,6 +442,16 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.set_config_cb  = ifcvf_vdpa_set_config_cb,
 };
 
+static struct virtio_device_id id_table_net[] = {
+	{VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
+	{0},
+};
+
+static struct virtio_device_id id_table_blk[] = {
+	{VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID},
+	{0},
+};
+
 static u32 get_dev_type(struct pci_dev *pdev)
 {
 	u32 dev_type;
@@ -462,48 +472,30 @@ static u32 get_dev_type(struct pci_dev *pdev)
 	return dev_type;
 }
 
-static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
 {
-	struct device *dev = &pdev->dev;
+	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
 	struct ifcvf_adapter *adapter;
+	struct pci_dev *pdev;
 	struct ifcvf_hw *vf;
+	struct device *dev;
 	int ret, i;
 
-	ret = pcim_enable_device(pdev);
-	if (ret) {
-		IFCVF_ERR(pdev, "Failed to enable device\n");
-		return ret;
-	}
-
-	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
-				 IFCVF_DRIVER_NAME);
-	if (ret) {
-		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
-		return ret;
-	}
-
-	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
-	if (ret) {
-		IFCVF_ERR(pdev, "No usable DMA configuration\n");
-		return ret;
-	}
-
-	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
-	if (ret) {
-		IFCVF_ERR(pdev,
-			  "Failed for adding devres for freeing irq vectors\n");
-		return ret;
-	}
+	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
+	if (ifcvf_mgmt_dev->adapter)
+		return -EOPNOTSUPP;
 
+	pdev = ifcvf_mgmt_dev->pdev;
+	dev = &pdev->dev;
 	adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
-				    dev, &ifc_vdpa_ops, NULL);
-	if (adapter == NULL) {
+				    dev, &ifc_vdpa_ops, name);
+	if (!adapter) {
 		IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
 		return -ENOMEM;
 	}
 
-	pci_set_master(pdev);
-	pci_set_drvdata(pdev, adapter);
+	ifcvf_mgmt_dev->adapter = adapter;
+	pci_set_drvdata(pdev, ifcvf_mgmt_dev);
 
 	vf = &adapter->vf;
 	vf->dev_type = get_dev_type(pdev);
@@ -523,9 +515,10 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	vf->hw_features = ifcvf_get_hw_features(vf);
 
-	ret = vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
+	adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
+	ret = _vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
 	if (ret) {
-		IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus");
+		IFCVF_ERR(pdev, "Failed to register to vDPA bus");
 		goto err;
 	}
 
@@ -536,11 +529,100 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return ret;
 }
 
+static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
+{
+	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
+
+	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
+	_vdpa_unregister_device(dev);
+	ifcvf_mgmt_dev->adapter = NULL;
+}
+
+static const struct vdpa_mgmtdev_ops ifcvf_vdpa_mgmt_dev_ops = {
+	.dev_add = ifcvf_vdpa_dev_add,
+	.dev_del = ifcvf_vdpa_dev_del
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
+	struct device *dev = &pdev->dev;
+	u32 dev_type;
+	int ret;
+
+	ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
+	if (!ifcvf_mgmt_dev) {
+		IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
+		return -ENOMEM;
+	}
+
+	dev_type = get_dev_type(pdev);
+	switch (dev_type) {
+	case VIRTIO_ID_NET:
+		ifcvf_mgmt_dev->mdev.id_table = id_table_net;
+		break;
+	case VIRTIO_ID_BLOCK:
+		ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
+		break;
+	default:
+		IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
+		ret = -EOPNOTSUPP;
+		goto err;
+	}
+
+	ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
+	ifcvf_mgmt_dev->mdev.device = dev;
+	ifcvf_mgmt_dev->pdev = pdev;
+
+	ret = pcim_enable_device(pdev);
+	if (ret) {
+		IFCVF_ERR(pdev, "Failed to enable device\n");
+		goto err;
+	}
+
+	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
+				 IFCVF_DRIVER_NAME);
+	if (ret) {
+		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
+		goto err;
+	}
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+	if (ret) {
+		IFCVF_ERR(pdev, "No usable DMA configuration\n");
+		goto err;
+	}
+
+	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
+	if (ret) {
+		IFCVF_ERR(pdev,
+			  "Failed for adding devres for freeing irq vectors\n");
+		goto err;
+	}
+
+	pci_set_master(pdev);
+
+	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
+	if (ret) {
+		IFCVF_ERR(pdev,
+			  "Failed to initialize the management interfaces\n");
+		goto err;
+	}
+
+	return 0;
+
+err:
+	kfree(ifcvf_mgmt_dev);
+	return ret;
+}
+
 static void ifcvf_remove(struct pci_dev *pdev)
 {
-	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
 
-	vdpa_unregister_device(&adapter->vdpa);
+	ifcvf_mgmt_dev = pci_get_drvdata(pdev);
+	vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
+	kfree(ifcvf_mgmt_dev);
 }
 
 static struct pci_device_id ifcvf_pci_ids[] = {
-- 
2.27.0

Re: [PATCH V2 2/2] vDPA/ifcvf: implement management netlink framework for ifcvf

[Michael S. Tsirkin]

On Mon, Jul 05, 2021 at 10:13:33PM +0800, Zhu Lingshan wrote:
> This commit implments the management netlink framework for ifcvf,

implements

> including register and add / remove a device
> 
> It works with iprouter2:

I am guessing iproute2?

> [root@localhost lszhu]# vdpa mgmtdev show -jp
> {
>     "mgmtdev": {
>         "pci/0000:01:00.5": {
>             "supported_classes": [ "net" ]
>         },
>         "pci/0000:01:00.6": {
>             "supported_classes": [ "net" ]
>         }
>     }
> }
> 
> [root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.5 name vdpa0
> [root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.6 name vdpa1
> 
> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
> ---
>  drivers/vdpa/ifcvf/ifcvf_base.h |   6 ++
>  drivers/vdpa/ifcvf/ifcvf_main.c | 154 ++++++++++++++++++++++++--------
>  2 files changed, 124 insertions(+), 36 deletions(-)
> 
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
> index ded1b1b5fb13..e5251fcbb200 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
> @@ -104,6 +104,12 @@ struct ifcvf_lm_cfg {
>  	struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUE_PAIRS];
>  };
>  
> +struct ifcvf_vdpa_mgmt_dev {
> +	struct vdpa_mgmt_dev mdev;
> +	struct ifcvf_adapter *adapter;
> +	struct pci_dev *pdev;
> +};
> +
>  int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
>  int ifcvf_start_hw(struct ifcvf_hw *hw);
>  void ifcvf_stop_hw(struct ifcvf_hw *hw);
> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> index 5f70ab1283a0..c72d9b36e4a0 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> @@ -218,7 +218,7 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
>  	int ret;
>  
>  	vf  = vdpa_to_vf(vdpa_dev);
> -	adapter = dev_get_drvdata(vdpa_dev->dev.parent);
> +	adapter = vdpa_to_adapter(vdpa_dev);
>  	status_old = ifcvf_get_status(vf);
>  
>  	if (status_old == status)
> @@ -442,6 +442,16 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
>  	.set_config_cb  = ifcvf_vdpa_set_config_cb,
>  };
>  
> +static struct virtio_device_id id_table_net[] = {
> +	{VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
> +	{0},
> +};
> +
> +static struct virtio_device_id id_table_blk[] = {
> +	{VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID},
> +	{0},
> +};
> +
>  static u32 get_dev_type(struct pci_dev *pdev)
>  {
>  	u32 dev_type;
> @@ -462,48 +472,30 @@ static u32 get_dev_type(struct pci_dev *pdev)
>  	return dev_type;
>  }
>  
> -static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> +static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
>  {
> -	struct device *dev = &pdev->dev;
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>  	struct ifcvf_adapter *adapter;
> +	struct pci_dev *pdev;
>  	struct ifcvf_hw *vf;
> +	struct device *dev;
>  	int ret, i;
>  
> -	ret = pcim_enable_device(pdev);
> -	if (ret) {
> -		IFCVF_ERR(pdev, "Failed to enable device\n");
> -		return ret;
> -	}
> -
> -	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
> -				 IFCVF_DRIVER_NAME);
> -	if (ret) {
> -		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
> -		return ret;
> -	}
> -
> -	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
> -	if (ret) {
> -		IFCVF_ERR(pdev, "No usable DMA configuration\n");
> -		return ret;
> -	}
> -
> -	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
> -	if (ret) {
> -		IFCVF_ERR(pdev,
> -			  "Failed for adding devres for freeing irq vectors\n");
> -		return ret;
> -	}
> +	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
> +	if (ifcvf_mgmt_dev->adapter)
> +		return -EOPNOTSUPP;
>  
> +	pdev = ifcvf_mgmt_dev->pdev;
> +	dev = &pdev->dev;
>  	adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
> -				    dev, &ifc_vdpa_ops, NULL);
> -	if (adapter == NULL) {
> +				    dev, &ifc_vdpa_ops, name);
> +	if (!adapter) {
>  		IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
>  		return -ENOMEM;
>  	}
>  
> -	pci_set_master(pdev);
> -	pci_set_drvdata(pdev, adapter);
> +	ifcvf_mgmt_dev->adapter = adapter;
> +	pci_set_drvdata(pdev, ifcvf_mgmt_dev);
>  
>  	vf = &adapter->vf;
>  	vf->dev_type = get_dev_type(pdev);
> @@ -523,9 +515,10 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>  
>  	vf->hw_features = ifcvf_get_hw_features(vf);
>  
> -	ret = vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
> +	adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
> +	ret = _vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
>  	if (ret) {
> -		IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus");
> +		IFCVF_ERR(pdev, "Failed to register to vDPA bus");
>  		goto err;
>  	}
>  
> @@ -536,11 +529,100 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>  	return ret;
>  }
>  
> +static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
> +{
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
> +
> +	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
> +	_vdpa_unregister_device(dev);
> +	ifcvf_mgmt_dev->adapter = NULL;
> +}
> +
> +static const struct vdpa_mgmtdev_ops ifcvf_vdpa_mgmt_dev_ops = {
> +	.dev_add = ifcvf_vdpa_dev_add,
> +	.dev_del = ifcvf_vdpa_dev_del
> +};
> +
> +static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> +{
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
> +	struct device *dev = &pdev->dev;
> +	u32 dev_type;
> +	int ret;
> +
> +	ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
> +	if (!ifcvf_mgmt_dev) {
> +		IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
> +		return -ENOMEM;
> +	}
> +
> +	dev_type = get_dev_type(pdev);
> +	switch (dev_type) {
> +	case VIRTIO_ID_NET:
> +		ifcvf_mgmt_dev->mdev.id_table = id_table_net;
> +		break;
> +	case VIRTIO_ID_BLOCK:
> +		ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
> +		break;
> +	default:
> +		IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
> +		ret = -EOPNOTSUPP;
> +		goto err;
> +	}
> +
> +	ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
> +	ifcvf_mgmt_dev->mdev.device = dev;
> +	ifcvf_mgmt_dev->pdev = pdev;
> +
> +	ret = pcim_enable_device(pdev);
> +	if (ret) {
> +		IFCVF_ERR(pdev, "Failed to enable device\n");
> +		goto err;
> +	}
> +
> +	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
> +				 IFCVF_DRIVER_NAME);
> +	if (ret) {
> +		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
> +		goto err;
> +	}
> +
> +	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
> +	if (ret) {
> +		IFCVF_ERR(pdev, "No usable DMA configuration\n");
> +		goto err;
> +	}
> +
> +	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
> +	if (ret) {
> +		IFCVF_ERR(pdev,
> +			  "Failed for adding devres for freeing irq vectors\n");
> +		goto err;
> +	}
> +
> +	pci_set_master(pdev);
> +
> +	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
> +	if (ret) {
> +		IFCVF_ERR(pdev,
> +			  "Failed to initialize the management interfaces\n");
> +		goto err;
> +	}
> +
> +	return 0;
> +
> +err:
> +	kfree(ifcvf_mgmt_dev);
> +	return ret;
> +}
> +
>  static void ifcvf_remove(struct pci_dev *pdev)
>  {
> -	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>  
> -	vdpa_unregister_device(&adapter->vdpa);
> +	ifcvf_mgmt_dev = pci_get_drvdata(pdev);
> +	vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
> +	kfree(ifcvf_mgmt_dev);
>  }
>  
>  static struct pci_device_id ifcvf_pci_ids[] = {
> -- 
> 2.27.0

Re: [PATCH V2 2/2] vDPA/ifcvf: implement management netlink framework for ifcvf

[Michael S. Tsirkin]

On Mon, Jul 05, 2021 at 10:13:33PM +0800, Zhu Lingshan wrote:
> This commit implments the management netlink framework for ifcvf,

implements

> including register and add / remove a device
> 
> It works with iprouter2:

I am guessing iproute2?

> [root@localhost lszhu]# vdpa mgmtdev show -jp
> {
>     "mgmtdev": {
>         "pci/0000:01:00.5": {
>             "supported_classes": [ "net" ]
>         },
>         "pci/0000:01:00.6": {
>             "supported_classes": [ "net" ]
>         }
>     }
> }
> 
> [root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.5 name vdpa0
> [root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.6 name vdpa1
> 
> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
> ---
>  drivers/vdpa/ifcvf/ifcvf_base.h |   6 ++
>  drivers/vdpa/ifcvf/ifcvf_main.c | 154 ++++++++++++++++++++++++--------
>  2 files changed, 124 insertions(+), 36 deletions(-)
> 
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
> index ded1b1b5fb13..e5251fcbb200 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
> @@ -104,6 +104,12 @@ struct ifcvf_lm_cfg {
>  	struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUE_PAIRS];
>  };
>  
> +struct ifcvf_vdpa_mgmt_dev {
> +	struct vdpa_mgmt_dev mdev;
> +	struct ifcvf_adapter *adapter;
> +	struct pci_dev *pdev;
> +};
> +
>  int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
>  int ifcvf_start_hw(struct ifcvf_hw *hw);
>  void ifcvf_stop_hw(struct ifcvf_hw *hw);
> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> index 5f70ab1283a0..c72d9b36e4a0 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> @@ -218,7 +218,7 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
>  	int ret;
>  
>  	vf  = vdpa_to_vf(vdpa_dev);
> -	adapter = dev_get_drvdata(vdpa_dev->dev.parent);
> +	adapter = vdpa_to_adapter(vdpa_dev);
>  	status_old = ifcvf_get_status(vf);
>  
>  	if (status_old == status)
> @@ -442,6 +442,16 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
>  	.set_config_cb  = ifcvf_vdpa_set_config_cb,
>  };
>  
> +static struct virtio_device_id id_table_net[] = {
> +	{VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
> +	{0},
> +};
> +
> +static struct virtio_device_id id_table_blk[] = {
> +	{VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID},
> +	{0},
> +};
> +
>  static u32 get_dev_type(struct pci_dev *pdev)
>  {
>  	u32 dev_type;
> @@ -462,48 +472,30 @@ static u32 get_dev_type(struct pci_dev *pdev)
>  	return dev_type;
>  }
>  
> -static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> +static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
>  {
> -	struct device *dev = &pdev->dev;
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>  	struct ifcvf_adapter *adapter;
> +	struct pci_dev *pdev;
>  	struct ifcvf_hw *vf;
> +	struct device *dev;
>  	int ret, i;
>  
> -	ret = pcim_enable_device(pdev);
> -	if (ret) {
> -		IFCVF_ERR(pdev, "Failed to enable device\n");
> -		return ret;
> -	}
> -
> -	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
> -				 IFCVF_DRIVER_NAME);
> -	if (ret) {
> -		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
> -		return ret;
> -	}
> -
> -	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
> -	if (ret) {
> -		IFCVF_ERR(pdev, "No usable DMA configuration\n");
> -		return ret;
> -	}
> -
> -	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
> -	if (ret) {
> -		IFCVF_ERR(pdev,
> -			  "Failed for adding devres for freeing irq vectors\n");
> -		return ret;
> -	}
> +	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
> +	if (ifcvf_mgmt_dev->adapter)
> +		return -EOPNOTSUPP;
>  
> +	pdev = ifcvf_mgmt_dev->pdev;
> +	dev = &pdev->dev;
>  	adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
> -				    dev, &ifc_vdpa_ops, NULL);
> -	if (adapter == NULL) {
> +				    dev, &ifc_vdpa_ops, name);
> +	if (!adapter) {
>  		IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
>  		return -ENOMEM;
>  	}
>  
> -	pci_set_master(pdev);
> -	pci_set_drvdata(pdev, adapter);
> +	ifcvf_mgmt_dev->adapter = adapter;
> +	pci_set_drvdata(pdev, ifcvf_mgmt_dev);
>  
>  	vf = &adapter->vf;
>  	vf->dev_type = get_dev_type(pdev);
> @@ -523,9 +515,10 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>  
>  	vf->hw_features = ifcvf_get_hw_features(vf);
>  
> -	ret = vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
> +	adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
> +	ret = _vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
>  	if (ret) {
> -		IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus");
> +		IFCVF_ERR(pdev, "Failed to register to vDPA bus");
>  		goto err;
>  	}
>  
> @@ -536,11 +529,100 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>  	return ret;
>  }
>  
> +static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
> +{
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
> +
> +	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
> +	_vdpa_unregister_device(dev);
> +	ifcvf_mgmt_dev->adapter = NULL;
> +}
> +
> +static const struct vdpa_mgmtdev_ops ifcvf_vdpa_mgmt_dev_ops = {
> +	.dev_add = ifcvf_vdpa_dev_add,
> +	.dev_del = ifcvf_vdpa_dev_del
> +};
> +
> +static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> +{
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
> +	struct device *dev = &pdev->dev;
> +	u32 dev_type;
> +	int ret;
> +
> +	ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
> +	if (!ifcvf_mgmt_dev) {
> +		IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
> +		return -ENOMEM;
> +	}
> +
> +	dev_type = get_dev_type(pdev);
> +	switch (dev_type) {
> +	case VIRTIO_ID_NET:
> +		ifcvf_mgmt_dev->mdev.id_table = id_table_net;
> +		break;
> +	case VIRTIO_ID_BLOCK:
> +		ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
> +		break;
> +	default:
> +		IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
> +		ret = -EOPNOTSUPP;
> +		goto err;
> +	}
> +
> +	ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
> +	ifcvf_mgmt_dev->mdev.device = dev;
> +	ifcvf_mgmt_dev->pdev = pdev;
> +
> +	ret = pcim_enable_device(pdev);
> +	if (ret) {
> +		IFCVF_ERR(pdev, "Failed to enable device\n");
> +		goto err;
> +	}
> +
> +	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
> +				 IFCVF_DRIVER_NAME);
> +	if (ret) {
> +		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
> +		goto err;
> +	}
> +
> +	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
> +	if (ret) {
> +		IFCVF_ERR(pdev, "No usable DMA configuration\n");
> +		goto err;
> +	}
> +
> +	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
> +	if (ret) {
> +		IFCVF_ERR(pdev,
> +			  "Failed for adding devres for freeing irq vectors\n");
> +		goto err;
> +	}
> +
> +	pci_set_master(pdev);
> +
> +	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
> +	if (ret) {
> +		IFCVF_ERR(pdev,
> +			  "Failed to initialize the management interfaces\n");
> +		goto err;
> +	}
> +
> +	return 0;
> +
> +err:
> +	kfree(ifcvf_mgmt_dev);
> +	return ret;
> +}
> +
>  static void ifcvf_remove(struct pci_dev *pdev)
>  {
> -	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>  
> -	vdpa_unregister_device(&adapter->vdpa);
> +	ifcvf_mgmt_dev = pci_get_drvdata(pdev);
> +	vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
> +	kfree(ifcvf_mgmt_dev);
>  }
>  
>  static struct pci_device_id ifcvf_pci_ids[] = {
> -- 
> 2.27.0

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH V2 2/2] vDPA/ifcvf: implement management netlink framework for ifcvf

[Zhu, Lingshan]

On 7/6/2021 2:30 AM, Michael S. Tsirkin wrote:
> On Mon, Jul 05, 2021 at 10:13:33PM +0800, Zhu Lingshan wrote:
>> This commit implments the management netlink framework for ifcvf,
> implements
>
>> including register and add / remove a device
>>
>> It works with iprouter2:
> I am guessing iproute2?
Thanks Michael, I have sent out a V3 series to fix these typos
>
>> [root@localhost lszhu]# vdpa mgmtdev show -jp
>> {
>>      "mgmtdev": {
>>          "pci/0000:01:00.5": {
>>              "supported_classes": [ "net" ]
>>          },
>>          "pci/0000:01:00.6": {
>>              "supported_classes": [ "net" ]
>>          }
>>      }
>> }
>>
>> [root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.5 name vdpa0
>> [root@localhost lszhu]# vdpa dev add mgmtdev pci/0000:01:00.6 name vdpa1
>>
>> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
>> ---
>>   drivers/vdpa/ifcvf/ifcvf_base.h |   6 ++
>>   drivers/vdpa/ifcvf/ifcvf_main.c | 154 ++++++++++++++++++++++++--------
>>   2 files changed, 124 insertions(+), 36 deletions(-)
>>
>> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
>> index ded1b1b5fb13..e5251fcbb200 100644
>> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
>> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
>> @@ -104,6 +104,12 @@ struct ifcvf_lm_cfg {
>>   	struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUE_PAIRS];
>>   };
>>   
>> +struct ifcvf_vdpa_mgmt_dev {
>> +	struct vdpa_mgmt_dev mdev;
>> +	struct ifcvf_adapter *adapter;
>> +	struct pci_dev *pdev;
>> +};
>> +
>>   int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
>>   int ifcvf_start_hw(struct ifcvf_hw *hw);
>>   void ifcvf_stop_hw(struct ifcvf_hw *hw);
>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
>> index 5f70ab1283a0..c72d9b36e4a0 100644
>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>> @@ -218,7 +218,7 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
>>   	int ret;
>>   
>>   	vf  = vdpa_to_vf(vdpa_dev);
>> -	adapter = dev_get_drvdata(vdpa_dev->dev.parent);
>> +	adapter = vdpa_to_adapter(vdpa_dev);
>>   	status_old = ifcvf_get_status(vf);
>>   
>>   	if (status_old == status)
>> @@ -442,6 +442,16 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
>>   	.set_config_cb  = ifcvf_vdpa_set_config_cb,
>>   };
>>   
>> +static struct virtio_device_id id_table_net[] = {
>> +	{VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
>> +	{0},
>> +};
>> +
>> +static struct virtio_device_id id_table_blk[] = {
>> +	{VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID},
>> +	{0},
>> +};
>> +
>>   static u32 get_dev_type(struct pci_dev *pdev)
>>   {
>>   	u32 dev_type;
>> @@ -462,48 +472,30 @@ static u32 get_dev_type(struct pci_dev *pdev)
>>   	return dev_type;
>>   }
>>   
>> -static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> +static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
>>   {
>> -	struct device *dev = &pdev->dev;
>> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>>   	struct ifcvf_adapter *adapter;
>> +	struct pci_dev *pdev;
>>   	struct ifcvf_hw *vf;
>> +	struct device *dev;
>>   	int ret, i;
>>   
>> -	ret = pcim_enable_device(pdev);
>> -	if (ret) {
>> -		IFCVF_ERR(pdev, "Failed to enable device\n");
>> -		return ret;
>> -	}
>> -
>> -	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
>> -				 IFCVF_DRIVER_NAME);
>> -	if (ret) {
>> -		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
>> -		return ret;
>> -	}
>> -
>> -	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
>> -	if (ret) {
>> -		IFCVF_ERR(pdev, "No usable DMA configuration\n");
>> -		return ret;
>> -	}
>> -
>> -	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
>> -	if (ret) {
>> -		IFCVF_ERR(pdev,
>> -			  "Failed for adding devres for freeing irq vectors\n");
>> -		return ret;
>> -	}
>> +	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
>> +	if (ifcvf_mgmt_dev->adapter)
>> +		return -EOPNOTSUPP;
>>   
>> +	pdev = ifcvf_mgmt_dev->pdev;
>> +	dev = &pdev->dev;
>>   	adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
>> -				    dev, &ifc_vdpa_ops, NULL);
>> -	if (adapter == NULL) {
>> +				    dev, &ifc_vdpa_ops, name);
>> +	if (!adapter) {
>>   		IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
>>   		return -ENOMEM;
>>   	}
>>   
>> -	pci_set_master(pdev);
>> -	pci_set_drvdata(pdev, adapter);
>> +	ifcvf_mgmt_dev->adapter = adapter;
>> +	pci_set_drvdata(pdev, ifcvf_mgmt_dev);
>>   
>>   	vf = &adapter->vf;
>>   	vf->dev_type = get_dev_type(pdev);
>> @@ -523,9 +515,10 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>   
>>   	vf->hw_features = ifcvf_get_hw_features(vf);
>>   
>> -	ret = vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
>> +	adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
>> +	ret = _vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
>>   	if (ret) {
>> -		IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus");
>> +		IFCVF_ERR(pdev, "Failed to register to vDPA bus");
>>   		goto err;
>>   	}
>>   
>> @@ -536,11 +529,100 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>   	return ret;
>>   }
>>   
>> +static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
>> +{
>> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>> +
>> +	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
>> +	_vdpa_unregister_device(dev);
>> +	ifcvf_mgmt_dev->adapter = NULL;
>> +}
>> +
>> +static const struct vdpa_mgmtdev_ops ifcvf_vdpa_mgmt_dev_ops = {
>> +	.dev_add = ifcvf_vdpa_dev_add,
>> +	.dev_del = ifcvf_vdpa_dev_del
>> +};
>> +
>> +static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> +{
>> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>> +	struct device *dev = &pdev->dev;
>> +	u32 dev_type;
>> +	int ret;
>> +
>> +	ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
>> +	if (!ifcvf_mgmt_dev) {
>> +		IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
>> +		return -ENOMEM;
>> +	}
>> +
>> +	dev_type = get_dev_type(pdev);
>> +	switch (dev_type) {
>> +	case VIRTIO_ID_NET:
>> +		ifcvf_mgmt_dev->mdev.id_table = id_table_net;
>> +		break;
>> +	case VIRTIO_ID_BLOCK:
>> +		ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
>> +		break;
>> +	default:
>> +		IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
>> +		ret = -EOPNOTSUPP;
>> +		goto err;
>> +	}
>> +
>> +	ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
>> +	ifcvf_mgmt_dev->mdev.device = dev;
>> +	ifcvf_mgmt_dev->pdev = pdev;
>> +
>> +	ret = pcim_enable_device(pdev);
>> +	if (ret) {
>> +		IFCVF_ERR(pdev, "Failed to enable device\n");
>> +		goto err;
>> +	}
>> +
>> +	ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
>> +				 IFCVF_DRIVER_NAME);
>> +	if (ret) {
>> +		IFCVF_ERR(pdev, "Failed to request MMIO region\n");
>> +		goto err;
>> +	}
>> +
>> +	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
>> +	if (ret) {
>> +		IFCVF_ERR(pdev, "No usable DMA configuration\n");
>> +		goto err;
>> +	}
>> +
>> +	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
>> +	if (ret) {
>> +		IFCVF_ERR(pdev,
>> +			  "Failed for adding devres for freeing irq vectors\n");
>> +		goto err;
>> +	}
>> +
>> +	pci_set_master(pdev);
>> +
>> +	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
>> +	if (ret) {
>> +		IFCVF_ERR(pdev,
>> +			  "Failed to initialize the management interfaces\n");
>> +		goto err;
>> +	}
>> +
>> +	return 0;
>> +
>> +err:
>> +	kfree(ifcvf_mgmt_dev);
>> +	return ret;
>> +}
>> +
>>   static void ifcvf_remove(struct pci_dev *pdev)
>>   {
>> -	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
>> +	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
>>   
>> -	vdpa_unregister_device(&adapter->vdpa);
>> +	ifcvf_mgmt_dev = pci_get_drvdata(pdev);
>> +	vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
>> +	kfree(ifcvf_mgmt_dev);
>>   }
>>   
>>   static struct pci_device_id ifcvf_pci_ids[] = {
>> -- 
>> 2.27.0