Re: [PATCH v1] vdpa/vp_vdpa : add vdpa tool support in vp_vdpa

[Jason Wang <jasowang@redhat.com>]

在 2022/4/19 09:40, Cindy Lu 写道:
> this patch is to add the support for vdpa tool in vp_vdpa
> here is the example steps
> modprobe vp_vdpa
> modprobe vhost_vdpa
>
> echo 0000:00:06.0>/sys/bus/pci/drivers/virtio-pci/unbind
> echo 1af4 1041 > /sys/bus/pci/drivers/vp-vdpa/new_id
>
> vdpa dev add name vdpa1 mgmtdev pci/0000:00:06.0
>
> Signed-off-by: Cindy Lu <lulu@redhat.com>
> ---
>   drivers/vdpa/virtio_pci/vp_vdpa.c | 86 ++++++++++++++++++++++++++++---
>   1 file changed, 78 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
> index cce101e6a940..d8a1d2f8e9bb 100644
> --- a/drivers/vdpa/virtio_pci/vp_vdpa.c
> +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
> @@ -17,6 +17,8 @@
>   #include <linux/virtio_ring.h>
>   #include <linux/virtio_pci.h>
>   #include <linux/virtio_pci_modern.h>
> +#include <uapi/linux/virtio_net.h>
> +#include <uapi/linux/vdpa.h>
>   
>   #define VP_VDPA_QUEUE_MAX 256
>   #define VP_VDPA_DRIVER_NAME "vp_vdpa"
> @@ -41,6 +43,18 @@ struct vp_vdpa {
>   	int vectors;
>   };
>   
> +struct vp_vdpa_mgmtdev {
> +	struct vdpa_mgmt_dev mgtdev;
> +	struct vp_vdpa *vp_vdpa;
> +	struct pci_dev *pdev;
> +};
> +
> +#define VP_VDPA_NET_FEATURES                                                   \
> +	((1ULL << VIRTIO_F_ANY_LAYOUT) | (1ULL << VIRTIO_F_VERSION_1) |        \
> +	 (1ULL << VIRTIO_F_ACCESS_PLATFORM))
> +
> +#define VP_VDPA_NET_VQ_NUM 2


Let's not go backwards, e.g we've already support any kind of virtio 
device with any kind of features. see the comment in vp_vdpa_probe().


> +
>   static struct vp_vdpa *vdpa_to_vp(struct vdpa_device *vdpa)
>   {
>   	return container_of(vdpa, struct vp_vdpa, vdpa);
> @@ -454,9 +468,14 @@ static void vp_vdpa_free_irq_vectors(void *data)
>   	pci_free_irq_vectors(data);
>   }
>   
> -static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> +static int vp_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
> +			   const struct vdpa_dev_set_config *add_config)
>   {
> +	struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev =
> +		container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev);
> +
>   	struct virtio_pci_modern_device *mdev;
> +	struct pci_dev *pdev = vp_vdpa_mgtdev->pdev;
>   	struct device *dev = &pdev->dev;
>   	struct vp_vdpa *vp_vdpa;
>   	int ret, i;
> @@ -465,8 +484,9 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>   	if (ret)
>   		return ret;
>   
> -	vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa,
> -				    dev, &vp_vdpa_ops, NULL, false);
> +	vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa, dev, &vp_vdpa_ops,
> +				    name, false);


Nit: let's keep the line of vdpa_alloc_device() unchanged to reduce the 
changeset.


> +
>   	if (IS_ERR(vp_vdpa)) {
>   		dev_err(dev, "vp_vdpa: Failed to allocate vDPA structure\n");
>   		return PTR_ERR(vp_vdpa);
> @@ -480,9 +500,10 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>   		dev_err(&pdev->dev, "Failed to probe modern PCI device\n");
>   		goto err;
>   	}
> +	vp_vdpa_mgtdev->vp_vdpa = vp_vdpa;
>   
>   	pci_set_master(pdev);
> -	pci_set_drvdata(pdev, vp_vdpa);
> +	pci_set_drvdata(pdev, vp_vdpa_mgtdev);
>   
>   	vp_vdpa->vdpa.dma_dev = &pdev->dev;
>   	vp_vdpa->queues = vp_modern_get_num_queues(mdev);
> @@ -516,7 +537,8 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>   	}
>   	vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR;
>   
> -	ret = vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues);
> +	vp_vdpa->vdpa.mdev = &vp_vdpa_mgtdev->mgtdev;
> +	ret = _vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues);
>   	if (ret) {
>   		dev_err(&pdev->dev, "Failed to register to vdpa bus\n");
>   		goto err;
> @@ -529,12 +551,60 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>   	return ret;
>   }
>   
> -static void vp_vdpa_remove(struct pci_dev *pdev)
> +static void vp_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev,
> +			    struct vdpa_device *dev)
>   {
> -	struct vp_vdpa *vp_vdpa = pci_get_drvdata(pdev);
> +	struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev =
> +		container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev);
> +
> +	struct vp_vdpa *vp_vdpa = vp_vdpa_mgtdev->vp_vdpa;
>   
>   	vp_modern_remove(&vp_vdpa->mdev);
> -	vdpa_unregister_device(&vp_vdpa->vdpa);
> +	_vdpa_unregister_device(&vp_vdpa->vdpa);
> +	vp_vdpa_mgtdev->vp_vdpa = NULL;
> +}
> +
> +static const struct vdpa_mgmtdev_ops vp_vdpa_mdev_ops = {
> +	.dev_add = vp_vdpa_dev_add,
> +	.dev_del = vp_vdpa_dev_del,
> +};
> +
> +static struct virtio_device_id vp_vdpa_id_table[] = {
> +	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
> +	{ 0 },
> +};


Let's not limit the vp_vdpa to net, it can support all other virtio devices.


> +
> +static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> +{
> +	struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev;
> +	struct vdpa_mgmt_dev *mgtdev;
> +	struct device *dev = &pdev->dev;
> +	int err;
> +
> +	vp_vdpa_mgtdev = kzalloc(sizeof(*vp_vdpa_mgtdev), GFP_KERNEL);
> +	if (!vp_vdpa_mgtdev)
> +		return -ENOMEM;
> +	mgtdev = &vp_vdpa_mgtdev->mgtdev;
> +
> +	mgtdev->ops = &vp_vdpa_mdev_ops;
> +	mgtdev->device = dev;
> +	mgtdev->id_table = vp_vdpa_id_table;
> +	vp_vdpa_mgtdev->pdev = pdev;
> +
> +	mgtdev->max_supported_vqs = VP_VDPA_NET_VQ_NUM;


Let's read this from the device instead of using the hard-coded value.


> +	mgtdev->supported_features = VP_VDPA_NET_FEATURES;


Let's read this from the device.


> +
> +	err = vdpa_mgmtdev_register(mgtdev);


Do we need to free the vp_vdpa_mgtdev here?

Thanks


> +
> +	return err;
> +}
> +
> +static void vp_vdpa_remove(struct pci_dev *pdev)
> +{
> +	struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = pci_get_drvdata(pdev);
> +
> +	vdpa_mgmtdev_unregister(&vp_vdpa_mgtdev->mgtdev);
> +	kfree(vp_vdpa_mgtdev);
>   }
>   
>   static struct pci_driver vp_vdpa_driver = {