[PATCH 0/8] add multi queue support to vDPA ifc driver

[PATCH 0/8] add multi queue support to vDPA ifc driver

[Andy Pei]

Add multi queue support to vDPA ifc driver.
Multi queue support for virtio-net device and virtio-blk device.

Andy Pei (7):
  vdpa/ifc: add multi queue suppoort
  vdpa/ifc: set max queues according to HW spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vdpa/ifc: set vring state callback update data path
  vhost: configure device when any queue is ready for BLK device
  vhost: vDPA BLK devices configure device when all queue callfds are
    set

Huang Wei (1):
  vdpa/ifc: add new device ID

 drivers/vdpa/ifc/base/ifcvf.c | 24 ++++++++++++++++-
 drivers/vdpa/ifc/base/ifcvf.h |  8 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 59 ++++++++++++++++++++++++++--------------
 lib/vhost/vhost_user.c        | 63 ++++++++++++++++++++++++++++++++++---------
 4 files changed, 119 insertions(+), 35 deletions(-)

-- 
1.8.3.1

[PATCH 1/8] vdpa/ifc: add new device ID

[Andy Pei]

From: Huang Wei <wei_huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).

Signed-off-by: Huang Wei <wei_huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h | 4 +++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 9 ++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..7ede738 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,11 +12,13 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
+#define IFCVF_NET_DEVICE_ID                 0x0001
 #define IFCVF_BLK_DEVICE_ID                 0x0002
 
 #define IFCVF_MAX_QUEUES		1
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index ac42de9..61d0250 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
-- 
1.8.3.1

[PATCH v2 0/8] vdpa/ifc: add multi queue support

[Andy Pei]

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (6):
  vdpa/ifc: add multi queue support
  vdpa/ifc: set max queues according to HW spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vhost: configure device when any queue is ready for BLK device
  vhost: vDPA BLK devices configure device when all queue callfds are
    set

Huang Wei (2):
  vdpa/ifc: add new device ID
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c       | 145 +++++++++++++++++++++++++++++++++++-
 drivers/vdpa/ifc/base/ifcvf.h       |  14 +++-
 drivers/vdpa/ifc/base/ifcvf_osdep.h |   1 +
 drivers/vdpa/ifc/ifcvf_vdpa.c       | 140 +++++++++++++++++++++++++++++-----
 lib/vhost/vhost_user.c              |  63 +++++++++++++---
 5 files changed, 329 insertions(+), 34 deletions(-)

-- 
1.8.3.1

[PATCH v2 1/8] vdpa/ifc: add new device ID

[Andy Pei]

From: Huang Wei <wei_huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).

Signed-off-by: Huang Wei <wei_huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h | 4 +++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 9 ++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..7ede738 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,11 +12,13 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
+#define IFCVF_NET_DEVICE_ID                 0x0001
 #define IFCVF_BLK_DEVICE_ID                 0x0002
 
 #define IFCVF_MAX_QUEUES		1
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index ac42de9..61d0250 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
-- 
1.8.3.1

RE: [PATCH v2 1/8] vdpa/ifc: add new device ID

[Xia, Chenbo]

Hi Andy,

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, September 8, 2022 1:54 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: [PATCH v2 1/8] vdpa/ifc: add new device ID

Title could be: add new device ID for legacy network device

> 
> From: Huang Wei <wei_huang@intel.com>
> 
> Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
> 
> Signed-off-by: Huang Wei <wei_huang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.h | 4 +++-
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 9 ++++++++-
>  2 files changed, 11 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index 9d95aac..7ede738 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -12,11 +12,13 @@
>  #define IFCVF_BLK	1
> 
>  #define IFCVF_VENDOR_ID                     0x1AF4
> -#define IFCVF_NET_DEVICE_ID                 0x1041
> +#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
>  #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
> +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
>  #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
>  #define IFCVF_SUBSYS_VENDOR_ID              0x8086
>  #define IFCVF_SUBSYS_DEVICE_ID              0x001A
> +#define IFCVF_NET_DEVICE_ID                 0x0001

For subsystem device ID, I suggest to add _SUBSYS_, please check all
Subsystem device ID and make all the names well-defined.

Thanks,
Chenbo

>  #define IFCVF_BLK_DEVICE_ID                 0x0002
> 
>  #define IFCVF_MAX_QUEUES		1
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index ac42de9..61d0250 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {
>  static const struct rte_pci_id pci_id_ifcvf_map[] = {
>  	{ .class_id = RTE_CLASS_ANY_ID,
>  	  .vendor_id = IFCVF_VENDOR_ID,
> -	  .device_id = IFCVF_NET_DEVICE_ID,
> +	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
>  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
>  	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
>  	},
> 
>  	{ .class_id = RTE_CLASS_ANY_ID,
>  	  .vendor_id = IFCVF_VENDOR_ID,
> +	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
> +	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> +	  .subsystem_device_id = IFCVF_NET_DEVICE_ID,
> +	},
> +
> +	{ .class_id = RTE_CLASS_ANY_ID,
> +	  .vendor_id = IFCVF_VENDOR_ID,
>  	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
>  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
>  	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> --
> 1.8.3.1

RE: [PATCH v2 1/8] vdpa/ifc: add new device ID

[Pei, Andy]

Hi Chenbo, 

Thanks for your reply.
I will send V3 patch to address your 2 comments.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Friday, September 9, 2022 8:29 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> <wei_huang@intel.com>
> Subject: RE: [PATCH v2 1/8] vdpa/ifc: add new device ID
> 
> Hi Andy,
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, September 8, 2022 1:54 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: [PATCH v2 1/8] vdpa/ifc: add new device ID
> 
> Title could be: add new device ID for legacy network device
> 
> >
> > From: Huang Wei <wei_huang@intel.com>
> >
> > Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
> >
> > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.h | 4 +++-  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 9 ++++++++-
> >  2 files changed, 11 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index 9d95aac..7ede738 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -12,11 +12,13 @@
> >  #define IFCVF_BLK	1
> >
> >  #define IFCVF_VENDOR_ID                     0x1AF4
> > -#define IFCVF_NET_DEVICE_ID                 0x1041
> > +#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
> >  #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
> > +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
> >  #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
> >  #define IFCVF_SUBSYS_VENDOR_ID              0x8086
> >  #define IFCVF_SUBSYS_DEVICE_ID              0x001A
> > +#define IFCVF_NET_DEVICE_ID                 0x0001
> 
> For subsystem device ID, I suggest to add _SUBSYS_, please check all Subsystem
> device ID and make all the names well-defined.
> 
> Thanks,
> Chenbo
> 
> >  #define IFCVF_BLK_DEVICE_ID                 0x0002
> >
> >  #define IFCVF_MAX_QUEUES		1
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index ac42de9..61d0250 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1684,13 +1684,20 @@ struct rte_vdpa_dev_info dev_info[] = {
> > static const struct rte_pci_id pci_id_ifcvf_map[] = {
> >  	{ .class_id = RTE_CLASS_ANY_ID,
> >  	  .vendor_id = IFCVF_VENDOR_ID,
> > -	  .device_id = IFCVF_NET_DEVICE_ID,
> > +	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
> >  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> >  	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
> >  	},
> >
> >  	{ .class_id = RTE_CLASS_ANY_ID,
> >  	  .vendor_id = IFCVF_VENDOR_ID,
> > +	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
> > +	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> > +	  .subsystem_device_id = IFCVF_NET_DEVICE_ID,
> > +	},
> > +
> > +	{ .class_id = RTE_CLASS_ANY_ID,
> > +	  .vendor_id = IFCVF_VENDOR_ID,
> >  	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
> >  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> >  	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> > --
> > 1.8.3.1

[PATCH v2 2/8] vdpa/ifc: add multi queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
ExposeIFCVF_MQ_OFFSET register to enable multi queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 5 +++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 8 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..34c8226 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,11 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 7ede738..ad505f1 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 61d0250..2d165c0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

RE: [PATCH v2 2/8] vdpa/ifc: add multi queue support

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, September 8, 2022 1:54 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: [PATCH v2 2/8] vdpa/ifc: add multi queue support

multi-queue

> 
> Enable VHOST_USER_PROTOCOL_F_MQ feature.
> ExposeIFCVF_MQ_OFFSET register to enable multi queue.

Please rephase it in a better way, at least add a space before
IFCVF_MP_OFFSET...

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei_huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 5 +++++
>  drivers/vdpa/ifc/base/ifcvf.h | 2 ++
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
>  3 files changed, 8 insertions(+)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index f1e1474..34c8226 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -90,6 +90,11 @@
>  	if (!hw->lm_cfg)
>  		WARNINGOUT("HW support live migration not support!\n");
> 
> +	if (hw->mem_resource[4].addr)
> +		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
> +	else
> +		hw->mq_cfg = NULL;

Could you help me understand the logic here? There are two cases that BAR 4
mmap-able and not mmap-able?

Thanks,
Chenbo

> +
>  	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
>  			hw->isr == NULL || hw->dev_cfg == NULL) {
>  		DEBUGOUT("capability incomplete\n");
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index 7ede738..ad505f1 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -50,6 +50,7 @@
> 
>  #define IFCVF_LM_CFG_SIZE		0x40
>  #define IFCVF_LM_RING_STATE_OFFSET	0x20
> +#define IFCVF_MQ_OFFSET			0x28
> 
>  #define IFCVF_LM_LOGGING_CTRL		0x0
> 
> @@ -149,6 +150,7 @@ struct ifcvf_hw {
>  	u16    *notify_base;
>  	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
>  	u8     *lm_cfg;
> +	u8     *mq_cfg;
>  	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
>  	u8 nr_vring;
>  	int device_type;
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 61d0250..2d165c0 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
>  		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
> +		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
> 
>  #define VDPA_BLK_PROTOCOL_FEATURES \
> --
> 1.8.3.1

RE: [PATCH v2 2/8] vdpa/ifc: add multi queue support

[Pei, Andy]

Hi Chenbo,

Thanks for your reply.
My reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Friday, September 9, 2022 8:29 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> <wei_huang@intel.com>
> Subject: RE: [PATCH v2 2/8] vdpa/ifc: add multi queue support
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, September 8, 2022 1:54 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: [PATCH v2 2/8] vdpa/ifc: add multi queue support
> 
> multi-queue
> 
Sure.
> >
> > Enable VHOST_USER_PROTOCOL_F_MQ feature.
> > ExposeIFCVF_MQ_OFFSET register to enable multi queue.
> 
> Please rephase it in a better way, at least add a space before
> IFCVF_MP_OFFSET...
> 
I will make it "Expose IFCVF_MQ_OFFSET register to enable multi queue."
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 5 +++++
> > drivers/vdpa/ifc/base/ifcvf.h | 2 ++  drivers/vdpa/ifc/ifcvf_vdpa.c |
> > 1 +
> >  3 files changed, 8 insertions(+)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index f1e1474..34c8226 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -90,6 +90,11 @@
> >  	if (!hw->lm_cfg)
> >  		WARNINGOUT("HW support live migration not support!\n");
> >
> > +	if (hw->mem_resource[4].addr)
> > +		hw->mq_cfg = hw->mem_resource[4].addr +
> IFCVF_MQ_OFFSET;
> > +	else
> > +		hw->mq_cfg = NULL;
> 
> Could you help me understand the logic here? There are two cases that BAR 4
> mmap-able and not mmap-able?
> 
For some hardware implementation, for example the BAR 4 of PF is NULL, while BAR 4 of VF is not.
This code makes sure hw->mq_cfg is a valid address.

I am planning a optimization to check parameter consistent at init phase.
For example, if hardware does not support HW-assist live migration, we cannot use parameter like "sw-live-migration=0".

> Thanks,
> Chenbo
> 
> > +
> >  	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
> >  			hw->isr == NULL || hw->dev_cfg == NULL) {
> >  		DEBUGOUT("capability incomplete\n"); diff --git
> > a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h index
> > 7ede738..ad505f1 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -50,6 +50,7 @@
> >
> >  #define IFCVF_LM_CFG_SIZE		0x40
> >  #define IFCVF_LM_RING_STATE_OFFSET	0x20
> > +#define IFCVF_MQ_OFFSET			0x28
> >
> >  #define IFCVF_LM_LOGGING_CTRL		0x0
> >
> > @@ -149,6 +150,7 @@ struct ifcvf_hw {
> >  	u16    *notify_base;
> >  	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
> >  	u8     *lm_cfg;
> > +	u8     *mq_cfg;
> >  	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
> >  	u8 nr_vring;
> >  	int device_type;
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 61d0250..2d165c0 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
> >  		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
> > +		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
> >
> >  #define VDPA_BLK_PROTOCOL_FEATURES \
> > --
> > 1.8.3.1

[PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec

[Andy Pei]

Set max_queues according to virtio HW spec.
For virtio BLK device, set max_queues to the value of "num_queues".
"num_queues" is element of struct virtio_blk_config.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 21 ++++++++++++++++++++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ad505f1..c17bf2a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_NET_DEVICE_ID                 0x0001
 #define IFCVF_BLK_DEVICE_ID                 0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 2d165c0..2b42850 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,18 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MAX() and RTE_MIN() cannot be used since braced-group within
+ * expression allowed only inside a function, but MAX() is used as
+ * a number of elements in array.
+ */
+#ifndef MAX
+#define MAX(v1, v2)	((v1) > (v2) ? (v1) : (v2))
+#endif
+#ifndef MIN
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+#endif
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1524,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1572,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1582,10 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1625,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

RE: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, September 8, 2022 1:54 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec

vdpa/ifc: set max queues based on virtio spec

> 
> Set max_queues according to virtio HW spec.
> For virtio BLK device, set max_queues to the value of "num_queues".
> "num_queues" is element of struct virtio_blk_config.

Both virtio-net/blk should be described.

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei_huang@intel.com>

Email is wrong, please fix all in next version

> ---
>  drivers/vdpa/ifc/base/ifcvf.h |  2 +-
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 21 ++++++++++++++++++++-
>  2 files changed, 21 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index ad505f1..c17bf2a 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -21,7 +21,7 @@
>  #define IFCVF_NET_DEVICE_ID                 0x0001
>  #define IFCVF_BLK_DEVICE_ID                 0x0002
> 
> -#define IFCVF_MAX_QUEUES		1
> +#define IFCVF_MAX_QUEUES		32
> 
>  #ifndef VIRTIO_F_IOMMU_PLATFORM
>  #define VIRTIO_F_IOMMU_PLATFORM		33
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 2d165c0..2b42850 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -26,6 +26,18 @@
> 
>  #include "base/ifcvf.h"
> 
> +/*
> + * RTE_MAX() and RTE_MIN() cannot be used since braced-group within
> + * expression allowed only inside a function, but MAX() is used as
> + * a number of elements in array.
> + */
> +#ifndef MAX
> +#define MAX(v1, v2)	((v1) > (v2) ? (v1) : (v2))
> +#endif
> +#ifndef MIN
> +#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
> +#endif

Above ifndef is not needed?

Seems MAX is not used, so remove it

> +
>  RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
>  #define DRV_LOG(level, fmt, args...) \
>  	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
> @@ -1512,6 +1524,7 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	uint64_t capacity = 0;
>  	uint8_t *byte;
>  	uint32_t i;
> +	uint16_t queue_pairs;
> 
>  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
>  		return 0;
> @@ -1559,7 +1572,6 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	}
> 
>  	internal->configured = 0;
> -	internal->max_queues = IFCVF_MAX_QUEUES;
>  	features = ifcvf_get_features(&internal->hw);
> 
>  	device_id = ifcvf_pci_get_device_type(pci_dev);
> @@ -1570,6 +1582,10 @@ struct rte_vdpa_dev_info dev_info[] = {
> 
>  	if (device_id == VIRTIO_ID_NET) {
>  		internal->hw.device_type = IFCVF_NET;
> +		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;

Please note this logic assumes CTRL_VQ is always there, if for the read hardware,
that is the case, then it's fine. You can decide yourself to check CTRL_VQ feature
is there or not.

Thanks,
Chenbo

> +		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
> +			queue_pairs);
> +		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
>  		internal->features = features &
>  					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
>  		internal->features |= dev_info[IFCVF_NET].features;
> @@ -1609,6 +1625,9 @@ struct rte_vdpa_dev_info dev_info[] = {
>  			internal->hw.blk_cfg->geometry.sectors);
>  		DRV_LOG(DEBUG, "num_queues: 0x%08x",
>  			internal->hw.blk_cfg->num_queues);
> +
> +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> +			internal->hw.blk_cfg->num_queues);
>  	}
> 
>  	list->internal = internal;
> --
> 1.8.3.1

RE: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec

[Pei, Andy]

Hi Chenbo,

Thanks for your reply.
My reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Friday, September 9, 2022 1:56 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, September 8, 2022 1:54 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: [PATCH v2 3/8] vdpa/ifc: set max queues according to HW spec
> 
> vdpa/ifc: set max queues based on virtio spec
> 
OK

> >
> > Set max_queues according to virtio HW spec.
> > For virtio BLK device, set max_queues to the value of "num_queues".
> > "num_queues" is element of struct virtio_blk_config.
> 
> Both virtio-net/blk should be described.
> 
OK
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei_huang@intel.com>
> 
> Email is wrong, please fix all in next version
> 
OK

> > ---
> >  drivers/vdpa/ifc/base/ifcvf.h |  2 +-  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 21 ++++++++++++++++++++-
> >  2 files changed, 21 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index ad505f1..c17bf2a 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -21,7 +21,7 @@
> >  #define IFCVF_NET_DEVICE_ID                 0x0001
> >  #define IFCVF_BLK_DEVICE_ID                 0x0002
> >
> > -#define IFCVF_MAX_QUEUES		1
> > +#define IFCVF_MAX_QUEUES		32
> >
> >  #ifndef VIRTIO_F_IOMMU_PLATFORM
> >  #define VIRTIO_F_IOMMU_PLATFORM		33
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 2d165c0..2b42850 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -26,6 +26,18 @@
> >
> >  #include "base/ifcvf.h"
> >
> > +/*
> > + * RTE_MAX() and RTE_MIN() cannot be used since braced-group within
> > + * expression allowed only inside a function, but MAX() is used as
> > + * a number of elements in array.
> > + */
> > +#ifndef MAX
> > +#define MAX(v1, v2)	((v1) > (v2) ? (v1) : (v2))
> > +#endif
> > +#ifndef MIN
> > +#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
> > +#endif
> 
> Above ifndef is not needed?
> 
> Seems MAX is not used, so remove it
> 
OK
> > +
> >  RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
> > #define DRV_LOG(level, fmt, args...) \
> >  	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6 +1524,7
> > @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	uint64_t capacity = 0;
> >  	uint8_t *byte;
> >  	uint32_t i;
> > +	uint16_t queue_pairs;
> >
> >  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> >  		return 0;
> > @@ -1559,7 +1572,6 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	}
> >
> >  	internal->configured = 0;
> > -	internal->max_queues = IFCVF_MAX_QUEUES;
> >  	features = ifcvf_get_features(&internal->hw);
> >
> >  	device_id = ifcvf_pci_get_device_type(pci_dev);
> > @@ -1570,6 +1582,10 @@ struct rte_vdpa_dev_info dev_info[] = {
> >
> >  	if (device_id == VIRTIO_ID_NET) {
> >  		internal->hw.device_type = IFCVF_NET;
> > +		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
> 
> Please note this logic assumes CTRL_VQ is always there, if for the read hardware,
> that is the case, then it's fine. You can decide yourself to check CTRL_VQ feature
> is there or not.
> 
I will keep the code as it is now, and add some comments to explain ifc driver assumes CTRL_VQ is always there.
> Thanks,
> Chenbo
> 
> > +		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
> > +			queue_pairs);
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> queue_pairs);
> >  		internal->features = features &
> >  					~(1ULL <<
> VIRTIO_F_IOMMU_PLATFORM);
> >  		internal->features |= dev_info[IFCVF_NET].features; @@ -
> 1609,6
> > +1625,9 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  			internal->hw.blk_cfg->geometry.sectors);
> >  		DRV_LOG(DEBUG, "num_queues: 0x%08x",
> >  			internal->hw.blk_cfg->num_queues);
> > +
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> > +			internal->hw.blk_cfg->num_queues);
> >  	}
> >
> >  	list->internal = internal;
> > --
> > 1.8.3.1

[PATCH v2 4/8] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 34c8226..0444d74 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -198,6 +198,38 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_multiqueue(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	int qid;
+	int nr_queue_pair = 0;
+
+	if (hw->device_type == IFCVF_NET) {
+		for (qid = 0; qid < hw->nr_vring; qid++) {
+			if (!hw->vring[qid].enable)
+				continue;
+			nr_queue_pair++;
+		}
+
+		if (nr_queue_pair == 0) {
+			WARNINGOUT("no enabled vring\n");
+			return;
+		}
+
+		nr_queue_pair = (nr_queue_pair + 1) / 2;
+	} else if (hw->device_type == IFCVF_BLK) {
+		nr_queue_pair = hw->nr_vring;
+	}
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		*(u32 *)mq_cfg = nr_queue_pair;
+		RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+		nr_queue_pair);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -215,6 +247,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_multiqueue(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

RE: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ register

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, September 8, 2022 1:54 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ register
> 
> Write queue count to IFCVF_MQ_OFFSET register
> to enable multi queue feature.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei_huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 33 +++++++++++++++++++++++++++++++++
>  1 file changed, 33 insertions(+)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 34c8226..0444d74 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -198,6 +198,38 @@
>  	IFCVF_WRITE_REG32(val >> 32, hi);
>  }
> 
> +STATIC void
> +ifcvf_enable_multiqueue(struct ifcvf_hw *hw)
> +{
> +	u8 *mq_cfg;
> +	int qid;
> +	int nr_queue_pair = 0;
> +
> +	if (hw->device_type == IFCVF_NET) {
> +		for (qid = 0; qid < hw->nr_vring; qid++) {
> +			if (!hw->vring[qid].enable)
> +				continue;
> +			nr_queue_pair++;
> +		}
> +
> +		if (nr_queue_pair == 0) {
> +			WARNINGOUT("no enabled vring\n");
> +			return;
> +		}
> +
> +		nr_queue_pair = (nr_queue_pair + 1) / 2;
> +	} else if (hw->device_type == IFCVF_BLK) {
> +		nr_queue_pair = hw->nr_vring;
> +	}

Why above logic will be different for NET/BLK, could you
help me understand it?

Thanks,
Chenbo

> +
> +	mq_cfg = hw->mq_cfg;
> +	if (mq_cfg) {
> +		*(u32 *)mq_cfg = nr_queue_pair;
> +		RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
> +		nr_queue_pair);
> +	}
> +}
> +
>  STATIC int
>  ifcvf_hw_enable(struct ifcvf_hw *hw)
>  {
> @@ -215,6 +247,7 @@
>  		return -1;
>  	}
> 
> +	ifcvf_enable_multiqueue(hw);
>  	for (i = 0; i < hw->nr_vring; i++) {
>  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
>  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> --
> 1.8.3.1

RE: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ register

[Pei, Andy]

Hi Chenbo, 

For BLK devices, ifc driver does not have VHOST_USER_SET_VRING_ENABLE msg.
So it does not need to dynamic configuration.

But now I think it is fine to use the same logic.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Friday, September 9, 2022 3:35 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> <wei_huang@intel.com>
> Subject: RE: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ register
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, September 8, 2022 1:54 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: [PATCH v2 4/8] vdpa/ifc: write queue count to MQ register
> >
> > Write queue count to IFCVF_MQ_OFFSET register to enable multi queue
> > feature.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 33 +++++++++++++++++++++++++++++++++
> >  1 file changed, 33 insertions(+)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 34c8226..0444d74 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -198,6 +198,38 @@
> >  	IFCVF_WRITE_REG32(val >> 32, hi);
> >  }
> >
> > +STATIC void
> > +ifcvf_enable_multiqueue(struct ifcvf_hw *hw) {
> > +	u8 *mq_cfg;
> > +	int qid;
> > +	int nr_queue_pair = 0;
> > +
> > +	if (hw->device_type == IFCVF_NET) {
> > +		for (qid = 0; qid < hw->nr_vring; qid++) {
> > +			if (!hw->vring[qid].enable)
> > +				continue;
> > +			nr_queue_pair++;
> > +		}
> > +
> > +		if (nr_queue_pair == 0) {
> > +			WARNINGOUT("no enabled vring\n");
> > +			return;
> > +		}
> > +
> > +		nr_queue_pair = (nr_queue_pair + 1) / 2;
> > +	} else if (hw->device_type == IFCVF_BLK) {
> > +		nr_queue_pair = hw->nr_vring;
> > +	}
> 
> Why above logic will be different for NET/BLK, could you help me understand it?
> 
> Thanks,
> Chenbo
> 
> > +
> > +	mq_cfg = hw->mq_cfg;
> > +	if (mq_cfg) {
> > +		*(u32 *)mq_cfg = nr_queue_pair;
> > +		RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
> > +		nr_queue_pair);
> > +	}
> > +}
> > +
> >  STATIC int
> >  ifcvf_hw_enable(struct ifcvf_hw *hw)
> >  {
> > @@ -215,6 +247,7 @@
> >  		return -1;
> >  	}
> >
> > +	ifcvf_enable_multiqueue(hw);
> >  	for (i = 0; i < hw->nr_vring; i++) {
> >  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
> >  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> > --
> > 1.8.3.1

[PATCH v2 5/8] vdpa/ifc: only configure enabled queue

[Andy Pei]

when configure the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  6 +++++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 0444d74..4875ea1 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -249,6 +249,9 @@
 
 	ifcvf_enable_multiqueue(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
@@ -283,7 +286,8 @@
 		notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
 		hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
 				notify_off * hw->notify_off_multiplier);
-		IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+		if (hw->vring[i].enable)
+			IFCVF_WRITE_REG16(1, &cfg->queue_enable);
 	}
 
 	return 0;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 2b42850..48f1a89 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -505,6 +507,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1064,6 +1068,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1077,11 +1083,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

RE: [PATCH v2 5/8] vdpa/ifc: only configure enabled queue

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, September 8, 2022 1:54 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: [PATCH v2 5/8] vdpa/ifc: only configure enabled queue
> 
> when configure the hardware queue, we only configure queues which
> have been enabled by vhost.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei_huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c |  6 +++++-
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
>  2 files changed, 19 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 0444d74..4875ea1 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -249,6 +249,9 @@
> 
>  	ifcvf_enable_multiqueue(hw);
>  	for (i = 0; i < hw->nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
> +
>  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
>  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
>  				&cfg->queue_desc_hi);
> @@ -283,7 +286,8 @@
>  		notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
>  		hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
>  				notify_off * hw->notify_off_multiplier);
> -		IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> +		if (hw->vring[i].enable)

Seems useless check as it already checked when the for loop starts

Thanks,
Chenbo

> +			IFCVF_WRITE_REG16(1, &cfg->queue_enable);
>  	}
> 
>  	return 0;
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 2b42850..48f1a89 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {
>  	rte_vhost_get_negotiated_features(vid, &hw->req_features);
> 
>  	for (i = 0; i < nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
>  		rte_vhost_get_vhost_vring(vid, i, &vq);
>  		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
>  		if (gpa == 0) {
> @@ -505,6 +507,8 @@ struct rte_vdpa_dev_info {
> 
>  	vring.kickfd = -1;
>  	for (qid = 0; qid < q_num; qid++) {
> +		if (!hw->vring[qid].enable)
> +			continue;
>  		ev.events = EPOLLIN | EPOLLPRI;
>  		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
>  		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
> @@ -1064,6 +1068,8 @@ struct rte_vdpa_dev_info {
>  	struct rte_vdpa_device *vdev;
>  	struct internal_list *list;
>  	struct ifcvf_internal *internal;
> +	struct ifcvf_hw *hw;
> +	uint16_t i;
> 
>  	vdev = rte_vhost_get_vdpa_device(vid);
>  	list = find_internal_resource_by_vdev(vdev);
> @@ -1077,11 +1083,17 @@ struct rte_vdpa_dev_info {
>  	rte_atomic32_set(&internal->dev_attached, 1);
>  	update_datapath(internal);
> 
> -	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) !=
> 0)
> -		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
> +	hw = &internal->hw;
> +	for (i = 0; i < hw->nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
> +		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
> +			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
>  				vdev->device->name);
> +	}
> 
>  	internal->configured = 1;
> +	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
>  	return 0;
>  }
> 
> --
> 1.8.3.1

RE: [PATCH v2 5/8] vdpa/ifc: only configure enabled queue

[Huang, Wei]

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, September 14, 2022 09:59
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> <wei_huang@intel.com>
> Subject: RE: [PATCH v2 5/8] vdpa/ifc: only configure enabled queue
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, September 8, 2022 1:54 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: [PATCH v2 5/8] vdpa/ifc: only configure enabled queue
> >
> > when configure the hardware queue, we only configure queues which have
> > been enabled by vhost.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c |  6 +++++-
> > drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
> >  2 files changed, 19 insertions(+), 3 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 0444d74..4875ea1 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -249,6 +249,9 @@
> >
> >  	ifcvf_enable_multiqueue(hw);
> >  	for (i = 0; i < hw->nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> > +
> >  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
> >  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> >  				&cfg->queue_desc_hi);
> > @@ -283,7 +286,8 @@
> >  		notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
> >  		hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
> >  				notify_off * hw->notify_off_multiplier);
> > -		IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> > +		if (hw->vring[i].enable)
> 
> Seems useless check as it already checked when the for loop starts
Yes, it should be removed.
> 
> Thanks,
> Chenbo
> 
> > +			IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> >  	}
> >
> >  	return 0;
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 2b42850..48f1a89 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {
> >  	rte_vhost_get_negotiated_features(vid, &hw->req_features);
> >
> >  	for (i = 0; i < nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> >  		rte_vhost_get_vhost_vring(vid, i, &vq);
> >  		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> >  		if (gpa == 0) {
> > @@ -505,6 +507,8 @@ struct rte_vdpa_dev_info {
> >
> >  	vring.kickfd = -1;
> >  	for (qid = 0; qid < q_num; qid++) {
> > +		if (!hw->vring[qid].enable)
> > +			continue;
> >  		ev.events = EPOLLIN | EPOLLPRI;
> >  		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
> >  		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32; @@ -1064,6
> > +1068,8 @@ struct rte_vdpa_dev_info {
> >  	struct rte_vdpa_device *vdev;
> >  	struct internal_list *list;
> >  	struct ifcvf_internal *internal;
> > +	struct ifcvf_hw *hw;
> > +	uint16_t i;
> >
> >  	vdev = rte_vhost_get_vdpa_device(vid);
> >  	list = find_internal_resource_by_vdev(vdev);
> > @@ -1077,11 +1083,17 @@ struct rte_vdpa_dev_info {
> >  	rte_atomic32_set(&internal->dev_attached, 1);
> >  	update_datapath(internal);
> >
> > -	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) !=
> > 0)
> > -		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
> > +	hw = &internal->hw;
> > +	for (i = 0; i < hw->nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> > +		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
> > +			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
> >  				vdev->device->name);
> > +	}
> >
> >  	internal->configured = 1;
> > +	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
> >  	return 0;
> >  }
> >
> > --
> > 1.8.3.1

[PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei_huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei_huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c       | 101 ++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h       |   6 +++
 drivers/vdpa/ifc/base/ifcvf_osdep.h |   1 +
 drivers/vdpa/ifc/ifcvf_vdpa.c       |  93 +++++++++++++++++++++++++++------
 4 files changed, 186 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 4875ea1..34f32f8 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -230,6 +230,107 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (!hw || (i >= (int)hw->nr_vring))
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		ERROUT("common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_multiqueue(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			ERROUT("queue %u, msix vec alloc failed\n", i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (!hw || (i >= (int)hw->nr_vring))
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		ERROUT("common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+		else
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+
+		if (hw->device_type == IFCVF_BLK)
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		else
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index c17bf2a..e67d4e8 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/base/ifcvf_osdep.h b/drivers/vdpa/ifc/base/ifcvf_osdep.h
index 3d56769..4a1bfec 100644
--- a/drivers/vdpa/ifc/base/ifcvf_osdep.h
+++ b/drivers/vdpa/ifc/base/ifcvf_osdep.h
@@ -16,6 +16,7 @@
 
 #define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
 #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
+#define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args)
 #define STATIC                  static
 
 #define msec_delay(x)	rte_delay_us_sleep(1000 * (x))
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 48f1a89..16fd0fd 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1304,6 +1350,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1311,27 +1360,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, September 8, 2022 1:54 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> From: Huang Wei <wei_huang@intel.com>
> 
> Support dynamic enable or disable queue.
> For front end, like QEMU, user can use ethtool to configurate queue.
> For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> 
> Signed-off-by: Huang Wei <wei_huang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c       | 101
> ++++++++++++++++++++++++++++++++++++
>  drivers/vdpa/ifc/base/ifcvf.h       |   6 +++
>  drivers/vdpa/ifc/base/ifcvf_osdep.h |   1 +
>  drivers/vdpa/ifc/ifcvf_vdpa.c       |  93 +++++++++++++++++++++++++++----
> --
>  4 files changed, 186 insertions(+), 15 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 4875ea1..34f32f8 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -230,6 +230,107 @@
>  	}
>  }
> 
> +int
> +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
> +{
> +	struct ifcvf_pci_common_cfg *cfg;
> +	u8 *lm_cfg;
> +	u16 notify_off;
> +	int msix_vector;
> +
> +	if (!hw || (i >= (int)hw->nr_vring))
> +		return -1;

Seems HW will always be not NULL

> +
> +	cfg = hw->common_cfg;
> +	if (!cfg) {
> +		ERROUT("common_cfg in HW is NULL.\n");

I am thinking why you introduce this new log? Why not just
use DRV_LOG that is already defined?

> +		return -1;
> +	}
> +
> +	ifcvf_enable_multiqueue(hw);
> +
> +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> +	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> +	if (msix_vector != (i + 1)) {
> +		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
> +		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> +		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
> +			ERROUT("queue %u, msix vec alloc failed\n", i);
> +			return -1;
> +		}
> +	}
> +
> +	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> +			&cfg->queue_desc_hi);
> +	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
> +			&cfg->queue_avail_hi);
> +	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
> +			&cfg->queue_used_hi);
> +	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
> +
> +	lm_cfg = hw->lm_cfg;
> +	if (lm_cfg) {
> +		if (hw->device_type == IFCVF_BLK)
> +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> +				i * IFCVF_LM_CFG_SIZE) =
> +				(u32)hw->vring[i].last_avail_idx |
> +				((u32)hw->vring[i].last_used_idx << 16);
> +		else
> +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> +				(i / 2) * IFCVF_LM_CFG_SIZE +
> +				(i % 2) * 4) =
> +				(u32)hw->vring[i].last_avail_idx |
> +				((u32)hw->vring[i].last_used_idx << 16);
> +	}

So the register layout is different for blk and net?

> +
> +	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
> +	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
> +			notify_off * hw->notify_off_multiplier);
> +	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> +
> +	return 0;
> +}
> +
> +void
> +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
> +{
> +	struct ifcvf_pci_common_cfg *cfg;
> +	u32 ring_state;
> +	u8 *lm_cfg;
> +
> +	if (!hw || (i >= (int)hw->nr_vring))
> +		return;
> +
> +	cfg = hw->common_cfg;
> +	if (!cfg) {
> +		ERROUT("common_cfg in HW is NULL.\n");
> +		return;
> +	}
> +
> +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> +	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
> +
> +	lm_cfg = hw->lm_cfg;
> +	if (lm_cfg) {
> +		if (hw->device_type == IFCVF_BLK)
> +			ring_state = *(u32 *)(lm_cfg +
> +					IFCVF_LM_RING_STATE_OFFSET +
> +					i * IFCVF_LM_CFG_SIZE);
> +		else
> +			ring_state = *(u32 *)(lm_cfg +
> +					IFCVF_LM_RING_STATE_OFFSET +
> +					(i / 2) * IFCVF_LM_CFG_SIZE +
> +					(i % 2) * 4);
> +
> +		if (hw->device_type == IFCVF_BLK)
> +			hw->vring[i].last_avail_idx =
> +				(u16)(ring_state & IFCVF_16_BIT_MASK);
> +		else
> +			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);

Above two if-else should be combined.

Thanks,
Chenbo

> +		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
> +	}
> +}
> +
>  STATIC int
>  ifcvf_hw_enable(struct ifcvf_hw *hw)
>  {
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index c17bf2a..e67d4e8 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -164,6 +164,12 @@ struct ifcvf_hw {
>  ifcvf_get_features(struct ifcvf_hw *hw);
> 
>  int
> +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
> +
> +void
> +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
> +
> +int
>  ifcvf_start_hw(struct ifcvf_hw *hw);
> 
>  void
> diff --git a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> index 3d56769..4a1bfec 100644
> --- a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> +++ b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> @@ -16,6 +16,7 @@
> 
>  #define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
>  #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
> +#define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args)
>  #define STATIC                  static
> 
>  #define msec_delay(x)	rte_delay_us_sleep(1000 * (x))
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 48f1a89..16fd0fd 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info {
>  }
> 
>  static int
> +ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
> +{
> +	struct ifcvf_hw *hw = &internal->hw;
> +	int vid = internal->vid;
> +	struct rte_vhost_vring vq;
> +	uint64_t gpa;
> +
> +	if (hw->vring[vring].enable) {
> +		rte_vhost_get_vhost_vring(vid, vring, &vq);
> +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> +		if (gpa == 0) {
> +			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
> +			return -1;
> +		}
> +		hw->vring[vring].desc = gpa;
> +
> +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
> +		if (gpa == 0) {
> +			DRV_LOG(ERR, "Fail to get GPA for available ring.");
> +			return -1;
> +		}
> +		hw->vring[vring].avail = gpa;
> +
> +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
> +		if (gpa == 0) {
> +			DRV_LOG(ERR, "Fail to get GPA for used ring.");
> +			return -1;
> +		}
> +		hw->vring[vring].used = gpa;
> +
> +		hw->vring[vring].size = vq.size;
> +		rte_vhost_get_vring_base(vid, vring,
> +				&hw->vring[vring].last_avail_idx,
> +				&hw->vring[vring].last_used_idx);
> +		ifcvf_enable_vring_hw(&internal->hw, vring);
> +	} else {
> +		ifcvf_disable_vring_hw(&internal->hw, vring);
> +		rte_vhost_set_vring_base(vid, vring,
> +				hw->vring[vring].last_avail_idx,
> +				hw->vring[vring].last_used_idx);
> +	}
> +
> +	return 0;
> +}
> +
> +static int
>  ifcvf_set_vring_state(int vid, int vring, int state)
>  {
>  	struct rte_vdpa_device *vdev;
>  	struct internal_list *list;
>  	struct ifcvf_internal *internal;
>  	struct ifcvf_hw *hw;
> -	struct ifcvf_pci_common_cfg *cfg;
> +	bool enable = !!state;
>  	int ret = 0;
> 
>  	vdev = rte_vhost_get_vdpa_device(vid);
> @@ -1304,6 +1350,9 @@ struct rte_vdpa_dev_info {
>  		return -1;
>  	}
> 
> +	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
> +		enable ? "enable" : "disable", vring, vdev->device->name);
> +
>  	internal = list->internal;
>  	if (vring < 0 || vring >= internal->max_queues * 2) {
>  		DRV_LOG(ERR, "Vring index %d not correct", vring);
> @@ -1311,27 +1360,41 @@ struct rte_vdpa_dev_info {
>  	}
> 
>  	hw = &internal->hw;
> +	hw->vring[vring].enable = enable;
> +
>  	if (!internal->configured)
> -		goto exit;
> +		return 0;
> 
> -	cfg = hw->common_cfg;
> -	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
> -	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
> +	unset_notify_relay(internal);
> 
> -	if (!state && hw->vring[vring].enable) {
> -		ret = vdpa_disable_vfio_intr(internal);
> -		if (ret)
> -			return ret;
> +	ret = vdpa_enable_vfio_intr(internal, false);
> +	if (ret) {
> +		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
> +			vdev->device->name);
> +		return ret;
>  	}
> 
> -	if (state && !hw->vring[vring].enable) {
> -		ret = vdpa_enable_vfio_intr(internal, false);
> -		if (ret)
> -			return ret;
> +	ret = ifcvf_config_vring(internal, vring);
> +	if (ret) {
> +		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
> +			vring, vdev->device->name);
> +		return ret;
> +	}
> +
> +	ret = setup_notify_relay(internal);
> +	if (ret) {
> +		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
> +			vdev->device->name);
> +		return ret;
> +	}
> +
> +	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
> +	if (ret) {
> +		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
> +			vdev->device->name, vring);
> +		return ret;
>  	}
> 
> -exit:
> -	hw->vring[vring].enable = !!state;
>  	return 0;
>  }
> 
> --
> 1.8.3.1

RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue

[Huang, Wei]

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, September 14, 2022 10:23
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> <wei_huang@intel.com>
> Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, September 8, 2022 1:54 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
> >
> > From: Huang Wei <wei_huang@intel.com>
> >
> > Support dynamic enable or disable queue.
> > For front end, like QEMU, user can use ethtool to configurate queue.
> > For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> >
> > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c       | 101
> > ++++++++++++++++++++++++++++++++++++
> >  drivers/vdpa/ifc/base/ifcvf.h       |   6 +++
> >  drivers/vdpa/ifc/base/ifcvf_osdep.h |   1 +
> >  drivers/vdpa/ifc/ifcvf_vdpa.c       |  93 +++++++++++++++++++++++++++----
> > --
> >  4 files changed, 186 insertions(+), 15 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 4875ea1..34f32f8 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -230,6 +230,107 @@
> >  	}
> >  }
> >
> > +int
> > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i) {
> > +	struct ifcvf_pci_common_cfg *cfg;
> > +	u8 *lm_cfg;
> > +	u16 notify_off;
> > +	int msix_vector;
> > +
> > +	if (!hw || (i >= (int)hw->nr_vring))
> > +		return -1;
> 
> Seems HW will always be not NULL
As a external function, we should not assume the input argument is always valid.
> 
> > +
> > +	cfg = hw->common_cfg;
> > +	if (!cfg) {
> > +		ERROUT("common_cfg in HW is NULL.\n");
> 
> I am thinking why you introduce this new log? Why not just use DRV_LOG that is
> already defined?
Because below type of log macros are already defined and used in original code, I just follow its tradition.
#define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
#define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
#define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args) 
> 
> > +		return -1;
> > +	}
> > +
> > +	ifcvf_enable_multiqueue(hw);
> > +
> > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > +	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> > +	if (msix_vector != (i + 1)) {
> > +		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
> > +		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> > +		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
> > +			ERROUT("queue %u, msix vec alloc failed\n", i);
> > +			return -1;
> > +		}
> > +	}
> > +
> > +	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> > +			&cfg->queue_desc_hi);
> > +	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
> > +			&cfg->queue_avail_hi);
> > +	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
> > +			&cfg->queue_used_hi);
> > +	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
> > +
> > +	lm_cfg = hw->lm_cfg;
> > +	if (lm_cfg) {
> > +		if (hw->device_type == IFCVF_BLK)
> > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > +				i * IFCVF_LM_CFG_SIZE) =
> > +				(u32)hw->vring[i].last_avail_idx |
> > +				((u32)hw->vring[i].last_used_idx << 16);
> > +		else
> > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > +				(i / 2) * IFCVF_LM_CFG_SIZE +
> > +				(i % 2) * 4) =
> > +				(u32)hw->vring[i].last_avail_idx |
> > +				((u32)hw->vring[i].last_used_idx << 16);
> > +	}
> 
> So the register layout is different for blk and net?
That's sure.
> 
> > +
> > +	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
> > +	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
> > +			notify_off * hw->notify_off_multiplier);
> > +	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> > +
> > +	return 0;
> > +}
> > +
> > +void
> > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i) {
> > +	struct ifcvf_pci_common_cfg *cfg;
> > +	u32 ring_state;
> > +	u8 *lm_cfg;
> > +
> > +	if (!hw || (i >= (int)hw->nr_vring))
> > +		return;
> > +
> > +	cfg = hw->common_cfg;
> > +	if (!cfg) {
> > +		ERROUT("common_cfg in HW is NULL.\n");
> > +		return;
> > +	}
> > +
> > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > +	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
> > +
> > +	lm_cfg = hw->lm_cfg;
> > +	if (lm_cfg) {
> > +		if (hw->device_type == IFCVF_BLK)
> > +			ring_state = *(u32 *)(lm_cfg +
> > +					IFCVF_LM_RING_STATE_OFFSET +
> > +					i * IFCVF_LM_CFG_SIZE);
> > +		else
> > +			ring_state = *(u32 *)(lm_cfg +
> > +					IFCVF_LM_RING_STATE_OFFSET +
> > +					(i / 2) * IFCVF_LM_CFG_SIZE +
> > +					(i % 2) * 4);
> > +
> > +		if (hw->device_type == IFCVF_BLK)
> > +			hw->vring[i].last_avail_idx =
> > +				(u16)(ring_state & IFCVF_16_BIT_MASK);
> > +		else
> > +			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
> 
> Above two if-else should be combined.
It's a good suggestion.
> 
> Thanks,
> Chenbo
> 
> > +		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
> > +	}
> > +}
> > +
> >  STATIC int
> >  ifcvf_hw_enable(struct ifcvf_hw *hw)
> >  {
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index c17bf2a..e67d4e8 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -164,6 +164,12 @@ struct ifcvf_hw {  ifcvf_get_features(struct
> > ifcvf_hw *hw);
> >
> >  int
> > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
> > +
> > +void
> > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
> > +
> > +int
> >  ifcvf_start_hw(struct ifcvf_hw *hw);
> >
> >  void
> > diff --git a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > index 3d56769..4a1bfec 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > @@ -16,6 +16,7 @@
> >
> >  #define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
> >  #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
> > +#define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args)
> >  #define STATIC                  static
> >
> >  #define msec_delay(x)	rte_delay_us_sleep(1000 * (x))
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 48f1a89..16fd0fd 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info {  }
> >
> >  static int
> > +ifcvf_config_vring(struct ifcvf_internal *internal, int vring) {
> > +	struct ifcvf_hw *hw = &internal->hw;
> > +	int vid = internal->vid;
> > +	struct rte_vhost_vring vq;
> > +	uint64_t gpa;
> > +
> > +	if (hw->vring[vring].enable) {
> > +		rte_vhost_get_vhost_vring(vid, vring, &vq);
> > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> > +		if (gpa == 0) {
> > +			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
> > +			return -1;
> > +		}
> > +		hw->vring[vring].desc = gpa;
> > +
> > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
> > +		if (gpa == 0) {
> > +			DRV_LOG(ERR, "Fail to get GPA for available ring.");
> > +			return -1;
> > +		}
> > +		hw->vring[vring].avail = gpa;
> > +
> > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
> > +		if (gpa == 0) {
> > +			DRV_LOG(ERR, "Fail to get GPA for used ring.");
> > +			return -1;
> > +		}
> > +		hw->vring[vring].used = gpa;
> > +
> > +		hw->vring[vring].size = vq.size;
> > +		rte_vhost_get_vring_base(vid, vring,
> > +				&hw->vring[vring].last_avail_idx,
> > +				&hw->vring[vring].last_used_idx);
> > +		ifcvf_enable_vring_hw(&internal->hw, vring);
> > +	} else {
> > +		ifcvf_disable_vring_hw(&internal->hw, vring);
> > +		rte_vhost_set_vring_base(vid, vring,
> > +				hw->vring[vring].last_avail_idx,
> > +				hw->vring[vring].last_used_idx);
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +static int
> >  ifcvf_set_vring_state(int vid, int vring, int state)  {
> >  	struct rte_vdpa_device *vdev;
> >  	struct internal_list *list;
> >  	struct ifcvf_internal *internal;
> >  	struct ifcvf_hw *hw;
> > -	struct ifcvf_pci_common_cfg *cfg;
> > +	bool enable = !!state;
> >  	int ret = 0;
> >
> >  	vdev = rte_vhost_get_vdpa_device(vid); @@ -1304,6 +1350,9 @@
> struct
> > rte_vdpa_dev_info {
> >  		return -1;
> >  	}
> >
> > +	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
> > +		enable ? "enable" : "disable", vring, vdev->device->name);
> > +
> >  	internal = list->internal;
> >  	if (vring < 0 || vring >= internal->max_queues * 2) {
> >  		DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -
> 1311,27
> > +1360,41 @@ struct rte_vdpa_dev_info {
> >  	}
> >
> >  	hw = &internal->hw;
> > +	hw->vring[vring].enable = enable;
> > +
> >  	if (!internal->configured)
> > -		goto exit;
> > +		return 0;
> >
> > -	cfg = hw->common_cfg;
> > -	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
> > -	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
> > +	unset_notify_relay(internal);
> >
> > -	if (!state && hw->vring[vring].enable) {
> > -		ret = vdpa_disable_vfio_intr(internal);
> > -		if (ret)
> > -			return ret;
> > +	ret = vdpa_enable_vfio_intr(internal, false);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
> > +			vdev->device->name);
> > +		return ret;
> >  	}
> >
> > -	if (state && !hw->vring[vring].enable) {
> > -		ret = vdpa_enable_vfio_intr(internal, false);
> > -		if (ret)
> > -			return ret;
> > +	ret = ifcvf_config_vring(internal, vring);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "failed to configure queue %d of vDPA
> device %s",
> > +			vring, vdev->device->name);
> > +		return ret;
> > +	}
> > +
> > +	ret = setup_notify_relay(internal);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
> > +			vdev->device->name);
> > +		return ret;
> > +	}
> > +
> > +	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
> > +			vdev->device->name, vring);
> > +		return ret;
> >  	}
> >
> > -exit:
> > -	hw->vring[vring].enable = !!state;
> >  	return 0;
> >  }
> >
> > --
> > 1.8.3.1

RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue

[Xia, Chenbo]

> -----Original Message-----
> From: Huang, Wei <wei.huang@intel.com>
> Sent: Wednesday, September 14, 2022 11:04 AM
> To: Xia, Chenbo <chenbo.xia@intel.com>; Pei, Andy <andy.pei@intel.com>;
> dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> 
> 
> > -----Original Message-----
> > From: Xia, Chenbo <chenbo.xia@intel.com>
> > Sent: Wednesday, September 14, 2022 10:23
> > To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> > Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao,
> > Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable
> queue
> >
> > > -----Original Message-----
> > > From: Pei, Andy <andy.pei@intel.com>
> > > Sent: Thursday, September 8, 2022 1:54 PM
> > > To: dev@dpdk.org
> > > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > > <wei_huang@intel.com>
> > > Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
> > >
> > > From: Huang Wei <wei_huang@intel.com>
> > >
> > > Support dynamic enable or disable queue.
> > > For front end, like QEMU, user can use ethtool to configurate queue.
> > > For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> > >
> > > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > > ---
> > >  drivers/vdpa/ifc/base/ifcvf.c       | 101
> > > ++++++++++++++++++++++++++++++++++++
> > >  drivers/vdpa/ifc/base/ifcvf.h       |   6 +++
> > >  drivers/vdpa/ifc/base/ifcvf_osdep.h |   1 +
> > >  drivers/vdpa/ifc/ifcvf_vdpa.c       |  93
> +++++++++++++++++++++++++++----
> > > --
> > >  4 files changed, 186 insertions(+), 15 deletions(-)
> > >
> > > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > > b/drivers/vdpa/ifc/base/ifcvf.c index 4875ea1..34f32f8 100644
> > > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > > @@ -230,6 +230,107 @@
> > >  	}
> > >  }
> > >
> > > +int
> > > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i) {
> > > +	struct ifcvf_pci_common_cfg *cfg;
> > > +	u8 *lm_cfg;
> > > +	u16 notify_off;
> > > +	int msix_vector;
> > > +
> > > +	if (!hw || (i >= (int)hw->nr_vring))
> > > +		return -1;
> >
> > Seems HW will always be not NULL
> As a external function, we should not assume the input argument is always
> valid.

It's actually internal: all logic is inside a your driver. It's not like
a library API.

> >
> > > +
> > > +	cfg = hw->common_cfg;
> > > +	if (!cfg) {
> > > +		ERROUT("common_cfg in HW is NULL.\n");
> >
> > I am thinking why you introduce this new log? Why not just use DRV_LOG
> that is
> > already defined?
> Because below type of log macros are already defined and used in original
> code, I just follow its tradition.
> #define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
> #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
> #define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args)

I think all above seems not needed. DRV_LOG could cover all.

Thanks,
Chenbo

> >
> > > +		return -1;
> > > +	}
> > > +
> > > +	ifcvf_enable_multiqueue(hw);
> > > +
> > > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > > +	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> > > +	if (msix_vector != (i + 1)) {
> > > +		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
> > > +		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> > > +		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
> > > +			ERROUT("queue %u, msix vec alloc failed\n", i);
> > > +			return -1;
> > > +		}
> > > +	}
> > > +
> > > +	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> > > +			&cfg->queue_desc_hi);
> > > +	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
> > > +			&cfg->queue_avail_hi);
> > > +	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
> > > +			&cfg->queue_used_hi);
> > > +	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
> > > +
> > > +	lm_cfg = hw->lm_cfg;
> > > +	if (lm_cfg) {
> > > +		if (hw->device_type == IFCVF_BLK)
> > > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > > +				i * IFCVF_LM_CFG_SIZE) =
> > > +				(u32)hw->vring[i].last_avail_idx |
> > > +				((u32)hw->vring[i].last_used_idx << 16);
> > > +		else
> > > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > > +				(i / 2) * IFCVF_LM_CFG_SIZE +
> > > +				(i % 2) * 4) =
> > > +				(u32)hw->vring[i].last_avail_idx |
> > > +				((u32)hw->vring[i].last_used_idx << 16);
> > > +	}
> >
> > So the register layout is different for blk and net?
> That's sure.
> >
> > > +
> > > +	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
> > > +	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
> > > +			notify_off * hw->notify_off_multiplier);
> > > +	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +void
> > > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i) {
> > > +	struct ifcvf_pci_common_cfg *cfg;
> > > +	u32 ring_state;
> > > +	u8 *lm_cfg;
> > > +
> > > +	if (!hw || (i >= (int)hw->nr_vring))
> > > +		return;
> > > +
> > > +	cfg = hw->common_cfg;
> > > +	if (!cfg) {
> > > +		ERROUT("common_cfg in HW is NULL.\n");
> > > +		return;
> > > +	}
> > > +
> > > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > > +	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
> > > +
> > > +	lm_cfg = hw->lm_cfg;
> > > +	if (lm_cfg) {
> > > +		if (hw->device_type == IFCVF_BLK)
> > > +			ring_state = *(u32 *)(lm_cfg +
> > > +					IFCVF_LM_RING_STATE_OFFSET +
> > > +					i * IFCVF_LM_CFG_SIZE);
> > > +		else
> > > +			ring_state = *(u32 *)(lm_cfg +
> > > +					IFCVF_LM_RING_STATE_OFFSET +
> > > +					(i / 2) * IFCVF_LM_CFG_SIZE +
> > > +					(i % 2) * 4);
> > > +
> > > +		if (hw->device_type == IFCVF_BLK)
> > > +			hw->vring[i].last_avail_idx =
> > > +				(u16)(ring_state & IFCVF_16_BIT_MASK);
> > > +		else
> > > +			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
> >
> > Above two if-else should be combined.
> It's a good suggestion.
> >
> > Thanks,
> > Chenbo
> >
> > > +		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
> > > +	}
> > > +}
> > > +
> > >  STATIC int
> > >  ifcvf_hw_enable(struct ifcvf_hw *hw)
> > >  {
> > > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > > b/drivers/vdpa/ifc/base/ifcvf.h index c17bf2a..e67d4e8 100644
> > > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > > @@ -164,6 +164,12 @@ struct ifcvf_hw {  ifcvf_get_features(struct
> > > ifcvf_hw *hw);
> > >
> > >  int
> > > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
> > > +
> > > +void
> > > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
> > > +
> > > +int
> > >  ifcvf_start_hw(struct ifcvf_hw *hw);
> > >
> > >  void
> > > diff --git a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > index 3d56769..4a1bfec 100644
> > > --- a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > +++ b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > @@ -16,6 +16,7 @@
> > >
> > >  #define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
> > >  #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
> > > +#define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args)
> > >  #define STATIC                  static
> > >
> > >  #define msec_delay(x)	rte_delay_us_sleep(1000 * (x))
> > > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 48f1a89..16fd0fd 100644
> > > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > > @@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info {  }
> > >
> > >  static int
> > > +ifcvf_config_vring(struct ifcvf_internal *internal, int vring) {
> > > +	struct ifcvf_hw *hw = &internal->hw;
> > > +	int vid = internal->vid;
> > > +	struct rte_vhost_vring vq;
> > > +	uint64_t gpa;
> > > +
> > > +	if (hw->vring[vring].enable) {
> > > +		rte_vhost_get_vhost_vring(vid, vring, &vq);
> > > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> > > +		if (gpa == 0) {
> > > +			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
> > > +			return -1;
> > > +		}
> > > +		hw->vring[vring].desc = gpa;
> > > +
> > > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
> > > +		if (gpa == 0) {
> > > +			DRV_LOG(ERR, "Fail to get GPA for available ring.");
> > > +			return -1;
> > > +		}
> > > +		hw->vring[vring].avail = gpa;
> > > +
> > > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
> > > +		if (gpa == 0) {
> > > +			DRV_LOG(ERR, "Fail to get GPA for used ring.");
> > > +			return -1;
> > > +		}
> > > +		hw->vring[vring].used = gpa;
> > > +
> > > +		hw->vring[vring].size = vq.size;
> > > +		rte_vhost_get_vring_base(vid, vring,
> > > +				&hw->vring[vring].last_avail_idx,
> > > +				&hw->vring[vring].last_used_idx);
> > > +		ifcvf_enable_vring_hw(&internal->hw, vring);
> > > +	} else {
> > > +		ifcvf_disable_vring_hw(&internal->hw, vring);
> > > +		rte_vhost_set_vring_base(vid, vring,
> > > +				hw->vring[vring].last_avail_idx,
> > > +				hw->vring[vring].last_used_idx);
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int
> > >  ifcvf_set_vring_state(int vid, int vring, int state)  {
> > >  	struct rte_vdpa_device *vdev;
> > >  	struct internal_list *list;
> > >  	struct ifcvf_internal *internal;
> > >  	struct ifcvf_hw *hw;
> > > -	struct ifcvf_pci_common_cfg *cfg;
> > > +	bool enable = !!state;
> > >  	int ret = 0;
> > >
> > >  	vdev = rte_vhost_get_vdpa_device(vid); @@ -1304,6 +1350,9 @@
> > struct
> > > rte_vdpa_dev_info {
> > >  		return -1;
> > >  	}
> > >
> > > +	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
> > > +		enable ? "enable" : "disable", vring, vdev->device->name);
> > > +
> > >  	internal = list->internal;
> > >  	if (vring < 0 || vring >= internal->max_queues * 2) {
> > >  		DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -
> > 1311,27
> > > +1360,41 @@ struct rte_vdpa_dev_info {
> > >  	}
> > >
> > >  	hw = &internal->hw;
> > > +	hw->vring[vring].enable = enable;
> > > +
> > >  	if (!internal->configured)
> > > -		goto exit;
> > > +		return 0;
> > >
> > > -	cfg = hw->common_cfg;
> > > -	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
> > > -	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
> > > +	unset_notify_relay(internal);
> > >
> > > -	if (!state && hw->vring[vring].enable) {
> > > -		ret = vdpa_disable_vfio_intr(internal);
> > > -		if (ret)
> > > -			return ret;
> > > +	ret = vdpa_enable_vfio_intr(internal, false);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
> > > +			vdev->device->name);
> > > +		return ret;
> > >  	}
> > >
> > > -	if (state && !hw->vring[vring].enable) {
> > > -		ret = vdpa_enable_vfio_intr(internal, false);
> > > -		if (ret)
> > > -			return ret;
> > > +	ret = ifcvf_config_vring(internal, vring);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "failed to configure queue %d of vDPA
> > device %s",
> > > +			vring, vdev->device->name);
> > > +		return ret;
> > > +	}
> > > +
> > > +	ret = setup_notify_relay(internal);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
> > > +			vdev->device->name);
> > > +		return ret;
> > > +	}
> > > +
> > > +	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
> > > +			vdev->device->name, vring);
> > > +		return ret;
> > >  	}
> > >
> > > -exit:
> > > -	hw->vring[vring].enable = !!state;
> > >  	return 0;
> > >  }
> > >
> > > --
> > > 1.8.3.1

RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue

[Pei, Andy]

Hi Chenbo, 

See my reply inline.

> -----Original Message-----
> From: Huang, Wei <wei.huang@intel.com>
> Sent: Wednesday, September 14, 2022 11:04 AM
> To: Xia, Chenbo <Chenbo.Xia@intel.com>; Pei, Andy <andy.pei@intel.com>;
> dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> 
> 
> > -----Original Message-----
> > From: Xia, Chenbo <chenbo.xia@intel.com>
> > Sent: Wednesday, September 14, 2022 10:23
> > To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> > Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> > Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: RE: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable
> > queue
> >
> > > -----Original Message-----
> > > From: Pei, Andy <andy.pei@intel.com>
> > > Sent: Thursday, September 8, 2022 1:54 PM
> > > To: dev@dpdk.org
> > > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > > <wei_huang@intel.com>
> > > Subject: [PATCH v2 6/8] vdpa/ifc: support dynamic enable/disable
> > > queue
> > >
> > > From: Huang Wei <wei_huang@intel.com>
> > >
> > > Support dynamic enable or disable queue.
> > > For front end, like QEMU, user can use ethtool to configurate queue.
> > > For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> > >
> > > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > > ---
> > >  drivers/vdpa/ifc/base/ifcvf.c       | 101
> > > ++++++++++++++++++++++++++++++++++++
> > >  drivers/vdpa/ifc/base/ifcvf.h       |   6 +++
> > >  drivers/vdpa/ifc/base/ifcvf_osdep.h |   1 +
> > >  drivers/vdpa/ifc/ifcvf_vdpa.c       |  93 +++++++++++++++++++++++++++----
> > > --
> > >  4 files changed, 186 insertions(+), 15 deletions(-)
> > >
> > > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > > b/drivers/vdpa/ifc/base/ifcvf.c index 4875ea1..34f32f8 100644
> > > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > > @@ -230,6 +230,107 @@
> > >  	}
> > >  }
> > >
> > > +int
> > > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i) {
> > > +	struct ifcvf_pci_common_cfg *cfg;
> > > +	u8 *lm_cfg;
> > > +	u16 notify_off;
> > > +	int msix_vector;
> > > +
> > > +	if (!hw || (i >= (int)hw->nr_vring))
> > > +		return -1;
> >
> > Seems HW will always be not NULL
> As a external function, we should not assume the input argument is always valid.
> >
> > > +
> > > +	cfg = hw->common_cfg;
> > > +	if (!cfg) {
> > > +		ERROUT("common_cfg in HW is NULL.\n");
> >
> > I am thinking why you introduce this new log? Why not just use DRV_LOG
> > that is already defined?
> Because below type of log macros are already defined and used in original code,
> I just follow its tradition.
> #define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
> #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
> #define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args)
> >
> > > +		return -1;
> > > +	}
> > > +
> > > +	ifcvf_enable_multiqueue(hw);
> > > +
> > > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > > +	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> > > +	if (msix_vector != (i + 1)) {
> > > +		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
> > > +		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> > > +		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
> > > +			ERROUT("queue %u, msix vec alloc failed\n", i);
> > > +			return -1;
> > > +		}
> > > +	}
> > > +
> > > +	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> > > +			&cfg->queue_desc_hi);
> > > +	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
> > > +			&cfg->queue_avail_hi);
> > > +	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
> > > +			&cfg->queue_used_hi);
> > > +	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
> > > +
> > > +	lm_cfg = hw->lm_cfg;
> > > +	if (lm_cfg) {
> > > +		if (hw->device_type == IFCVF_BLK)
> > > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > > +				i * IFCVF_LM_CFG_SIZE) =
> > > +				(u32)hw->vring[i].last_avail_idx |
> > > +				((u32)hw->vring[i].last_used_idx << 16);
> > > +		else
> > > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > > +				(i / 2) * IFCVF_LM_CFG_SIZE +
> > > +				(i % 2) * 4) =
> > > +				(u32)hw->vring[i].last_avail_idx |
> > > +				((u32)hw->vring[i].last_used_idx << 16);
> > > +	}
> >
> > So the register layout is different for blk and net?
> That's sure.
For the register layout differences, the story is as follow:
When I add support for blk device, I tried to re-use the existing code.
However, the BAR4 layout of blk HW is different.
I keep code for net device unchanged, and add code for blk device.
> >
> > > +
> > > +	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
> > > +	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
> > > +			notify_off * hw->notify_off_multiplier);
> > > +	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +void
> > > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i) {
> > > +	struct ifcvf_pci_common_cfg *cfg;
> > > +	u32 ring_state;
> > > +	u8 *lm_cfg;
> > > +
> > > +	if (!hw || (i >= (int)hw->nr_vring))
> > > +		return;
> > > +
> > > +	cfg = hw->common_cfg;
> > > +	if (!cfg) {
> > > +		ERROUT("common_cfg in HW is NULL.\n");
> > > +		return;
> > > +	}
> > > +
> > > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > > +	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
> > > +
> > > +	lm_cfg = hw->lm_cfg;
> > > +	if (lm_cfg) {
> > > +		if (hw->device_type == IFCVF_BLK)
> > > +			ring_state = *(u32 *)(lm_cfg +
> > > +					IFCVF_LM_RING_STATE_OFFSET +
> > > +					i * IFCVF_LM_CFG_SIZE);
> > > +		else
> > > +			ring_state = *(u32 *)(lm_cfg +
> > > +					IFCVF_LM_RING_STATE_OFFSET +
> > > +					(i / 2) * IFCVF_LM_CFG_SIZE +
> > > +					(i % 2) * 4);
> > > +
> > > +		if (hw->device_type == IFCVF_BLK)
> > > +			hw->vring[i].last_avail_idx =
> > > +				(u16)(ring_state & IFCVF_16_BIT_MASK);
> > > +		else
> > > +			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
> >
> > Above two if-else should be combined.
> It's a good suggestion.
> >
> > Thanks,
> > Chenbo
> >
> > > +		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
> > > +	}
> > > +}
> > > +
> > >  STATIC int
> > >  ifcvf_hw_enable(struct ifcvf_hw *hw)  { diff --git
> > > a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> > > index c17bf2a..e67d4e8 100644
> > > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > > @@ -164,6 +164,12 @@ struct ifcvf_hw {  ifcvf_get_features(struct
> > > ifcvf_hw *hw);
> > >
> > >  int
> > > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
> > > +
> > > +void
> > > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
> > > +
> > > +int
> > >  ifcvf_start_hw(struct ifcvf_hw *hw);
> > >
> > >  void
> > > diff --git a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > index 3d56769..4a1bfec 100644
> > > --- a/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > +++ b/drivers/vdpa/ifc/base/ifcvf_osdep.h
> > > @@ -16,6 +16,7 @@
> > >
> > >  #define WARNINGOUT(S, args...)  RTE_LOG(WARNING, PMD, S, ##args)
> > >  #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
> > > +#define ERROUT(S, args...)      RTE_LOG(ERR, PMD, S, ##args)
> > >  #define STATIC                  static
> > >
> > >  #define msec_delay(x)	rte_delay_us_sleep(1000 * (x))
> > > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 48f1a89..16fd0fd 100644
> > > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > > @@ -1288,13 +1288,59 @@ struct rte_vdpa_dev_info {  }
> > >
> > >  static int
> > > +ifcvf_config_vring(struct ifcvf_internal *internal, int vring) {
> > > +	struct ifcvf_hw *hw = &internal->hw;
> > > +	int vid = internal->vid;
> > > +	struct rte_vhost_vring vq;
> > > +	uint64_t gpa;
> > > +
> > > +	if (hw->vring[vring].enable) {
> > > +		rte_vhost_get_vhost_vring(vid, vring, &vq);
> > > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> > > +		if (gpa == 0) {
> > > +			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
> > > +			return -1;
> > > +		}
> > > +		hw->vring[vring].desc = gpa;
> > > +
> > > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
> > > +		if (gpa == 0) {
> > > +			DRV_LOG(ERR, "Fail to get GPA for available ring.");
> > > +			return -1;
> > > +		}
> > > +		hw->vring[vring].avail = gpa;
> > > +
> > > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
> > > +		if (gpa == 0) {
> > > +			DRV_LOG(ERR, "Fail to get GPA for used ring.");
> > > +			return -1;
> > > +		}
> > > +		hw->vring[vring].used = gpa;
> > > +
> > > +		hw->vring[vring].size = vq.size;
> > > +		rte_vhost_get_vring_base(vid, vring,
> > > +				&hw->vring[vring].last_avail_idx,
> > > +				&hw->vring[vring].last_used_idx);
> > > +		ifcvf_enable_vring_hw(&internal->hw, vring);
> > > +	} else {
> > > +		ifcvf_disable_vring_hw(&internal->hw, vring);
> > > +		rte_vhost_set_vring_base(vid, vring,
> > > +				hw->vring[vring].last_avail_idx,
> > > +				hw->vring[vring].last_used_idx);
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static int
> > >  ifcvf_set_vring_state(int vid, int vring, int state)  {
> > >  	struct rte_vdpa_device *vdev;
> > >  	struct internal_list *list;
> > >  	struct ifcvf_internal *internal;
> > >  	struct ifcvf_hw *hw;
> > > -	struct ifcvf_pci_common_cfg *cfg;
> > > +	bool enable = !!state;
> > >  	int ret = 0;
> > >
> > >  	vdev = rte_vhost_get_vdpa_device(vid); @@ -1304,6 +1350,9 @@
> > struct
> > > rte_vdpa_dev_info {
> > >  		return -1;
> > >  	}
> > >
> > > +	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
> > > +		enable ? "enable" : "disable", vring, vdev->device->name);
> > > +
> > >  	internal = list->internal;
> > >  	if (vring < 0 || vring >= internal->max_queues * 2) {
> > >  		DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -
> > 1311,27
> > > +1360,41 @@ struct rte_vdpa_dev_info {
> > >  	}
> > >
> > >  	hw = &internal->hw;
> > > +	hw->vring[vring].enable = enable;
> > > +
> > >  	if (!internal->configured)
> > > -		goto exit;
> > > +		return 0;
> > >
> > > -	cfg = hw->common_cfg;
> > > -	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
> > > -	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
> > > +	unset_notify_relay(internal);
> > >
> > > -	if (!state && hw->vring[vring].enable) {
> > > -		ret = vdpa_disable_vfio_intr(internal);
> > > -		if (ret)
> > > -			return ret;
> > > +	ret = vdpa_enable_vfio_intr(internal, false);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
> > > +			vdev->device->name);
> > > +		return ret;
> > >  	}
> > >
> > > -	if (state && !hw->vring[vring].enable) {
> > > -		ret = vdpa_enable_vfio_intr(internal, false);
> > > -		if (ret)
> > > -			return ret;
> > > +	ret = ifcvf_config_vring(internal, vring);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "failed to configure queue %d of vDPA
> > device %s",
> > > +			vring, vdev->device->name);
> > > +		return ret;
> > > +	}
> > > +
> > > +	ret = setup_notify_relay(internal);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
> > > +			vdev->device->name);
> > > +		return ret;
> > > +	}
> > > +
> > > +	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
> > > +	if (ret) {
> > > +		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
> > > +			vdev->device->name, vring);
> > > +		return ret;
> > >  	}
> > >
> > > -exit:
> > > -	hw->vring[vring].enable = !!state;
> > >  	return 0;
> > >  }
> > >
> > > --
> > > 1.8.3.1

[PATCH v2 7/8] vhost: configure device when any queue is ready for BLK device

[Andy Pei]

When boot from virtio blk device, seabois in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end conf_dev when any
queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 lib/vhost/vhost_user.c | 56 +++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 12 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index 4ad28ba..b65fba3 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1451,6 +1451,25 @@
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
 
 static int
+virtio_has_queue_ready(struct virtio_net *dev)
+{
+	struct vhost_virtqueue *vq;
+	uint32_t i, nr_vring = dev->nr_vring;
+
+	if (!dev->nr_vring)
+		return 0;
+
+	for (i = 0; i < nr_vring; i++) {
+		vq = dev->virtqueue[i];
+
+		if (vq_is_ready(dev, vq))
+			return 1;
+	}
+
+	return 0;
+}
+
+static int
 virtio_is_ready(struct virtio_net *dev)
 {
 	struct vhost_virtqueue *vq;
@@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (unlock_required)
 		vhost_user_unlock_all_queue_pairs(dev);
 
-	if (ret != 0 || !virtio_is_ready(dev))
+	if (ret != 0)
 		goto out;
 
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev) {
+		if (vdpa_dev->ops->get_dev_type) {
+			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
+			if (ret) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+					"failed to get vdpa dev type.\n");
+				ret = -1;
+				goto out;
+			}
+		} else {
+			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+		}
+	}
+
+	if (!virtio_is_ready(dev)) {
+		if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+			if (!virtio_has_queue_ready(dev))
+				goto out;
+		} else {
+			goto out;
+		}
+	}
+
 	/*
 	 * Virtio is now ready. If not done already, it is time
 	 * to notify the application it can process the rings and
@@ -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
 			dev->flags |= VIRTIO_DEV_RUNNING;
 	}
 
-	vdpa_dev = dev->vdpa_dev;
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
 	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
-- 
1.8.3.1

RE: [PATCH v2 7/8] vhost: configure device when any queue is ready for BLK device

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, September 8, 2022 1:54 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com; Huang Wei <wei_huang@intel.com>
> Subject: [PATCH v2 7/8] vhost: configure device when any queue is ready
> for BLK device
> 
> When boot from virtio blk device, seabois in QEMU only enables one queue.
> To work in this scenario, vDPA BLK device back-end conf_dev when any
> queue is ready.

So this is the case only for vdpa blk, for SW vhost-blk, all queues need to
be ready?

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei_huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 56 +++++++++++++++++++++++++++++++++++++++------
> -----
>  1 file changed, 44 insertions(+), 12 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index 4ad28ba..b65fba3 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -1451,6 +1451,25 @@
>  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> 
>  static int
> +virtio_has_queue_ready(struct virtio_net *dev)
> +{
> +	struct vhost_virtqueue *vq;
> +	uint32_t i, nr_vring = dev->nr_vring;
> +
> +	if (!dev->nr_vring)
> +		return 0;
> +
> +	for (i = 0; i < nr_vring; i++) {
> +		vq = dev->virtqueue[i];
> +
> +		if (vq_is_ready(dev, vq))
> +			return 1;
> +	}
> +
> +	return 0;
> +}
> +
> +static int
>  virtio_is_ready(struct virtio_net *dev)
>  {
>  	struct vhost_virtqueue *vq;
> @@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	if (unlock_required)
>  		vhost_user_unlock_all_queue_pairs(dev);
> 
> -	if (ret != 0 || !virtio_is_ready(dev))
> +	if (ret != 0)
>  		goto out;
> 
> +	vdpa_dev = dev->vdpa_dev;
> +	if (vdpa_dev) {
> +		if (vdpa_dev->ops->get_dev_type) {
> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> +			if (ret) {
> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> +					"failed to get vdpa dev type.\n");
> +				ret = -1;
> +				goto out;
> +			}
> +		} else {
> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> +		}
> +	}
> +
> +	if (!virtio_is_ready(dev)) {
> +		if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +			if (!virtio_has_queue_ready(dev))
> +				goto out;
> +		} else {
> +			goto out;
> +		}
> +	}
> +

I feel like if possible, above logic should all be in virtio_is_ready.

Thanks,
Chenbo

>  	/*
>  	 * Virtio is now ready. If not done already, it is time
>  	 * to notify the application it can process the rings and
> @@ -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  			dev->flags |= VIRTIO_DEV_RUNNING;
>  	}
> 
> -	vdpa_dev = dev->vdpa_dev;
>  	if (!vdpa_dev)
>  		goto out;
> 
> -	if (vdpa_dev->ops->get_dev_type) {
> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> -		if (ret) {
> -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa
> dev type.\n");
> -			ret = -1;
> -			goto out;
> -		}
> -	} else {
> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> -	}
>  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
>  		&& request != VHOST_USER_SET_VRING_CALL)
>  		goto out;
> --
> 1.8.3.1

RE: [PATCH v2 7/8] vhost: configure device when any queue is ready for BLK device

[Pei, Andy]

HI Chenbo, 

Thanks for your reply, my reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, September 14, 2022 10:51 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v2 7/8] vhost: configure device when any queue is ready for
> BLK device
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, September 8, 2022 1:54 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com; Huang Wei
> > <wei_huang@intel.com>
> > Subject: [PATCH v2 7/8] vhost: configure device when any queue is
> > ready for BLK device
> >
> > When boot from virtio blk device, seabois in QEMU only enables one queue.
> > To work in this scenario, vDPA BLK device back-end conf_dev when any
> > queue is ready.
> 
> So this is the case only for vdpa blk, for SW vhost-blk, all queues need to be
> ready?
> 
The case I mention is that the OS image is in the vdpa device.
In this case, QEMU bios (seabios) will take charge. There is a virtio driver in seabios.
This driver only use one queue.
So in this case vdpa driver need to conf_dev when one queue is ready.

for SW vhost-blk, I am not sure you mean OS image is in your back-end vhost-blk.
I have not test that case.

> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei_huang@intel.com>
> > ---
> >  lib/vhost/vhost_user.c | 56
> > +++++++++++++++++++++++++++++++++++++++------
> > -----
> >  1 file changed, 44 insertions(+), 12 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > 4ad28ba..b65fba3 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -1451,6 +1451,25 @@
> >  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> >
> >  static int
> > +virtio_has_queue_ready(struct virtio_net *dev) {
> > +	struct vhost_virtqueue *vq;
> > +	uint32_t i, nr_vring = dev->nr_vring;
> > +
> > +	if (!dev->nr_vring)
> > +		return 0;
> > +
> > +	for (i = 0; i < nr_vring; i++) {
> > +		vq = dev->virtqueue[i];
> > +
> > +		if (vq_is_ready(dev, vq))
> > +			return 1;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +static int
> >  virtio_is_ready(struct virtio_net *dev)  {
> >  	struct vhost_virtqueue *vq;
> > @@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	if (unlock_required)
> >  		vhost_user_unlock_all_queue_pairs(dev);
> >
> > -	if (ret != 0 || !virtio_is_ready(dev))
> > +	if (ret != 0)
> >  		goto out;
> >
> > +	vdpa_dev = dev->vdpa_dev;
> > +	if (vdpa_dev) {
> > +		if (vdpa_dev->ops->get_dev_type) {
> > +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
> &vdpa_type);
> > +			if (ret) {
> > +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> > +					"failed to get vdpa dev type.\n");
> > +				ret = -1;
> > +				goto out;
> > +			}
> > +		} else {
> > +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > +		}
> > +	}
> > +
> > +	if (!virtio_is_ready(dev)) {
> > +		if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > +			if (!virtio_has_queue_ready(dev))
> > +				goto out;
> > +		} else {
> > +			goto out;
> > +		}
> > +	}
> > +
> 
> I feel like if possible, above logic should all be in virtio_is_ready.
> 
Maybe I can introduce something like VIRTIO_DEV_BUILTIN_VIRTIO_NET,
What do you think?

> Thanks,
> Chenbo
> 
> >  	/*
> >  	 * Virtio is now ready. If not done already, it is time
> >  	 * to notify the application it can process the rings and @@
> > -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  			dev->flags |= VIRTIO_DEV_RUNNING;
> >  	}
> >
> > -	vdpa_dev = dev->vdpa_dev;
> >  	if (!vdpa_dev)
> >  		goto out;
> >
> > -	if (vdpa_dev->ops->get_dev_type) {
> > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > -		if (ret) {
> > -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get
> vdpa
> > dev type.\n");
> > -			ret = -1;
> > -			goto out;
> > -		}
> > -	} else {
> > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > -	}
> >  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> >  		&& request != VHOST_USER_SET_VRING_CALL)
> >  		goto out;
> > --
> > 1.8.3.1

[PATCH v2 8/8] vhost: vDPA BLK devices configure device when all queue callfds are set

[Andy Pei]

In the virtio blk vDPA live migration use case, before the live
migration process, QEMU will set call fd to vDPA back-end. QEMU
and vDPA back-end stand by until live migration starts.
During live migration process, QEMU sets kick fd and a new call
fd. However, after the kick fd is set to the vDPA back-end, the
vDPA back-end configures device and data path starts. The new
call fd will cause some kind of "re-configuration", this kind
of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd
and call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact
net device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index b65fba3..568030a 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2994,6 +2994,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	uint32_t vdpa_type = 0;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3227,9 +3228,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

[PATCH v3 0/8] vdpa/ifc: add multi queue support

[Andy Pei]

v3:
 rename device ID macro name.
 fix some patch title and commit message.
 delete some used marco.
 rework some code logic.

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (6):
  vdpa/ifc: add multi-queue support
  vdpa/ifc: set max queues based on virtio spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vhost: vDPA blk device gets ready when any queue is ready
  vhost: improve vDPA blk device readiness condition

Huang Wei (2):
  vdpa/ifc: add new device ID for legacy network device
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c | 137 ++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  16 ++++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++-------
 lib/vhost/vhost_user.c        |  58 +++++++++++------
 4 files changed, 311 insertions(+), 42 deletions(-)

-- 
1.8.3.1

[PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..ef7697a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,12 +12,14 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
-#define IFCVF_BLK_DEVICE_ID                 0x0002
+#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
+#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
 #define IFCVF_MAX_QUEUES		1
 
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index ac42de9..07cc63d 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .vendor_id = 0, /* sentinel */
-- 
1.8.3.1

Re: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device

[Maxime Coquelin]

On 9/16/22 08:16, Andy Pei wrote:
> From: Huang Wei <wei.huang@intel.com>
> 
> Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
> Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
> 
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
>   drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
>   2 files changed, 14 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index 9d95aac..ef7697a 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -12,12 +12,14 @@
>   #define IFCVF_BLK	1
>   
>   #define IFCVF_VENDOR_ID                     0x1AF4
> -#define IFCVF_NET_DEVICE_ID                 0x1041
> +#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
>   #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
> +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
>   #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
>   #define IFCVF_SUBSYS_VENDOR_ID              0x8086
>   #define IFCVF_SUBSYS_DEVICE_ID              0x001A
> -#define IFCVF_BLK_DEVICE_ID                 0x0002
> +#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
> +#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
>   
>   #define IFCVF_MAX_QUEUES		1
>   
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index ac42de9..07cc63d 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
>   static const struct rte_pci_id pci_id_ifcvf_map[] = {
>   	{ .class_id = RTE_CLASS_ANY_ID,
>   	  .vendor_id = IFCVF_VENDOR_ID,
> -	  .device_id = IFCVF_NET_DEVICE_ID,
> +	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
>   	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
>   	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
>   	},
>   
>   	{ .class_id = RTE_CLASS_ANY_ID,
>   	  .vendor_id = IFCVF_VENDOR_ID,
> +	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
> +	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> +	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
> +	},
> +
> +	{ .class_id = RTE_CLASS_ANY_ID,
> +	  .vendor_id = IFCVF_VENDOR_ID,
>   	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
>   	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
>   	},
>   
>   	{ .class_id = RTE_CLASS_ANY_ID,
>   	  .vendor_id = IFCVF_VENDOR_ID,
>   	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
>   	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
>   	},
>   
>   	{ .vendor_id = 0, /* sentinel */

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks,
Maxime

RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device

[Pei, Andy]

Hi Maxime,

Thanks for your effort.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Wednesday, October 12, 2022 12:49 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network
> device
> 
> 
> 
> On 9/16/22 08:16, Andy Pei wrote:
> > From: Huang Wei <wei.huang@intel.com>
> >
> > Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID
> (0x1000).
> > Rename macro from "IFCVF_BLK_DEVICE_ID" to
> "IFCVF_SUBSYS_BLK_DEVICE_ID".
> >
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >   drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
> >   drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
> >   2 files changed, 14 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index 9d95aac..ef7697a 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -12,12 +12,14 @@
> >   #define IFCVF_BLK	1
> >
> >   #define IFCVF_VENDOR_ID                     0x1AF4
> > -#define IFCVF_NET_DEVICE_ID                 0x1041
> > +#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
> >   #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
> > +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
> >   #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
> >   #define IFCVF_SUBSYS_VENDOR_ID              0x8086
> >   #define IFCVF_SUBSYS_DEVICE_ID              0x001A
> > -#define IFCVF_BLK_DEVICE_ID                 0x0002
> > +#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
> > +#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> >
> >   #define IFCVF_MAX_QUEUES		1
> >
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index ac42de9..07cc63d 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
> >   static const struct rte_pci_id pci_id_ifcvf_map[] = {
> >   	{ .class_id = RTE_CLASS_ANY_ID,
> >   	  .vendor_id = IFCVF_VENDOR_ID,
> > -	  .device_id = IFCVF_NET_DEVICE_ID,
> > +	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
> >   	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> >   	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
> >   	},
> >
> >   	{ .class_id = RTE_CLASS_ANY_ID,
> >   	  .vendor_id = IFCVF_VENDOR_ID,
> > +	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
> > +	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> > +	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
> > +	},
> > +
> > +	{ .class_id = RTE_CLASS_ANY_ID,
> > +	  .vendor_id = IFCVF_VENDOR_ID,
> >   	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
> >   	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> > -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> > +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
> >   	},
> >
> >   	{ .class_id = RTE_CLASS_ANY_ID,
> >   	  .vendor_id = IFCVF_VENDOR_ID,
> >   	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
> >   	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> > -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> > +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
> >   	},
> >
> >   	{ .vendor_id = 0, /* sentinel */
> 
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> 
> Thanks,
> Maxime

RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:16 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network
> device
> 
> From: Huang Wei <wei.huang@intel.com>
> 
> Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
> Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".
> 
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
>  2 files changed, 14 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index 9d95aac..ef7697a 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -12,12 +12,14 @@
>  #define IFCVF_BLK	1
> 
>  #define IFCVF_VENDOR_ID                     0x1AF4
> -#define IFCVF_NET_DEVICE_ID                 0x1041
> +#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
>  #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
> +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
>  #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
>  #define IFCVF_SUBSYS_VENDOR_ID              0x8086
>  #define IFCVF_SUBSYS_DEVICE_ID              0x001A
> -#define IFCVF_BLK_DEVICE_ID                 0x0002
> +#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
> +#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> 
>  #define IFCVF_MAX_QUEUES		1
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index ac42de9..07cc63d 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
>  static const struct rte_pci_id pci_id_ifcvf_map[] = {
>  	{ .class_id = RTE_CLASS_ANY_ID,
>  	  .vendor_id = IFCVF_VENDOR_ID,
> -	  .device_id = IFCVF_NET_DEVICE_ID,
> +	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
>  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
>  	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
>  	},
> 
>  	{ .class_id = RTE_CLASS_ANY_ID,
>  	  .vendor_id = IFCVF_VENDOR_ID,
> +	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
> +	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> +	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
> +	},
> +
> +	{ .class_id = RTE_CLASS_ANY_ID,
> +	  .vendor_id = IFCVF_VENDOR_ID,
>  	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
>  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
>  	},
> 
>  	{ .class_id = RTE_CLASS_ANY_ID,
>  	  .vendor_id = IFCVF_VENDOR_ID,
>  	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
>  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
>  	},
> 
>  	{ .vendor_id = 0, /* sentinel */
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network device

[Pei, Andy]

Hi Chenbo,

Thanks for your effort.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 11:00 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network
> device
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 1/8] vdpa/ifc: add new device ID for legacy network
> > device
> >
> > From: Huang Wei <wei.huang@intel.com>
> >
> > Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID
> (0x1000).
> > Rename macro from "IFCVF_BLK_DEVICE_ID" to
> "IFCVF_SUBSYS_BLK_DEVICE_ID".
> >
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
> > drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
> >  2 files changed, 14 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index 9d95aac..ef7697a 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -12,12 +12,14 @@
> >  #define IFCVF_BLK	1
> >
> >  #define IFCVF_VENDOR_ID                     0x1AF4
> > -#define IFCVF_NET_DEVICE_ID                 0x1041
> > +#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
> >  #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
> > +#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
> >  #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
> >  #define IFCVF_SUBSYS_VENDOR_ID              0x8086
> >  #define IFCVF_SUBSYS_DEVICE_ID              0x001A
> > -#define IFCVF_BLK_DEVICE_ID                 0x0002
> > +#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
> > +#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> >
> >  #define IFCVF_MAX_QUEUES		1
> >
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index ac42de9..07cc63d 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
> > static const struct rte_pci_id pci_id_ifcvf_map[] = {
> >  	{ .class_id = RTE_CLASS_ANY_ID,
> >  	  .vendor_id = IFCVF_VENDOR_ID,
> > -	  .device_id = IFCVF_NET_DEVICE_ID,
> > +	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
> >  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> >  	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
> >  	},
> >
> >  	{ .class_id = RTE_CLASS_ANY_ID,
> >  	  .vendor_id = IFCVF_VENDOR_ID,
> > +	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
> > +	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> > +	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
> > +	},
> > +
> > +	{ .class_id = RTE_CLASS_ANY_ID,
> > +	  .vendor_id = IFCVF_VENDOR_ID,
> >  	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
> >  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> > -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> > +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
> >  	},
> >
> >  	{ .class_id = RTE_CLASS_ANY_ID,
> >  	  .vendor_id = IFCVF_VENDOR_ID,
> >  	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
> >  	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
> > -	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
> > +	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
> >  	},
> >
> >  	{ .vendor_id = 0, /* sentinel */
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v3 2/8] vdpa/ifc: add multi-queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
Expose IFCVF_MQ_OFFSET register to enable multi queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 5 +++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 8 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..34c8226 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,11 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ef7697a..d16d9ab 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 07cc63d..3e5ffba 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

RE: [PATCH v3 2/8] vdpa/ifc: add multi-queue support

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:16 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 2/8] vdpa/ifc: add multi-queue support
> 
> Enable VHOST_USER_PROTOCOL_F_MQ feature.
> Expose IFCVF_MQ_OFFSET register to enable multi queue.

Multi-queue

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 5 +++++
>  drivers/vdpa/ifc/base/ifcvf.h | 2 ++
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
>  3 files changed, 8 insertions(+)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index f1e1474..34c8226 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -90,6 +90,11 @@
>  	if (!hw->lm_cfg)
>  		WARNINGOUT("HW support live migration not support!\n");
> 
> +	if (hw->mem_resource[4].addr)
> +		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
> +	else
> +		hw->mq_cfg = NULL;
> +

It will be better to add comments about the explanation that you sent before.

Thanks,
Chenbo

>  	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
>  			hw->isr == NULL || hw->dev_cfg == NULL) {
>  		DEBUGOUT("capability incomplete\n");
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index ef7697a..d16d9ab 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -50,6 +50,7 @@
> 
>  #define IFCVF_LM_CFG_SIZE		0x40
>  #define IFCVF_LM_RING_STATE_OFFSET	0x20
> +#define IFCVF_MQ_OFFSET			0x28
> 
>  #define IFCVF_LM_LOGGING_CTRL		0x0
> 
> @@ -149,6 +150,7 @@ struct ifcvf_hw {
>  	u16    *notify_base;
>  	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
>  	u8     *lm_cfg;
> +	u8     *mq_cfg;
>  	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
>  	u8 nr_vring;
>  	int device_type;
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 07cc63d..3e5ffba 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
>  		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
> +		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
> 
>  #define VDPA_BLK_PROTOCOL_FEATURES \
> --
> 1.8.3.1

RE: [PATCH v3 2/8] vdpa/ifc: add multi-queue support

[Pei, Andy]

Hi Chenbo,

Sure. I will address in next version.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 11:02 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 2/8] vdpa/ifc: add multi-queue support
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 2/8] vdpa/ifc: add multi-queue support
> >
> > Enable VHOST_USER_PROTOCOL_F_MQ feature.
> > Expose IFCVF_MQ_OFFSET register to enable multi queue.
> 
> Multi-queue
> 
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 5 +++++
> > drivers/vdpa/ifc/base/ifcvf.h | 2 ++  drivers/vdpa/ifc/ifcvf_vdpa.c |
> > 1 +
> >  3 files changed, 8 insertions(+)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index f1e1474..34c8226 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -90,6 +90,11 @@
> >  	if (!hw->lm_cfg)
> >  		WARNINGOUT("HW support live migration not support!\n");
> >
> > +	if (hw->mem_resource[4].addr)
> > +		hw->mq_cfg = hw->mem_resource[4].addr +
> IFCVF_MQ_OFFSET;
> > +	else
> > +		hw->mq_cfg = NULL;
> > +
> 
> It will be better to add comments about the explanation that you sent before.
> 
> Thanks,
> Chenbo
> 
> >  	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
> >  			hw->isr == NULL || hw->dev_cfg == NULL) {
> >  		DEBUGOUT("capability incomplete\n"); diff --git
> > a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h index
> > ef7697a..d16d9ab 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -50,6 +50,7 @@
> >
> >  #define IFCVF_LM_CFG_SIZE		0x40
> >  #define IFCVF_LM_RING_STATE_OFFSET	0x20
> > +#define IFCVF_MQ_OFFSET			0x28
> >
> >  #define IFCVF_LM_LOGGING_CTRL		0x0
> >
> > @@ -149,6 +150,7 @@ struct ifcvf_hw {
> >  	u16    *notify_base;
> >  	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
> >  	u8     *lm_cfg;
> > +	u8     *mq_cfg;
> >  	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
> >  	u8 nr_vring;
> >  	int device_type;
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 07cc63d..3e5ffba 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
> >  		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
> > +		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
> >
> >  #define VDPA_BLK_PROTOCOL_FEATURES \
> > --
> > 1.8.3.1

[PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec

[Andy Pei]

Set max_queues according to virtio HW spec.
For virtio BLK device, set max_queues to the value of "num_queues".
"num_queues" is element of struct virtio_blk_config.
For virtio NET device, read num_queues from truct ifcvf_pci_common_cfg,
calculate "queue_pairs = (num_queues - 1) / 2" and get queue_pairs.
Set max_queues to the value of "queue_pairs".

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index d16d9ab..1e133c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
 #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 3e5ffba..376239a 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,12 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MIN() cannot be used since braced-group within expression allowed
+ * only inside a function.
+ */
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		/*
+		 * ifc driver always has CTRL_VQ,
+		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
+		 */
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:16 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
> 
> Set max_queues according to virtio HW spec.

Virtio spec

> For virtio BLK device, set max_queues to the value of "num_queues".
> "num_queues" is element of struct virtio_blk_config.

Set max_queues to the value of num_queues in struct virtio_blk_config

> For virtio NET device, read num_queues from truct ifcvf_pci_common_cfg,

truct -> struct 

> calculate "queue_pairs = (num_queues - 1) / 2" and get queue_pairs.
> Set max_queues to the value of "queue_pairs".

And say 'get the queue pair number using num_queues and set max_queues to it'.

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.h |  2 +-
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
>  2 files changed, 19 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index d16d9ab..1e133c0 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -21,7 +21,7 @@
>  #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
>  #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> 
> -#define IFCVF_MAX_QUEUES		1
> +#define IFCVF_MAX_QUEUES		32
> 
>  #ifndef VIRTIO_F_IOMMU_PLATFORM
>  #define VIRTIO_F_IOMMU_PLATFORM		33
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 3e5ffba..376239a 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -26,6 +26,12 @@
> 
>  #include "base/ifcvf.h"
> 
> +/*
> + * RTE_MIN() cannot be used since braced-group within expression allowed
> + * only inside a function.
> + */
> +#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
> +
>  RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
>  #define DRV_LOG(level, fmt, args...) \
>  	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
> @@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	uint64_t capacity = 0;
>  	uint8_t *byte;
>  	uint32_t i;
> +	uint16_t queue_pairs;
> 
>  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
>  		return 0;
> @@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	}
> 
>  	internal->configured = 0;
> -	internal->max_queues = IFCVF_MAX_QUEUES;
>  	features = ifcvf_get_features(&internal->hw);
> 
>  	device_id = ifcvf_pci_get_device_type(pci_dev);
> @@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
> 
>  	if (device_id == VIRTIO_ID_NET) {
>  		internal->hw.device_type = IFCVF_NET;
> +		/*
> +		 * ifc driver always has CTRL_VQ,

I think it's ifc _device_ has CTRL_VQ

Thanks,
Chenbo

> +		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
> +		 */
> +		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
> +		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
> +			queue_pairs);
> +		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
>  		internal->features = features &
>  					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
>  		internal->features |= dev_info[IFCVF_NET].features;
> @@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
>  			internal->hw.blk_cfg->geometry.sectors);
>  		DRV_LOG(DEBUG, "num_queues: 0x%08x",
>  			internal->hw.blk_cfg->num_queues);
> +
> +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> +			internal->hw.blk_cfg->num_queues);
>  	}
> 
>  	list->internal = internal;
> --
> 1.8.3.1

RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec

[Pei, Andy]

Hi Chenbo,

Sure. I will address in next version.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 2:09 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
> >
> > Set max_queues according to virtio HW spec.
> 
> Virtio spec
> 
> > For virtio BLK device, set max_queues to the value of "num_queues".
> > "num_queues" is element of struct virtio_blk_config.
> 
> Set max_queues to the value of num_queues in struct virtio_blk_config
> 
> > For virtio NET device, read num_queues from truct
> > ifcvf_pci_common_cfg,
> 
> truct -> struct
> 
> > calculate "queue_pairs = (num_queues - 1) / 2" and get queue_pairs.
> > Set max_queues to the value of "queue_pairs".
> 
> And say 'get the queue pair number using num_queues and set max_queues
> to it'.
> 
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.h |  2 +-  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 19 ++++++++++++++++++-
> >  2 files changed, 19 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index d16d9ab..1e133c0 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -21,7 +21,7 @@
> >  #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
> >  #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> >
> > -#define IFCVF_MAX_QUEUES		1
> > +#define IFCVF_MAX_QUEUES		32
> >
> >  #ifndef VIRTIO_F_IOMMU_PLATFORM
> >  #define VIRTIO_F_IOMMU_PLATFORM		33
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 3e5ffba..376239a 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -26,6 +26,12 @@
> >
> >  #include "base/ifcvf.h"
> >
> > +/*
> > + * RTE_MIN() cannot be used since braced-group within expression
> > +allowed
> > + * only inside a function.
> > + */
> > +#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
> > +
> >  RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
> > #define DRV_LOG(level, fmt, args...) \
> >  	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6
> +1518,7
> > @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	uint64_t capacity = 0;
> >  	uint8_t *byte;
> >  	uint32_t i;
> > +	uint16_t queue_pairs;
> >
> >  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> >  		return 0;
> > @@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	}
> >
> >  	internal->configured = 0;
> > -	internal->max_queues = IFCVF_MAX_QUEUES;
> >  	features = ifcvf_get_features(&internal->hw);
> >
> >  	device_id = ifcvf_pci_get_device_type(pci_dev);
> > @@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
> >
> >  	if (device_id == VIRTIO_ID_NET) {
> >  		internal->hw.device_type = IFCVF_NET;
> > +		/*
> > +		 * ifc driver always has CTRL_VQ,
> 
> I think it's ifc _device_ has CTRL_VQ
> 
> Thanks,
> Chenbo
> 
> > +		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
> > +		 */
> > +		queue_pairs = (internal->hw.common_cfg->num_queues - 1)
> / 2;
> > +		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev-
> >name,
> > +			queue_pairs);
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> queue_pairs);
> >  		internal->features = features &
> >  					~(1ULL <<
> VIRTIO_F_IOMMU_PLATFORM);
> >  		internal->features |= dev_info[IFCVF_NET].features; @@ -
> 1609,6
> > +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  			internal->hw.blk_cfg->geometry.sectors);
> >  		DRV_LOG(DEBUG, "num_queues: 0x%08x",
> >  			internal->hw.blk_cfg->num_queues);
> > +
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> > +			internal->hw.blk_cfg->num_queues);
> >  	}
> >
> >  	list->internal = internal;
> > --
> > 1.8.3.1

RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec

[Pei, Andy]

Hi Chenbo,

Sure. I will address in next version.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 2:09 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 3/8] vdpa/ifc: set max queues based on virtio spec
> >
> > Set max_queues according to virtio HW spec.
> 
> Virtio spec
> 
> > For virtio BLK device, set max_queues to the value of "num_queues".
> > "num_queues" is element of struct virtio_blk_config.
> 
> Set max_queues to the value of num_queues in struct virtio_blk_config
> 
> > For virtio NET device, read num_queues from truct
> > ifcvf_pci_common_cfg,
> 
> truct -> struct
> 
> > calculate "queue_pairs = (num_queues - 1) / 2" and get queue_pairs.
> > Set max_queues to the value of "queue_pairs".
> 
> And say 'get the queue pair number using num_queues and set max_queues
> to it'.
> 
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.h |  2 +-  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 19 ++++++++++++++++++-
> >  2 files changed, 19 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index d16d9ab..1e133c0 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -21,7 +21,7 @@
> >  #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
> >  #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> >
> > -#define IFCVF_MAX_QUEUES		1
> > +#define IFCVF_MAX_QUEUES		32
> >
> >  #ifndef VIRTIO_F_IOMMU_PLATFORM
> >  #define VIRTIO_F_IOMMU_PLATFORM		33
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 3e5ffba..376239a 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -26,6 +26,12 @@
> >
> >  #include "base/ifcvf.h"
> >
> > +/*
> > + * RTE_MIN() cannot be used since braced-group within expression
> > +allowed
> > + * only inside a function.
> > + */
> > +#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
> > +
> >  RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
> > #define DRV_LOG(level, fmt, args...) \
> >  	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6
> +1518,7
> > @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	uint64_t capacity = 0;
> >  	uint8_t *byte;
> >  	uint32_t i;
> > +	uint16_t queue_pairs;
> >
> >  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> >  		return 0;
> > @@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	}
> >
> >  	internal->configured = 0;
> > -	internal->max_queues = IFCVF_MAX_QUEUES;
> >  	features = ifcvf_get_features(&internal->hw);
> >
> >  	device_id = ifcvf_pci_get_device_type(pci_dev);
> > @@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
> >
> >  	if (device_id == VIRTIO_ID_NET) {
> >  		internal->hw.device_type = IFCVF_NET;
> > +		/*
> > +		 * ifc driver always has CTRL_VQ,
> 
> I think it's ifc _device_ has CTRL_VQ
> 
> Thanks,
> Chenbo
> 
> > +		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
> > +		 */
> > +		queue_pairs = (internal->hw.common_cfg->num_queues - 1)
> / 2;
> > +		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev-
> >name,
> > +			queue_pairs);
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> queue_pairs);
> >  		internal->features = features &
> >  					~(1ULL <<
> VIRTIO_F_IOMMU_PLATFORM);
> >  		internal->features |= dev_info[IFCVF_NET].features; @@ -
> 1609,6
> > +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  			internal->hw.blk_cfg->geometry.sectors);
> >  		DRV_LOG(DEBUG, "num_queues: 0x%08x",
> >  			internal->hw.blk_cfg->num_queues);
> > +
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> > +			internal->hw.blk_cfg->num_queues);
> >  	}
> >
> >  	list->internal = internal;
> > --
> > 1.8.3.1

[PATCH v3 4/8] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 34c8226..7efb408 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -198,6 +198,35 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_multiqueue(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	int qid;
+	int nr_queue_pair = 0;
+
+	for (qid = 0; qid < hw->nr_vring; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
+		nr_queue_pair++;
+	}
+
+	if (nr_queue_pair == 0) {
+		WARNINGOUT("no enabled vring\n");
+		return;
+	}
+
+	if (hw->device_type == IFCVF_NET)
+		nr_queue_pair = (nr_queue_pair + 1) / 2;
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		*(u32 *)mq_cfg = nr_queue_pair;
+		RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+			nr_queue_pair);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -215,6 +244,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_multiqueue(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

RE: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:16 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register
> 
> Write queue count to IFCVF_MQ_OFFSET register
> to enable multi queue feature.

Multi-queue

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 30 ++++++++++++++++++++++++++++++
>  1 file changed, 30 insertions(+)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 34c8226..7efb408 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -198,6 +198,35 @@
>  	IFCVF_WRITE_REG32(val >> 32, hi);
>  }
> 
> +STATIC void
> +ifcvf_enable_multiqueue(struct ifcvf_hw *hw)

Maybe ifcvf_enable_mq?

> +{
> +	u8 *mq_cfg;
> +	int qid;

Qid should be the same type as nr_vring, so u8

> +	int nr_queue_pair = 0;
> +
> +	for (qid = 0; qid < hw->nr_vring; qid++) {
> +		if (!hw->vring[qid].enable)
> +			continue;
> +		nr_queue_pair++;
> +	}
> +
> +	if (nr_queue_pair == 0) {
> +		WARNINGOUT("no enabled vring\n");
> +		return;
> +	}
> +
> +	if (hw->device_type == IFCVF_NET)
> +		nr_queue_pair = (nr_queue_pair + 1) / 2;

Why +1?

> +
> +	mq_cfg = hw->mq_cfg;
> +	if (mq_cfg) {
> +		*(u32 *)mq_cfg = nr_queue_pair;
> +		RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
> +			nr_queue_pair);
> +	}

As blk does not have queue pair, the log should be different for net/blk.

The variable nr_queue_pair should be nr_queue as it's the number of queues.
When device type is net, you can make *(u32 *)mq_cfg equal to nr_queue / 2

Thanks,
Chenbo

> +}
> +
>  STATIC int
>  ifcvf_hw_enable(struct ifcvf_hw *hw)
>  {
> @@ -215,6 +244,7 @@
>  		return -1;
>  	}
> 
> +	ifcvf_enable_multiqueue(hw);
>  	for (i = 0; i < hw->nr_vring; i++) {
>  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
>  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> --
> 1.8.3.1

RE: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register

[Pei, Andy]

Hi Chenbo

Thanks for your reply, my reply inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 4:08 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 4/8] vdpa/ifc: write queue count to MQ register
> >
> > Write queue count to IFCVF_MQ_OFFSET register to enable multi queue
> > feature.
> 
> Multi-queue
> 
Sure.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 30 ++++++++++++++++++++++++++++++
> >  1 file changed, 30 insertions(+)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 34c8226..7efb408 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -198,6 +198,35 @@
> >  	IFCVF_WRITE_REG32(val >> 32, hi);
> >  }
> >
> > +STATIC void
> > +ifcvf_enable_multiqueue(struct ifcvf_hw *hw)
> 
> Maybe ifcvf_enable_mq?
> 
OK.
> > +{
> > +	u8 *mq_cfg;
> > +	int qid;
> 
> Qid should be the same type as nr_vring, so u8
> 
> > +	int nr_queue_pair = 0;
> > +
> > +	for (qid = 0; qid < hw->nr_vring; qid++) {
> > +		if (!hw->vring[qid].enable)
> > +			continue;
> > +		nr_queue_pair++;
> > +	}
> > +
> > +	if (nr_queue_pair == 0) {
> > +		WARNINGOUT("no enabled vring\n");
> > +		return;
> > +	}
> > +
> > +	if (hw->device_type == IFCVF_NET)
> > +		nr_queue_pair = (nr_queue_pair + 1) / 2;
> 
> Why +1?
> 
> > +
> > +	mq_cfg = hw->mq_cfg;
> > +	if (mq_cfg) {
> > +		*(u32 *)mq_cfg = nr_queue_pair;
> > +		RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
> > +			nr_queue_pair);
> > +	}
> 
> As blk does not have queue pair, the log should be different for net/blk.
> 
> The variable nr_queue_pair should be nr_queue as it's the number of
> queues.
> When device type is net, you can make *(u32 *)mq_cfg equal to nr_queue / 2
> 
OK. I will sent a new version to address it.

> Thanks,
> Chenbo
> 
> > +}
> > +
> >  STATIC int
> >  ifcvf_hw_enable(struct ifcvf_hw *hw)
> >  {
> > @@ -215,6 +244,7 @@
> >  		return -1;
> >  	}
> >
> > +	ifcvf_enable_multiqueue(hw);
> >  	for (i = 0; i < hw->nr_vring; i++) {
> >  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
> >  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> > --
> > 1.8.3.1

[PATCH v3 5/8] vdpa/ifc: only configure enabled queue

[Andy Pei]

when configure the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  3 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 7efb408..619b034 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -246,6 +246,9 @@
 
 	ifcvf_enable_multiqueue(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 376239a..b00afdb 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

RE: [PATCH v3 5/8] vdpa/ifc: only configure enabled queue

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:16 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 5/8] vdpa/ifc: only configure enabled queue
> 
> when configure the hardware queue, we only configure queues which
> have been enabled by vhost.

Should be capital 'W' and 'When configuring'

With this fixed:

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c |  3 +++
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
>  2 files changed, 17 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 7efb408..619b034 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -246,6 +246,9 @@
> 
>  	ifcvf_enable_multiqueue(hw);
>  	for (i = 0; i < hw->nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
> +
>  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
>  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
>  				&cfg->queue_desc_hi);
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 376239a..b00afdb 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
>  	rte_vhost_get_negotiated_features(vid, &hw->req_features);
> 
>  	for (i = 0; i < nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
>  		rte_vhost_get_vhost_vring(vid, i, &vq);
>  		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
>  		if (gpa == 0) {
> @@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
> 
>  	vring.kickfd = -1;
>  	for (qid = 0; qid < q_num; qid++) {
> +		if (!hw->vring[qid].enable)
> +			continue;
>  		ev.events = EPOLLIN | EPOLLPRI;
>  		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
>  		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
> @@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
>  	struct rte_vdpa_device *vdev;
>  	struct internal_list *list;
>  	struct ifcvf_internal *internal;
> +	struct ifcvf_hw *hw;
> +	uint16_t i;
> 
>  	vdev = rte_vhost_get_vdpa_device(vid);
>  	list = find_internal_resource_by_vdev(vdev);
> @@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
>  	rte_atomic32_set(&internal->dev_attached, 1);
>  	update_datapath(internal);
> 
> -	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) !=
> 0)
> -		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
> +	hw = &internal->hw;
> +	for (i = 0; i < hw->nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
> +		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
> +			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
>  				vdev->device->name);
> +	}
> 
>  	internal->configured = 1;
> +	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
>  	return 0;
>  }
> 
> --
> 1.8.3.1

RE: [PATCH v3 5/8] vdpa/ifc: only configure enabled queue

[Pei, Andy]

Hi Chenbo,

Thanks for your effort.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 4:12 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 5/8] vdpa/ifc: only configure enabled queue
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 5/8] vdpa/ifc: only configure enabled queue
> >
> > when configure the hardware queue, we only configure queues which have
> > been enabled by vhost.
> 
> Should be capital 'W' and 'When configuring'
> 
> With this fixed:
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
> 
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c |  3 +++  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 16 ++++++++++++++--
> >  2 files changed, 17 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 7efb408..619b034 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -246,6 +246,9 @@
> >
> >  	ifcvf_enable_multiqueue(hw);
> >  	for (i = 0; i < hw->nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> > +
> >  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
> >  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> >  				&cfg->queue_desc_hi);
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 376239a..b00afdb 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
> >  	rte_vhost_get_negotiated_features(vid, &hw->req_features);
> >
> >  	for (i = 0; i < nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> >  		rte_vhost_get_vhost_vring(vid, i, &vq);
> >  		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> >  		if (gpa == 0) {
> > @@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
> >
> >  	vring.kickfd = -1;
> >  	for (qid = 0; qid < q_num; qid++) {
> > +		if (!hw->vring[qid].enable)
> > +			continue;
> >  		ev.events = EPOLLIN | EPOLLPRI;
> >  		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
> >  		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32; @@ -1058,6
> > +1062,8 @@ struct rte_vdpa_dev_info {
> >  	struct rte_vdpa_device *vdev;
> >  	struct internal_list *list;
> >  	struct ifcvf_internal *internal;
> > +	struct ifcvf_hw *hw;
> > +	uint16_t i;
> >
> >  	vdev = rte_vhost_get_vdpa_device(vid);
> >  	list = find_internal_resource_by_vdev(vdev);
> > @@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
> >  	rte_atomic32_set(&internal->dev_attached, 1);
> >  	update_datapath(internal);
> >
> > -	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) !=
> > 0)
> > -		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
> > +	hw = &internal->hw;
> > +	for (i = 0; i < hw->nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> > +		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
> > +			DRV_LOG(NOTICE, "vDPA (%s): software relay is
> used.",
> >  				vdev->device->name);
> > +	}
> >
> >  	internal->configured = 1;
> > +	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device-
> >name);
> >  	return 0;
> >  }
> >
> > --
> > 1.8.3.1

[PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configurate queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 99 +++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  6 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++++++++-------
 3 files changed, 183 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 619b034..792d258 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -227,6 +227,105 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (i >= (int)hw->nr_vring)
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		WARNINGOUT("common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_multiqueue(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			WARNINGOUT("queue %u, msix vec alloc failed\n", i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (i >= (int)hw->nr_vring)
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		WARNINGOUT("common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		} else {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		}
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 1e133c0..3726da7 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index b00afdb..32bc1c9 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

RE: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:16 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> From: Huang Wei <wei.huang@intel.com>
> 
> Support dynamic enable or disable queue.
> For front end, like QEMU, user can use ethtool to configurate queue.

configure

> For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> 
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 99
> +++++++++++++++++++++++++++++++++++++++++++
>  drivers/vdpa/ifc/base/ifcvf.h |  6 +++
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 93 +++++++++++++++++++++++++++++++++-----
> --
>  3 files changed, 183 insertions(+), 15 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 619b034..792d258 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -227,6 +227,105 @@
>  	}
>  }
> 
> +int
> +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
> +{
> +	struct ifcvf_pci_common_cfg *cfg;
> +	u8 *lm_cfg;
> +	u16 notify_off;
> +	int msix_vector;
> +
> +	if (i >= (int)hw->nr_vring)
> +		return -1;
> +
> +	cfg = hw->common_cfg;
> +	if (!cfg) {
> +		WARNINGOUT("common_cfg in HW is NULL.\n");

This should be error log

> +		return -1;
> +	}
> +
> +	ifcvf_enable_multiqueue(hw);
> +
> +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> +	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> +	if (msix_vector != (i + 1)) {
> +		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
> +		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> +		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
> +			WARNINGOUT("queue %u, msix vec alloc failed\n", i);

Ditto. And %u -> %d

Same for the function ifcvf_disable_vring_hw.

Thanks,
Chenbo

> +			return -1;
> +		}
> +	}
> +
> +	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> +			&cfg->queue_desc_hi);
> +	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
> +			&cfg->queue_avail_hi);
> +	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
> +			&cfg->queue_used_hi);
> +	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
> +
> +	lm_cfg = hw->lm_cfg;
> +	if (lm_cfg) {
> +		if (hw->device_type == IFCVF_BLK)
> +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> +				i * IFCVF_LM_CFG_SIZE) =
> +				(u32)hw->vring[i].last_avail_idx |
> +				((u32)hw->vring[i].last_used_idx << 16);
> +		else
> +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> +				(i / 2) * IFCVF_LM_CFG_SIZE +
> +				(i % 2) * 4) =
> +				(u32)hw->vring[i].last_avail_idx |
> +				((u32)hw->vring[i].last_used_idx << 16);
> +	}
> +
> +	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
> +	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
> +			notify_off * hw->notify_off_multiplier);
> +	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> +
> +	return 0;
> +}
> +
> +void
> +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
> +{
> +	struct ifcvf_pci_common_cfg *cfg;
> +	u32 ring_state;
> +	u8 *lm_cfg;
> +
> +	if (i >= (int)hw->nr_vring)
> +		return;
> +
> +	cfg = hw->common_cfg;
> +	if (!cfg) {
> +		WARNINGOUT("common_cfg in HW is NULL.\n");
> +		return;
> +	}
> +
> +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> +	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
> +
> +	lm_cfg = hw->lm_cfg;
> +	if (lm_cfg) {
> +		if (hw->device_type == IFCVF_BLK) {
> +			ring_state = *(u32 *)(lm_cfg +
> +					IFCVF_LM_RING_STATE_OFFSET +
> +					i * IFCVF_LM_CFG_SIZE);
> +			hw->vring[i].last_avail_idx =
> +				(u16)(ring_state & IFCVF_16_BIT_MASK);
> +		} else {
> +			ring_state = *(u32 *)(lm_cfg +
> +					IFCVF_LM_RING_STATE_OFFSET +
> +					(i / 2) * IFCVF_LM_CFG_SIZE +
> +					(i % 2) * 4);
> +			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
> +		}
> +		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
> +	}
> +}
> +
>  STATIC int
>  ifcvf_hw_enable(struct ifcvf_hw *hw)
>  {
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index 1e133c0..3726da7 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -164,6 +164,12 @@ struct ifcvf_hw {
>  ifcvf_get_features(struct ifcvf_hw *hw);
> 
>  int
> +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
> +
> +void
> +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
> +
> +int
>  ifcvf_start_hw(struct ifcvf_hw *hw);
> 
>  void
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index b00afdb..32bc1c9 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
>  }
> 
>  static int
> +ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
> +{
> +	struct ifcvf_hw *hw = &internal->hw;
> +	int vid = internal->vid;
> +	struct rte_vhost_vring vq;
> +	uint64_t gpa;
> +
> +	if (hw->vring[vring].enable) {
> +		rte_vhost_get_vhost_vring(vid, vring, &vq);
> +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> +		if (gpa == 0) {
> +			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
> +			return -1;
> +		}
> +		hw->vring[vring].desc = gpa;
> +
> +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
> +		if (gpa == 0) {
> +			DRV_LOG(ERR, "Fail to get GPA for available ring.");
> +			return -1;
> +		}
> +		hw->vring[vring].avail = gpa;
> +
> +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
> +		if (gpa == 0) {
> +			DRV_LOG(ERR, "Fail to get GPA for used ring.");
> +			return -1;
> +		}
> +		hw->vring[vring].used = gpa;
> +
> +		hw->vring[vring].size = vq.size;
> +		rte_vhost_get_vring_base(vid, vring,
> +				&hw->vring[vring].last_avail_idx,
> +				&hw->vring[vring].last_used_idx);
> +		ifcvf_enable_vring_hw(&internal->hw, vring);
> +	} else {
> +		ifcvf_disable_vring_hw(&internal->hw, vring);
> +		rte_vhost_set_vring_base(vid, vring,
> +				hw->vring[vring].last_avail_idx,
> +				hw->vring[vring].last_used_idx);
> +	}
> +
> +	return 0;
> +}
> +
> +static int
>  ifcvf_set_vring_state(int vid, int vring, int state)
>  {
>  	struct rte_vdpa_device *vdev;
>  	struct internal_list *list;
>  	struct ifcvf_internal *internal;
>  	struct ifcvf_hw *hw;
> -	struct ifcvf_pci_common_cfg *cfg;
> +	bool enable = !!state;
>  	int ret = 0;
> 
>  	vdev = rte_vhost_get_vdpa_device(vid);
> @@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
>  		return -1;
>  	}
> 
> +	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
> +		enable ? "enable" : "disable", vring, vdev->device->name);
> +
>  	internal = list->internal;
>  	if (vring < 0 || vring >= internal->max_queues * 2) {
>  		DRV_LOG(ERR, "Vring index %d not correct", vring);
> @@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
>  	}
> 
>  	hw = &internal->hw;
> +	hw->vring[vring].enable = enable;
> +
>  	if (!internal->configured)
> -		goto exit;
> +		return 0;
> 
> -	cfg = hw->common_cfg;
> -	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
> -	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
> +	unset_notify_relay(internal);
> 
> -	if (!state && hw->vring[vring].enable) {
> -		ret = vdpa_disable_vfio_intr(internal);
> -		if (ret)
> -			return ret;
> +	ret = vdpa_enable_vfio_intr(internal, false);
> +	if (ret) {
> +		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
> +			vdev->device->name);
> +		return ret;
>  	}
> 
> -	if (state && !hw->vring[vring].enable) {
> -		ret = vdpa_enable_vfio_intr(internal, false);
> -		if (ret)
> -			return ret;
> +	ret = ifcvf_config_vring(internal, vring);
> +	if (ret) {
> +		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
> +			vring, vdev->device->name);
> +		return ret;
> +	}
> +
> +	ret = setup_notify_relay(internal);
> +	if (ret) {
> +		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
> +			vdev->device->name);
> +		return ret;
> +	}
> +
> +	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
> +	if (ret) {
> +		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
> +			vdev->device->name, vring);
> +		return ret;
>  	}
> 
> -exit:
> -	hw->vring[vring].enable = !!state;
>  	return 0;
>  }
> 
> --
> 1.8.3.1

RE: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue

[Pei, Andy]

Hi Chenbo,

Thanks for your reply.
My reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 4:20 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 6/8] vdpa/ifc: support dynamic enable/disable queue
> >
> > From: Huang Wei <wei.huang@intel.com>
> >
> > Support dynamic enable or disable queue.
> > For front end, like QEMU, user can use ethtool to configurate queue.
> 
> configure
> 
Fix in next version.
> > For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> >
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 99
> > +++++++++++++++++++++++++++++++++++++++++++
> >  drivers/vdpa/ifc/base/ifcvf.h |  6 +++  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 93 +++++++++++++++++++++++++++++++++-----
> > --
> >  3 files changed, 183 insertions(+), 15 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 619b034..792d258 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -227,6 +227,105 @@
> >  	}
> >  }
> >
> > +int
> > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i) {
> > +	struct ifcvf_pci_common_cfg *cfg;
> > +	u8 *lm_cfg;
> > +	u16 notify_off;
> > +	int msix_vector;
> > +
> > +	if (i >= (int)hw->nr_vring)
> > +		return -1;
> > +
> > +	cfg = hw->common_cfg;
> > +	if (!cfg) {
> > +		WARNINGOUT("common_cfg in HW is NULL.\n");
> 
> This should be error log
> 
> > +		return -1;
> > +	}
> > +
> > +	ifcvf_enable_multiqueue(hw);
> > +
> > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > +	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
> > +	if (msix_vector != (i + 1)) {
> > +		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
> > +		msix_vector = IFCVF_READ_REG16(&cfg-
> >queue_msix_vector);
> > +		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
> > +			WARNINGOUT("queue %u, msix vec alloc failed\n", i);
> 
> Ditto. And %u -> %d
> 
> Same for the function ifcvf_disable_vring_hw.
> 
I will use 
RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");

> Thanks,
> Chenbo
> 
> > +			return -1;
> > +		}
> > +	}
> > +
> > +	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> > +			&cfg->queue_desc_hi);
> > +	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
> > +			&cfg->queue_avail_hi);
> > +	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
> > +			&cfg->queue_used_hi);
> > +	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
> > +
> > +	lm_cfg = hw->lm_cfg;
> > +	if (lm_cfg) {
> > +		if (hw->device_type == IFCVF_BLK)
> > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > +				i * IFCVF_LM_CFG_SIZE) =
> > +				(u32)hw->vring[i].last_avail_idx |
> > +				((u32)hw->vring[i].last_used_idx << 16);
> > +		else
> > +			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
> > +				(i / 2) * IFCVF_LM_CFG_SIZE +
> > +				(i % 2) * 4) =
> > +				(u32)hw->vring[i].last_avail_idx |
> > +				((u32)hw->vring[i].last_used_idx << 16);
> > +	}
> > +
> > +	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
> > +	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
> > +			notify_off * hw->notify_off_multiplier);
> > +	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
> > +
> > +	return 0;
> > +}
> > +
> > +void
> > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i) {
> > +	struct ifcvf_pci_common_cfg *cfg;
> > +	u32 ring_state;
> > +	u8 *lm_cfg;
> > +
> > +	if (i >= (int)hw->nr_vring)
> > +		return;
> > +
> > +	cfg = hw->common_cfg;
> > +	if (!cfg) {
> > +		WARNINGOUT("common_cfg in HW is NULL.\n");
> > +		return;
> > +	}
> > +
> > +	IFCVF_WRITE_REG16(i, &cfg->queue_select);
> > +	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
> > +
> > +	lm_cfg = hw->lm_cfg;
> > +	if (lm_cfg) {
> > +		if (hw->device_type == IFCVF_BLK) {
> > +			ring_state = *(u32 *)(lm_cfg +
> > +					IFCVF_LM_RING_STATE_OFFSET +
> > +					i * IFCVF_LM_CFG_SIZE);
> > +			hw->vring[i].last_avail_idx =
> > +				(u16)(ring_state & IFCVF_16_BIT_MASK);
> > +		} else {
> > +			ring_state = *(u32 *)(lm_cfg +
> > +					IFCVF_LM_RING_STATE_OFFSET +
> > +					(i / 2) * IFCVF_LM_CFG_SIZE +
> > +					(i % 2) * 4);
> > +			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
> > +		}
> > +		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
> > +	}
> > +}
> > +
> >  STATIC int
> >  ifcvf_hw_enable(struct ifcvf_hw *hw)
> >  {
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index 1e133c0..3726da7 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -164,6 +164,12 @@ struct ifcvf_hw {  ifcvf_get_features(struct
> > ifcvf_hw *hw);
> >
> >  int
> > +ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
> > +
> > +void
> > +ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
> > +
> > +int
> >  ifcvf_start_hw(struct ifcvf_hw *hw);
> >
> >  void
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index b00afdb..32bc1c9 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {  }
> >
> >  static int
> > +ifcvf_config_vring(struct ifcvf_internal *internal, int vring) {
> > +	struct ifcvf_hw *hw = &internal->hw;
> > +	int vid = internal->vid;
> > +	struct rte_vhost_vring vq;
> > +	uint64_t gpa;
> > +
> > +	if (hw->vring[vring].enable) {
> > +		rte_vhost_get_vhost_vring(vid, vring, &vq);
> > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> > +		if (gpa == 0) {
> > +			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
> > +			return -1;
> > +		}
> > +		hw->vring[vring].desc = gpa;
> > +
> > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
> > +		if (gpa == 0) {
> > +			DRV_LOG(ERR, "Fail to get GPA for available ring.");
> > +			return -1;
> > +		}
> > +		hw->vring[vring].avail = gpa;
> > +
> > +		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
> > +		if (gpa == 0) {
> > +			DRV_LOG(ERR, "Fail to get GPA for used ring.");
> > +			return -1;
> > +		}
> > +		hw->vring[vring].used = gpa;
> > +
> > +		hw->vring[vring].size = vq.size;
> > +		rte_vhost_get_vring_base(vid, vring,
> > +				&hw->vring[vring].last_avail_idx,
> > +				&hw->vring[vring].last_used_idx);
> > +		ifcvf_enable_vring_hw(&internal->hw, vring);
> > +	} else {
> > +		ifcvf_disable_vring_hw(&internal->hw, vring);
> > +		rte_vhost_set_vring_base(vid, vring,
> > +				hw->vring[vring].last_avail_idx,
> > +				hw->vring[vring].last_used_idx);
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +static int
> >  ifcvf_set_vring_state(int vid, int vring, int state)  {
> >  	struct rte_vdpa_device *vdev;
> >  	struct internal_list *list;
> >  	struct ifcvf_internal *internal;
> >  	struct ifcvf_hw *hw;
> > -	struct ifcvf_pci_common_cfg *cfg;
> > +	bool enable = !!state;
> >  	int ret = 0;
> >
> >  	vdev = rte_vhost_get_vdpa_device(vid); @@ -1298,6 +1344,9 @@
> struct
> > rte_vdpa_dev_info {
> >  		return -1;
> >  	}
> >
> > +	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
> > +		enable ? "enable" : "disable", vring, vdev->device->name);
> > +
> >  	internal = list->internal;
> >  	if (vring < 0 || vring >= internal->max_queues * 2) {
> >  		DRV_LOG(ERR, "Vring index %d not correct", vring); @@ -
> 1305,27
> > +1354,41 @@ struct rte_vdpa_dev_info {
> >  	}
> >
> >  	hw = &internal->hw;
> > +	hw->vring[vring].enable = enable;
> > +
> >  	if (!internal->configured)
> > -		goto exit;
> > +		return 0;
> >
> > -	cfg = hw->common_cfg;
> > -	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
> > -	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
> > +	unset_notify_relay(internal);
> >
> > -	if (!state && hw->vring[vring].enable) {
> > -		ret = vdpa_disable_vfio_intr(internal);
> > -		if (ret)
> > -			return ret;
> > +	ret = vdpa_enable_vfio_intr(internal, false);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA
> device %s",
> > +			vdev->device->name);
> > +		return ret;
> >  	}
> >
> > -	if (state && !hw->vring[vring].enable) {
> > -		ret = vdpa_enable_vfio_intr(internal, false);
> > -		if (ret)
> > -			return ret;
> > +	ret = ifcvf_config_vring(internal, vring);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "failed to configure queue %d of vDPA
> device %s",
> > +			vring, vdev->device->name);
> > +		return ret;
> > +	}
> > +
> > +	ret = setup_notify_relay(internal);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "failed to setup notify relay of vDPA
> device %s",
> > +			vdev->device->name);
> > +		return ret;
> > +	}
> > +
> > +	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
> > +	if (ret) {
> > +		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl
> fail",
> > +			vdev->device->name, vring);
> > +		return ret;
> >  	}
> >
> > -exit:
> > -	hw->vring[vring].enable = !!state;
> >  	return 0;
> >  }
> >
> > --
> > 1.8.3.1

[PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Andy Pei]

When boot from virtio blk device, seabios in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end conf_dev when any
queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index 4ad28ba..9169cf5 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1449,9 +1449,10 @@
 }
 
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
+#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
 
 static int
-virtio_is_ready(struct virtio_net *dev)
+virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
 {
 	struct vhost_virtqueue *vq;
 	uint32_t i, nr_vring = dev->nr_vring;
@@ -1462,13 +1463,20 @@
 	if (!dev->nr_vring)
 		return 0;
 
-	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
-		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
-
-		if (dev->nr_vring < nr_vring)
-			return 0;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_NET) {
+		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
+			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
+	} else {
+		/*
+		 * vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
+		 * is the only case currently
+		 */
+		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
 	}
 
+	if (dev->nr_vring < nr_vring)
+		return 0;
+
 	for (i = 0; i < nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
@@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (unlock_required)
 		vhost_user_unlock_all_queue_pairs(dev);
 
-	if (ret != 0 || !virtio_is_ready(dev))
+	if (ret != 0)
+		goto out;
+
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev) {
+		if (vdpa_dev->ops->get_dev_type) {
+			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
+			if (ret) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+					"failed to get vdpa dev type.\n");
+				ret = -1;
+				goto out;
+			}
+		} else {
+			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+		}
+	}
+
+	if (!virtio_is_ready(dev, vdpa_type))
 		goto out;
 
 	/*
@@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
 			dev->flags |= VIRTIO_DEV_RUNNING;
 	}
 
-	vdpa_dev = dev->vdpa_dev;
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
 	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
-- 
1.8.3.1

RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:16 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue
> is ready
> 
> When boot from virtio blk device, seabios in QEMU only enables one queue.
> To work in this scenario, vDPA BLK device back-end conf_dev when any

What is conf_dev?

> queue is ready.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 51 ++++++++++++++++++++++++++++++++-------------
> -----
>  1 file changed, 33 insertions(+), 18 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index 4ad28ba..9169cf5 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -1449,9 +1449,10 @@
>  }
> 
>  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> 
>  static int
> -virtio_is_ready(struct virtio_net *dev)
> +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
>  {
>  	struct vhost_virtqueue *vq;
>  	uint32_t i, nr_vring = dev->nr_vring;
> @@ -1462,13 +1463,20 @@
>  	if (!dev->nr_vring)
>  		return 0;
> 
> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> -
> -		if (dev->nr_vring < nr_vring)
> -			return 0;
> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_NET) {
> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> +			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> +	} else {
> +		/*
> +		 * vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> +		 * is the only case currently
> +		 */
> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;

You should consider the case when vdpa device is not there. Maybe you can
use int for vdpa_type, -1 for non-vdpa.

Also note that below check is only needed for some cases.

Thanks,
Chenbo

>  	}
> 
> +	if (dev->nr_vring < nr_vring)
> +		return 0;
> +
>  	for (i = 0; i < nr_vring; i++) {
>  		vq = dev->virtqueue[i];
> 
> @@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	if (unlock_required)
>  		vhost_user_unlock_all_queue_pairs(dev);
> 
> -	if (ret != 0 || !virtio_is_ready(dev))
> +	if (ret != 0)
> +		goto out;
> +
> +	vdpa_dev = dev->vdpa_dev;
> +	if (vdpa_dev) {
> +		if (vdpa_dev->ops->get_dev_type) {
> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> +			if (ret) {
> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> +					"failed to get vdpa dev type.\n");
> +				ret = -1;
> +				goto out;
> +			}
> +		} else {
> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> +		}
> +	}
> +
> +	if (!virtio_is_ready(dev, vdpa_type))
>  		goto out;
> 
>  	/*
> @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  			dev->flags |= VIRTIO_DEV_RUNNING;
>  	}
> 
> -	vdpa_dev = dev->vdpa_dev;
>  	if (!vdpa_dev)
>  		goto out;
> 
> -	if (vdpa_dev->ops->get_dev_type) {
> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> -		if (ret) {
> -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa
> dev type.\n");
> -			ret = -1;
> -			goto out;
> -		}
> -	} else {
> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> -	}
>  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
>  		&& request != VHOST_USER_SET_VRING_CALL)
>  		goto out;
> --
> 1.8.3.1

RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Pei, Andy]

Hi Chenbo,

Thanks for your reply.
My reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 5:09 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> queue is ready
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:16 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> > queue is ready
> >
> > When boot from virtio blk device, seabios in QEMU only enables one queue.
> > To work in this scenario, vDPA BLK device back-end conf_dev when any
> 
> What is conf_dev?
> 
I refer to
	/** Driver configure the device (Mandatory) */
	int (*dev_conf)(int vid);
So do you think I should use "configure device"?

> > queue is ready.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  lib/vhost/vhost_user.c | 51
> > ++++++++++++++++++++++++++++++++-------------
> > -----
> >  1 file changed, 33 insertions(+), 18 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > 4ad28ba..9169cf5 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -1449,9 +1449,10 @@
> >  }
> >
> >  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> > +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> >
> >  static int
> > -virtio_is_ready(struct virtio_net *dev)
> > +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
> >  {
> >  	struct vhost_virtqueue *vq;
> >  	uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@
> >  	if (!dev->nr_vring)
> >  		return 0;
> >
> > -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> > -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > -
> > -		if (dev->nr_vring < nr_vring)
> > -			return 0;
> > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_NET) {
> > +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> > +			nr_vring =
> VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > +	} else {
> > +		/*
> > +		 * vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > +		 * is the only case currently
> > +		 */
> > +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> 
> You should consider the case when vdpa device is not there. Maybe you can
> use int for vdpa_type, -1 for non-vdpa.
> 
I init vdpa_type to 0;
#define RTE_VHOST_VDPA_DEVICE_TYPE_NET 0
#define RTE_VHOST_VDPA_DEVICE_TYPE_BLK 1
And get_dev_type only return RTE_VHOST_VDPA_DEVICE_TYPE_BLK or RTE_VHOST_VDPA_DEVICE_TYPE_NET.
I think if when vdpa device is not there, this code runs in the original way.
Do you think use init vdpa_type to -1 is better?


> Also note that below check is only needed for some cases.
> 
Yes, I got it. I will fix it in next version.
> Thanks,
> Chenbo
> 
> >  	}
> >
> > +	if (dev->nr_vring < nr_vring)
> > +		return 0;
> > +
> >  	for (i = 0; i < nr_vring; i++) {
> >  		vq = dev->virtqueue[i];
> >
> > @@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	if (unlock_required)
> >  		vhost_user_unlock_all_queue_pairs(dev);
> >
> > -	if (ret != 0 || !virtio_is_ready(dev))
> > +	if (ret != 0)
> > +		goto out;
> > +
> > +	vdpa_dev = dev->vdpa_dev;
> > +	if (vdpa_dev) {
> > +		if (vdpa_dev->ops->get_dev_type) {
> > +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
> &vdpa_type);
> > +			if (ret) {
> > +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> > +					"failed to get vdpa dev type.\n");
> > +				ret = -1;
> > +				goto out;
> > +			}
> > +		} else {
> > +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > +		}
> > +	}
> > +
> > +	if (!virtio_is_ready(dev, vdpa_type))
> >  		goto out;
> >
> >  	/*
> > @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  			dev->flags |= VIRTIO_DEV_RUNNING;
> >  	}
> >
> > -	vdpa_dev = dev->vdpa_dev;
> >  	if (!vdpa_dev)
> >  		goto out;
> >
> > -	if (vdpa_dev->ops->get_dev_type) {
> > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > -		if (ret) {
> > -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to
> get vdpa
> > dev type.\n");
> > -			ret = -1;
> > -			goto out;
> > -		}
> > -	} else {
> > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > -	}
> >  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> >  		&& request != VHOST_USER_SET_VRING_CALL)
> >  		goto out;
> > --
> > 1.8.3.1

RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Wednesday, October 12, 2022 8:13 PM
> To: Xia, Chenbo <chenbo.xia@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> queue is ready
> 
> Hi Chenbo,
> 
> Thanks for your reply.
> My reply is inline.
> 
> > -----Original Message-----
> > From: Xia, Chenbo <chenbo.xia@intel.com>
> > Sent: Wednesday, October 12, 2022 5:09 PM
> > To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> > Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> > Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> > queue is ready
> >
> > > -----Original Message-----
> > > From: Pei, Andy <andy.pei@intel.com>
> > > Sent: Friday, September 16, 2022 2:16 PM
> > > To: dev@dpdk.org
> > > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > > Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> > > queue is ready
> > >
> > > When boot from virtio blk device, seabios in QEMU only enables one
> queue.
> > > To work in this scenario, vDPA BLK device back-end conf_dev when any
> >
> > What is conf_dev?
> >
> I refer to
> 	/** Driver configure the device (Mandatory) */
> 	int (*dev_conf)(int vid);
> So do you think I should use "configure device"?

Yes. It will be better

> 
> > > queue is ready.
> > >
> > > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > > ---
> > >  lib/vhost/vhost_user.c | 51
> > > ++++++++++++++++++++++++++++++++-------------
> > > -----
> > >  1 file changed, 33 insertions(+), 18 deletions(-)
> > >
> > > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > > 4ad28ba..9169cf5 100644
> > > --- a/lib/vhost/vhost_user.c
> > > +++ b/lib/vhost/vhost_user.c
> > > @@ -1449,9 +1449,10 @@
> > >  }
> > >
> > >  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> > > +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> > >
> > >  static int
> > > -virtio_is_ready(struct virtio_net *dev)
> > > +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
> > >  {
> > >  	struct vhost_virtqueue *vq;
> > >  	uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@
> > >  	if (!dev->nr_vring)
> > >  		return 0;
> > >
> > > -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> > > -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > > -
> > > -		if (dev->nr_vring < nr_vring)
> > > -			return 0;
> > > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_NET) {
> > > +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> > > +			nr_vring =
> > VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > > +	} else {
> > > +		/*
> > > +		 * vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > > +		 * is the only case currently
> > > +		 */
> > > +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> >
> > You should consider the case when vdpa device is not there. Maybe you
> can
> > use int for vdpa_type, -1 for non-vdpa.
> >
> I init vdpa_type to 0;
> #define RTE_VHOST_VDPA_DEVICE_TYPE_NET 0
> #define RTE_VHOST_VDPA_DEVICE_TYPE_BLK 1
> And get_dev_type only return RTE_VHOST_VDPA_DEVICE_TYPE_BLK or
> RTE_VHOST_VDPA_DEVICE_TYPE_NET.
> I think if when vdpa device is not there, this code runs in the original
> way.
> Do you think use init vdpa_type to -1 is better?

I was talking about readability, current way will be confusing. So adding
-1 will be better. The check could be if (type == blk) ... else ... as
Type 0/-1 has the same handling.

Thanks,
Chenbo

> 
> 
> > Also note that below check is only needed for some cases.
> >
> Yes, I got it. I will fix it in next version.
> > Thanks,
> > Chenbo
> >
> > >  	}
> > >
> > > +	if (dev->nr_vring < nr_vring)
> > > +		return 0;
> > > +
> > >  	for (i = 0; i < nr_vring; i++) {
> > >  		vq = dev->virtqueue[i];
> > >
> > > @@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net
> *dev,
> > >  	if (unlock_required)
> > >  		vhost_user_unlock_all_queue_pairs(dev);
> > >
> > > -	if (ret != 0 || !virtio_is_ready(dev))
> > > +	if (ret != 0)
> > > +		goto out;
> > > +
> > > +	vdpa_dev = dev->vdpa_dev;
> > > +	if (vdpa_dev) {
> > > +		if (vdpa_dev->ops->get_dev_type) {
> > > +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
> > &vdpa_type);
> > > +			if (ret) {
> > > +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> > > +					"failed to get vdpa dev type.\n");
> > > +				ret = -1;
> > > +				goto out;
> > > +			}
> > > +		} else {
> > > +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > > +		}
> > > +	}
> > > +
> > > +	if (!virtio_is_ready(dev, vdpa_type))
> > >  		goto out;
> > >
> > >  	/*
> > > @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net
> *dev,
> > >  			dev->flags |= VIRTIO_DEV_RUNNING;
> > >  	}
> > >
> > > -	vdpa_dev = dev->vdpa_dev;
> > >  	if (!vdpa_dev)
> > >  		goto out;
> > >
> > > -	if (vdpa_dev->ops->get_dev_type) {
> > > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > > -		if (ret) {
> > > -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to
> > get vdpa
> > > dev type.\n");
> > > -			ret = -1;
> > > -			goto out;
> > > -		}
> > > -	} else {
> > > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > > -	}
> > >  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > >  		&& request != VHOST_USER_SET_VRING_CALL)
> > >  		goto out;
> > > --
> > > 1.8.3.1

RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Pei, Andy]

Hi Chenbo,

Thanks for your reply.
My reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Thursday, October 13, 2022 9:00 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> queue is ready
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Wednesday, October 12, 2022 8:13 PM
> > To: Xia, Chenbo <chenbo.xia@intel.com>; dev@dpdk.org
> > Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> > Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> > queue is ready
> >
> > Hi Chenbo,
> >
> > Thanks for your reply.
> > My reply is inline.
> >
> > > -----Original Message-----
> > > From: Xia, Chenbo <chenbo.xia@intel.com>
> > > Sent: Wednesday, October 12, 2022 5:09 PM
> > > To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> > > Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei
> > > <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> > > maxime.coquelin@redhat.com
> > > Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when
> > > any queue is ready
> > >
> > > > -----Original Message-----
> > > > From: Pei, Andy <andy.pei@intel.com>
> > > > Sent: Friday, September 16, 2022 2:16 PM
> > > > To: dev@dpdk.org
> > > > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > > > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > > > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > > > Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> > > > queue is ready
> > > >
> > > > When boot from virtio blk device, seabios in QEMU only enables one
> > queue.
> > > > To work in this scenario, vDPA BLK device back-end conf_dev when
> > > > any
> > >
> > > What is conf_dev?
> > >
> > I refer to
> > 	/** Driver configure the device (Mandatory) */
> > 	int (*dev_conf)(int vid);
> > So do you think I should use "configure device"?
> 
> Yes. It will be better
> 
OK. I will make it this way in next version.
> >
> > > > queue is ready.
> > > >
> > > > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > > > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > > > ---
> > > >  lib/vhost/vhost_user.c | 51
> > > > ++++++++++++++++++++++++++++++++-------------
> > > > -----
> > > >  1 file changed, 33 insertions(+), 18 deletions(-)
> > > >
> > > > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > > > 4ad28ba..9169cf5 100644
> > > > --- a/lib/vhost/vhost_user.c
> > > > +++ b/lib/vhost/vhost_user.c
> > > > @@ -1449,9 +1449,10 @@
> > > >  }
> > > >
> > > >  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> > > > +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> > > >
> > > >  static int
> > > > -virtio_is_ready(struct virtio_net *dev)
> > > > +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
> > > >  {
> > > >  	struct vhost_virtqueue *vq;
> > > >  	uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@
> > > >  	if (!dev->nr_vring)
> > > >  		return 0;
> > > >
> > > > -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> > > > -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > > > -
> > > > -		if (dev->nr_vring < nr_vring)
> > > > -			return 0;
> > > > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_NET) {
> > > > +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> > > > +			nr_vring =
> > > VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > > > +	} else {
> > > > +		/*
> > > > +		 * vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > > > +		 * is the only case currently
> > > > +		 */
> > > > +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> > >
> > > You should consider the case when vdpa device is not there. Maybe
> > > you
> > can
> > > use int for vdpa_type, -1 for non-vdpa.
> > >
> > I init vdpa_type to 0;
> > #define RTE_VHOST_VDPA_DEVICE_TYPE_NET 0 #define
> > RTE_VHOST_VDPA_DEVICE_TYPE_BLK 1 And get_dev_type only return
> > RTE_VHOST_VDPA_DEVICE_TYPE_BLK or
> RTE_VHOST_VDPA_DEVICE_TYPE_NET.
> > I think if when vdpa device is not there, this code runs in the
> > original way.
> > Do you think use init vdpa_type to -1 is better?
> 
> I was talking about readability, current way will be confusing. So adding
> -1 will be better. The check could be if (type == blk) ... else ... as Type 0/-1 has
> the same handling.
> 
I will add 
#define RTE_VHOST_NO_VDPA_DEVICE       -1
And init vdpa_type to RTE_VHOST_NO_VDPA_DEVICE and follow your suggestion.

> Thanks,
> Chenbo
> 
> >
> >
> > > Also note that below check is only needed for some cases.
> > >
> > Yes, I got it. I will fix it in next version.
> > > Thanks,
> > > Chenbo
> > >
> > > >  	}
> > > >
> > > > +	if (dev->nr_vring < nr_vring)
> > > > +		return 0;
> > > > +
> > > >  	for (i = 0; i < nr_vring; i++) {
> > > >  		vq = dev->virtqueue[i];
> > > >
To make sure queue index is not out of range, I will keep above check for all the cases.

> > > > @@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net
> > *dev,
> > > >  	if (unlock_required)
> > > >  		vhost_user_unlock_all_queue_pairs(dev);
> > > >
> > > > -	if (ret != 0 || !virtio_is_ready(dev))
> > > > +	if (ret != 0)
> > > > +		goto out;
> > > > +
> > > > +	vdpa_dev = dev->vdpa_dev;
> > > > +	if (vdpa_dev) {
> > > > +		if (vdpa_dev->ops->get_dev_type) {
> > > > +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
> > > &vdpa_type);
> > > > +			if (ret) {
> > > > +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> > > > +					"failed to get vdpa dev type.\n");
> > > > +				ret = -1;
> > > > +				goto out;
> > > > +			}
> > > > +		} else {
> > > > +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > > > +		}
> > > > +	}
> > > > +
> > > > +	if (!virtio_is_ready(dev, vdpa_type))
> > > >  		goto out;
> > > >
> > > >  	/*
> > > > @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net
> > *dev,
> > > >  			dev->flags |= VIRTIO_DEV_RUNNING;
> > > >  	}
> > > >
> > > > -	vdpa_dev = dev->vdpa_dev;
> > > >  	if (!vdpa_dev)
> > > >  		goto out;
> > > >
> > > > -	if (vdpa_dev->ops->get_dev_type) {
> > > > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > > > -		if (ret) {
> > > > -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to
> > > get vdpa
> > > > dev type.\n");
> > > > -			ret = -1;
> > > > -			goto out;
> > > > -		}
> > > > -	} else {
> > > > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > > > -	}
> > > >  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > > >  		&& request != VHOST_USER_SET_VRING_CALL)
> > > >  		goto out;
> > > > --
> > > > 1.8.3.1

Re: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Maxime Coquelin]

On 10/13/22 03:00, Xia, Chenbo wrote:
>> -----Original Message-----
>> From: Pei, Andy <andy.pei@intel.com>
>> Sent: Wednesday, October 12, 2022 8:13 PM
>> To: Xia, Chenbo <chenbo.xia@intel.com>; dev@dpdk.org
>> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
>> Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
>> Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
>> queue is ready
>>
>> Hi Chenbo,
>>
>> Thanks for your reply.
>> My reply is inline.
>>
>>> -----Original Message-----
>>> From: Xia, Chenbo <chenbo.xia@intel.com>
>>> Sent: Wednesday, October 12, 2022 5:09 PM
>>> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
>>> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
>>> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
>>> Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
>>> queue is ready
>>>
>>>> -----Original Message-----
>>>> From: Pei, Andy <andy.pei@intel.com>
>>>> Sent: Friday, September 16, 2022 2:16 PM
>>>> To: dev@dpdk.org
>>>> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
>>>> <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
>>>> <gang.cao@intel.com>; maxime.coquelin@redhat.com
>>>> Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
>>>> queue is ready
>>>>
>>>> When boot from virtio blk device, seabios in QEMU only enables one
>> queue.
>>>> To work in this scenario, vDPA BLK device back-end conf_dev when any
>>>
>>> What is conf_dev?
>>>
>> I refer to
>> 	/** Driver configure the device (Mandatory) */
>> 	int (*dev_conf)(int vid);
>> So do you think I should use "configure device"?
> 
> Yes. It will be better
> 
>>
>>>> queue is ready.
>>>>
>>>> Signed-off-by: Andy Pei <andy.pei@intel.com>
>>>> Signed-off-by: Huang Wei <wei.huang@intel.com>
>>>> ---
>>>>   lib/vhost/vhost_user.c | 51
>>>> ++++++++++++++++++++++++++++++++-------------
>>>> -----
>>>>   1 file changed, 33 insertions(+), 18 deletions(-)
>>>>
>>>> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
>>>> 4ad28ba..9169cf5 100644
>>>> --- a/lib/vhost/vhost_user.c
>>>> +++ b/lib/vhost/vhost_user.c
>>>> @@ -1449,9 +1449,10 @@
>>>>   }
>>>>
>>>>   #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
>>>> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
>>>>
>>>>   static int
>>>> -virtio_is_ready(struct virtio_net *dev)
>>>> +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
>>>>   {
>>>>   	struct vhost_virtqueue *vq;
>>>>   	uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@
>>>>   	if (!dev->nr_vring)
>>>>   		return 0;
>>>>
>>>> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
>>>> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>>>> -
>>>> -		if (dev->nr_vring < nr_vring)
>>>> -			return 0;
>>>> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_NET) {
>>>> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
>>>> +			nr_vring =
>>> VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>>>> +	} else {
>>>> +		/*
>>>> +		 * vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
>>>> +		 * is the only case currently
>>>> +		 */
>>>> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
>>>
>>> You should consider the case when vdpa device is not there. Maybe you
>> can
>>> use int for vdpa_type, -1 for non-vdpa.
>>>
>> I init vdpa_type to 0;
>> #define RTE_VHOST_VDPA_DEVICE_TYPE_NET 0
>> #define RTE_VHOST_VDPA_DEVICE_TYPE_BLK 1
>> And get_dev_type only return RTE_VHOST_VDPA_DEVICE_TYPE_BLK or
>> RTE_VHOST_VDPA_DEVICE_TYPE_NET.
>> I think if when vdpa device is not there, this code runs in the original
>> way.
>> Do you think use init vdpa_type to -1 is better?
> 
> I was talking about readability, current way will be confusing. So adding
> -1 will be better. The check could be if (type == blk) ... else ... as
> Type 0/-1 has the same handling.

Also, the vdpa_type can be obtained from dev, so instead of passing the
vdpa_type as argument for the function, it should get fetched directly
in virtio_is_ready().

Maxime

> Thanks,
> Chenbo
> 
>>
>>
>>> Also note that below check is only needed for some cases.
>>>
>> Yes, I got it. I will fix it in next version.
>>> Thanks,
>>> Chenbo
>>>
>>>>   	}
>>>>
>>>> +	if (dev->nr_vring < nr_vring)
>>>> +		return 0;
>>>> +
>>>>   	for (i = 0; i < nr_vring; i++) {
>>>>   		vq = dev->virtqueue[i];
>>>>
>>>> @@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net
>> *dev,
>>>>   	if (unlock_required)
>>>>   		vhost_user_unlock_all_queue_pairs(dev);
>>>>
>>>> -	if (ret != 0 || !virtio_is_ready(dev))
>>>> +	if (ret != 0)
>>>> +		goto out;
>>>> +
>>>> +	vdpa_dev = dev->vdpa_dev;
>>>> +	if (vdpa_dev) {
>>>> +		if (vdpa_dev->ops->get_dev_type) {
>>>> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
>>> &vdpa_type);
>>>> +			if (ret) {
>>>> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
>>>> +					"failed to get vdpa dev type.\n");
>>>> +				ret = -1;
>>>> +				goto out;
>>>> +			}
>>>> +		} else {
>>>> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
>>>> +		}
>>>> +	}
>>>> +
>>>> +	if (!virtio_is_ready(dev, vdpa_type))
>>>>   		goto out;
>>>>
>>>>   	/*
>>>> @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net
>> *dev,
>>>>   			dev->flags |= VIRTIO_DEV_RUNNING;
>>>>   	}
>>>>
>>>> -	vdpa_dev = dev->vdpa_dev;
>>>>   	if (!vdpa_dev)
>>>>   		goto out;
>>>>
>>>> -	if (vdpa_dev->ops->get_dev_type) {
>>>> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
>>>> -		if (ret) {
>>>> -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to
>>> get vdpa
>>>> dev type.\n");
>>>> -			ret = -1;
>>>> -			goto out;
>>>> -		}
>>>> -	} else {
>>>> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
>>>> -	}
>>>>   	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
>>>>   		&& request != VHOST_USER_SET_VRING_CALL)
>>>>   		goto out;
>>>> --
>>>> 1.8.3.1
> 

RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Pei, Andy]

Hi Maxime,

Sorry, I did not notice your email before I send out V4 version.
My reply is inline.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Thursday, October 13, 2022 3:17 PM
> To: Xia, Chenbo <Chenbo.Xia@intel.com>; Pei, Andy <andy.pei@intel.com>;
> dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> queue is ready
> 
> 
> 
> On 10/13/22 03:00, Xia, Chenbo wrote:
> >> -----Original Message-----
> >> From: Pei, Andy <andy.pei@intel.com>
> >> Sent: Wednesday, October 12, 2022 8:13 PM
> >> To: Xia, Chenbo <chenbo.xia@intel.com>; dev@dpdk.org
> >> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> >> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> >> Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when
> >> any queue is ready
> >>
> >> Hi Chenbo,
> >>
> >> Thanks for your reply.
> >> My reply is inline.
> >>
> >>> -----Original Message-----
> >>> From: Xia, Chenbo <chenbo.xia@intel.com>
> >>> Sent: Wednesday, October 12, 2022 5:09 PM
> >>> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> >>> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei
> >>> <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> >>> maxime.coquelin@redhat.com
> >>> Subject: RE: [PATCH v3 7/8] vhost: vDPA blk device gets ready when
> >>> any queue is ready
> >>>
> >>>> -----Original Message-----
> >>>> From: Pei, Andy <andy.pei@intel.com>
> >>>> Sent: Friday, September 16, 2022 2:16 PM
> >>>> To: dev@dpdk.org
> >>>> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> >>>> <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> >>>> <gang.cao@intel.com>; maxime.coquelin@redhat.com
> >>>> Subject: [PATCH v3 7/8] vhost: vDPA blk device gets ready when any
> >>>> queue is ready
> >>>>
> >>>> When boot from virtio blk device, seabios in QEMU only enables one
> >> queue.
> >>>> To work in this scenario, vDPA BLK device back-end conf_dev when
> >>>> any
> >>>
> >>> What is conf_dev?
> >>>
> >> I refer to
> >> 	/** Driver configure the device (Mandatory) */
> >> 	int (*dev_conf)(int vid);
> >> So do you think I should use "configure device"?
> >
> > Yes. It will be better
> >
> >>
> >>>> queue is ready.
> >>>>
> >>>> Signed-off-by: Andy Pei <andy.pei@intel.com>
> >>>> Signed-off-by: Huang Wei <wei.huang@intel.com>
> >>>> ---
> >>>>   lib/vhost/vhost_user.c | 51
> >>>> ++++++++++++++++++++++++++++++++-------------
> >>>> -----
> >>>>   1 file changed, 33 insertions(+), 18 deletions(-)
> >>>>
> >>>> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> >>>> 4ad28ba..9169cf5 100644
> >>>> --- a/lib/vhost/vhost_user.c
> >>>> +++ b/lib/vhost/vhost_user.c
> >>>> @@ -1449,9 +1449,10 @@
> >>>>   }
> >>>>
> >>>>   #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> >>>> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> >>>>
> >>>>   static int
> >>>> -virtio_is_ready(struct virtio_net *dev)
> >>>> +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
> >>>>   {
> >>>>   	struct vhost_virtqueue *vq;
> >>>>   	uint32_t i, nr_vring = dev->nr_vring; @@ -1462,13 +1463,20 @@
> >>>>   	if (!dev->nr_vring)
> >>>>   		return 0;
> >>>>
> >>>> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> >>>> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> >>>> -
> >>>> -		if (dev->nr_vring < nr_vring)
> >>>> -			return 0;
> >>>> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_NET) {
> >>>> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> >>>> +			nr_vring =
> >>> VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> >>>> +	} else {
> >>>> +		/*
> >>>> +		 * vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> >>>> +		 * is the only case currently
> >>>> +		 */
> >>>> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> >>>
> >>> You should consider the case when vdpa device is not there. Maybe
> >>> you
> >> can
> >>> use int for vdpa_type, -1 for non-vdpa.
> >>>
> >> I init vdpa_type to 0;
> >> #define RTE_VHOST_VDPA_DEVICE_TYPE_NET 0 #define
> >> RTE_VHOST_VDPA_DEVICE_TYPE_BLK 1 And get_dev_type only return
> >> RTE_VHOST_VDPA_DEVICE_TYPE_BLK or
> RTE_VHOST_VDPA_DEVICE_TYPE_NET.
> >> I think if when vdpa device is not there, this code runs in the
> >> original way.
> >> Do you think use init vdpa_type to -1 is better?
> >
> > I was talking about readability, current way will be confusing. So
> > adding
> > -1 will be better. The check could be if (type == blk) ... else ... as
> > Type 0/-1 has the same handling.
> 
> Also, the vdpa_type can be obtained from dev, so instead of passing the
> vdpa_type as argument for the function, it should get fetched directly in
> virtio_is_ready().
> 
As you mention, vdpa_type can be fetch directly in virtio_is_ready().
virtio_is_ready() is an internal function, and virtio_is_ready() is used only once.
vdpa_type is also used vhost_user_msg_handler(), just need virtio_is_ready().
I just think I can reduce one function call, at the cost of passing a parameter to 
virtio_is_ready().

I do not know which way is better, what is your opinion?

> Maxime
> 
> > Thanks,
> > Chenbo
> >
> >>
> >>
> >>> Also note that below check is only needed for some cases.
> >>>
> >> Yes, I got it. I will fix it in next version.
> >>> Thanks,
> >>> Chenbo
> >>>
> >>>>   	}
> >>>>
> >>>> +	if (dev->nr_vring < nr_vring)
> >>>> +		return 0;
> >>>> +
> >>>>   	for (i = 0; i < nr_vring; i++) {
> >>>>   		vq = dev->virtqueue[i];
> >>>>
> >>>> @@ -3167,7 +3175,25 @@ static int is_vring_iotlb(struct virtio_net
> >> *dev,
> >>>>   	if (unlock_required)
> >>>>   		vhost_user_unlock_all_queue_pairs(dev);
> >>>>
> >>>> -	if (ret != 0 || !virtio_is_ready(dev))
> >>>> +	if (ret != 0)
> >>>> +		goto out;
> >>>> +
> >>>> +	vdpa_dev = dev->vdpa_dev;
> >>>> +	if (vdpa_dev) {
> >>>> +		if (vdpa_dev->ops->get_dev_type) {
> >>>> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
> >>> &vdpa_type);
> >>>> +			if (ret) {
> >>>> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> >>>> +					"failed to get vdpa dev type.\n");
> >>>> +				ret = -1;
> >>>> +				goto out;
> >>>> +			}
> >>>> +		} else {
> >>>> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> >>>> +		}
> >>>> +	}
> >>>> +
> >>>> +	if (!virtio_is_ready(dev, vdpa_type))
> >>>>   		goto out;
> >>>>
> >>>>   	/*
> >>>> @@ -3181,20 +3207,9 @@ static int is_vring_iotlb(struct virtio_net
> >> *dev,
> >>>>   			dev->flags |= VIRTIO_DEV_RUNNING;
> >>>>   	}
> >>>>
> >>>> -	vdpa_dev = dev->vdpa_dev;
> >>>>   	if (!vdpa_dev)
> >>>>   		goto out;
> >>>>
> >>>> -	if (vdpa_dev->ops->get_dev_type) {
> >>>> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> >>>> -		if (ret) {
> >>>> -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to
> >>> get vdpa
> >>>> dev type.\n");
> >>>> -			ret = -1;
> >>>> -			goto out;
> >>>> -		}
> >>>> -	} else {
> >>>> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> >>>> -	}
> >>>>   	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> >>>>   		&& request != VHOST_USER_SET_VRING_CALL)
> >>>>   		goto out;
> >>>> --
> >>>> 1.8.3.1
> >

[PATCH v3 8/8] vhost: improve vDPA blk device readiness condition

[Andy Pei]

In the virtio blk vDPA live migration use case, for the target VM,
before the live migration process, QEMU will set call fd to all
queues of vDPA back-end. QEMU and vDPA back-end stand by
until live migration starts. During live migration process,
QEMU sets kick fd and new call fd. However, after the kick fd
is set to the vDPA back-end, the vDPA back-end configures device
and data path starts. The new call fd will cause some kind of
"re-configuration", this kind of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd
and call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact
net device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index 9169cf5..14ff266 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	uint32_t vdpa_type = 0;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Friday, September 16, 2022 2:17 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition
> 
> In the virtio blk vDPA live migration use case, for the target VM,
> before the live migration process, QEMU will set call fd to all
> queues of vDPA back-end. QEMU and vDPA back-end stand by
> until live migration starts. During live migration process,
> QEMU sets kick fd and new call fd. However, after the kick fd
> is set to the vDPA back-end, the vDPA back-end configures device
> and data path starts. The new call fd will cause some kind of
> "re-configuration", this kind of "re-configuration" cause IO drop.
> After this patch, vDPA back-end configures device after kick fd
> and call fd are well set and make sure no IO drops.
> This patch only impact virtio blk vDPA device and does not impact
> net device.

IIUC, this is an improvement for MQ to make sure all call fds are well-set,
but previously it only makes sure one is well-set. If I am right, the title
and commit message should describe it in better way.

Thanks,
Chenbo

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 13 ++++++++++---
>  1 file changed, 10 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index 9169cf5..14ff266 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	uint32_t vdpa_type = 0;
>  	uint32_t request;
>  	uint32_t i;
> +	uint16_t blk_call_fd;
> 
>  	dev = get_device(vid);
>  	if (dev == NULL)
> @@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	if (!vdpa_dev)
>  		goto out;
> 
> -	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> -		&& request != VHOST_USER_SET_VRING_CALL)
> -		goto out;
> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +		if (request == VHOST_USER_SET_VRING_CALL) {
> +			blk_call_fd = ctx.msg.payload.u64 &
> VHOST_USER_VRING_IDX_MASK;
> +			if (blk_call_fd != dev->nr_vring - 1)
> +				goto out;
> +		} else {
> +			goto out;
> +		}
> +	}
> 
>  	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
>  		if (vdpa_dev->ops->dev_conf(dev->vid))
> --
> 1.8.3.1

RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition

[Pei, Andy]

Hi Chenbo,

Thanks for your reply, my reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 12, 2022 5:36 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness
> condition
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Friday, September 16, 2022 2:17 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v3 8/8] vhost: improve vDPA blk device readiness
> > condition
> >
> > In the virtio blk vDPA live migration use case, for the target VM,
> > before the live migration process, QEMU will set call fd to all queues
> > of vDPA back-end. QEMU and vDPA back-end stand by until live migration
> > starts. During live migration process, QEMU sets kick fd and new call
> > fd. However, after the kick fd is set to the vDPA back-end, the vDPA
> > back-end configures device and data path starts. The new call fd will
> > cause some kind of "re-configuration", this kind of "re-configuration"
> > cause IO drop.
> > After this patch, vDPA back-end configures device after kick fd and
> > call fd are well set and make sure no IO drops.
> > This patch only impact virtio blk vDPA device and does not impact net
> > device.
> 
> IIUC, this is an improvement for MQ to make sure all call fds are well-set, but
> previously it only makes sure one is well-set. If I am right, the title and
> commit message should describe it in better way.
> 
> Thanks,
> Chenbo
> 
In the virtio blk vDPA live migration use case, for the target VM, 
before the live migration process, QEMU will set call fd to all queues of vDPA back-end.
QEMU and vDPA back-end stand by until live migration starts.
During live migration process, QEMU sets kick fd to all queues one by one,
and new call fd all queues one by one.
However, after the kick fd is set to the vDPA back-end,
with the original call fd, the queue is ready.
Then the vDPA  back-end configures device and data path starts.
The new call fd will cause some kind of "re-configuration",
this kind of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd and 
call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact net device.

Is this new commit message good?
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  lib/vhost/vhost_user.c | 13 ++++++++++---
> >  1 file changed, 10 insertions(+), 3 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > 9169cf5..14ff266 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	uint32_t vdpa_type = 0;
> >  	uint32_t request;
> >  	uint32_t i;
> > +	uint16_t blk_call_fd;
> >
> >  	dev = get_device(vid);
> >  	if (dev == NULL)
> > @@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	if (!vdpa_dev)
> >  		goto out;
> >
> > -	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > -		&& request != VHOST_USER_SET_VRING_CALL)
> > -		goto out;
> > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > +		if (request == VHOST_USER_SET_VRING_CALL) {
> > +			blk_call_fd = ctx.msg.payload.u64 &
> > VHOST_USER_VRING_IDX_MASK;
> > +			if (blk_call_fd != dev->nr_vring - 1)
> > +				goto out;
> > +		} else {
> > +			goto out;
> > +		}
> > +	}
> >
> >  	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
> >  		if (vdpa_dev->ops->dev_conf(dev->vid))
> > --
> > 1.8.3.1

RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness condition

[Pei, Andy]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 3:56 PM
> To: Xia, Chenbo <Chenbo.Xia@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness
> condition
> 
> Hi Chenbo,
> 
> Thanks for your reply, my reply is inline.
> 
> > -----Original Message-----
> > From: Xia, Chenbo <chenbo.xia@intel.com>
> > Sent: Wednesday, October 12, 2022 5:36 PM
> > To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> > Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> > Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: RE: [PATCH v3 8/8] vhost: improve vDPA blk device readiness
> > condition
> >
> > > -----Original Message-----
> > > From: Pei, Andy <andy.pei@intel.com>
> > > Sent: Friday, September 16, 2022 2:17 PM
> > > To: dev@dpdk.org
> > > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > > Subject: [PATCH v3 8/8] vhost: improve vDPA blk device readiness
> > > condition
> > >
> > > In the virtio blk vDPA live migration use case, for the target VM,
> > > before the live migration process, QEMU will set call fd to all
> > > queues of vDPA back-end. QEMU and vDPA back-end stand by until live
> > > migration starts. During live migration process, QEMU sets kick fd
> > > and new call fd. However, after the kick fd is set to the vDPA
> > > back-end, the vDPA back-end configures device and data path starts.
> > > The new call fd will cause some kind of "re-configuration", this kind of
> "re-configuration"
> > > cause IO drop.
> > > After this patch, vDPA back-end configures device after kick fd and
> > > call fd are well set and make sure no IO drops.
> > > This patch only impact virtio blk vDPA device and does not impact
> > > net device.
> >
> > IIUC, this is an improvement for MQ to make sure all call fds are
> > well-set, but previously it only makes sure one is well-set. If I am
> > right, the title and commit message should describe it in better way.
> >
> > Thanks,
> > Chenbo
> >
> In the virtio blk vDPA live migration use case, for the target VM, before the
> live migration process, QEMU will set call fd to all queues of vDPA back-end.
> QEMU and vDPA back-end stand by until live migration starts.
> During live migration process, QEMU sets kick fd to all queues one by one,
> and new call fd all queues one by one.
> However, after the kick fd is set to the vDPA back-end, with the original call fd,
> the queue is ready.
> Then the vDPA  back-end configures device and data path starts.
> The new call fd will cause some kind of "re-configuration", this kind of "re-
> configuration" cause IO drop.
> After this patch, vDPA back-end configures device after kick fd and call fd are
> well set and make sure no IO drops.
> This patch only impact virtio blk vDPA device and does not impact net device.
> 
> Is this new commit message good?

Explanation seems duplicate with former commit
" vhost: fix virtio block vDPA live migration IO drop".
So I will make it simple here,
"To support multi-queue, configure device after call fd of all queues are set."

I will use " vhost: improve vDPA blk device configure  condition" for title 
To reduce confusion.


> > >
> > > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > > ---
> > >  lib/vhost/vhost_user.c | 13 ++++++++++---
> > >  1 file changed, 10 insertions(+), 3 deletions(-)
> > >
> > > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > > 9169cf5..14ff266 100644
> > > --- a/lib/vhost/vhost_user.c
> > > +++ b/lib/vhost/vhost_user.c
> > > @@ -2983,6 +2983,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
> > >  	uint32_t vdpa_type = 0;
> > >  	uint32_t request;
> > >  	uint32_t i;
> > > +	uint16_t blk_call_fd;
> > >
> > >  	dev = get_device(vid);
> > >  	if (dev == NULL)
> > > @@ -3210,9 +3211,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
> > >  	if (!vdpa_dev)
> > >  		goto out;
> > >
> > > -	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > > -		&& request != VHOST_USER_SET_VRING_CALL)
> > > -		goto out;
> > > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > > +		if (request == VHOST_USER_SET_VRING_CALL) {
> > > +			blk_call_fd = ctx.msg.payload.u64 &
> > > VHOST_USER_VRING_IDX_MASK;
> > > +			if (blk_call_fd != dev->nr_vring - 1)
> > > +				goto out;
> > > +		} else {
> > > +			goto out;
> > > +		}
> > > +	}
> > >
> > >  	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
> > >  		if (vdpa_dev->ops->dev_conf(dev->vid))
> > > --
> > > 1.8.3.1

[PATCH v4 0/8] vdpa/ifc: add multi queue support

[Andy Pei]

v4:
 fix some commit message.
 add some commets to code.
 fix some code to reduce confusion.

v3:
 rename device ID macro name.
 fix some patch title and commit message.
 delete some used marco.
 rework some code logic.

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (6):
  vdpa/ifc: add multi-queue support
  vdpa/ifc: set max queues based on virtio spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vhost: vDPA blk device gets ready when any queue is ready
  vhost: improve vDPA blk device readiness condition

Huang Wei (2):
  vdpa/ifc: add new device ID for legacy network device
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  16 ++++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++------
 lib/vhost/vhost_user.c        |  56 ++++++++++------
 4 files changed, 315 insertions(+), 43 deletions(-)

-- 
1.8.3.1

[PATCH v4 1/8] vdpa/ifc: add new device ID for legacy network device

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..ef7697a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,12 +12,14 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
-#define IFCVF_BLK_DEVICE_ID                 0x0002
+#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
+#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
 #define IFCVF_MAX_QUEUES		1
 
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index d5ac583..b4389a0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .vendor_id = 0, /* sentinel */
-- 
1.8.3.1

[PATCH v4 2/8] vdpa/ifc: add multi-queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
Expose IFCVF_MQ_OFFSET register to enable multi-queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 12 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..81c68c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,15 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	/* For some hardware implementation, for example:
+	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
+	 * This code makes sure hw->mq_cfg is a valid address.
+	 */
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ef7697a..d16d9ab 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index b4389a0..008cf89 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

RE: [PATCH v4 2/8] vdpa/ifc: add multi-queue support

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 4:44 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v4 2/8] vdpa/ifc: add multi-queue support
> 
> Enable VHOST_USER_PROTOCOL_F_MQ feature.
> Expose IFCVF_MQ_OFFSET register to enable multi-queue.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
>  drivers/vdpa/ifc/base/ifcvf.h | 2 ++
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
>  3 files changed, 12 insertions(+)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index f1e1474..81c68c0 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -90,6 +90,15 @@
>  	if (!hw->lm_cfg)
>  		WARNINGOUT("HW support live migration not support!\n");
> 
> +	/* For some hardware implementation, for example:
> +	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
> +	 * This code makes sure hw->mq_cfg is a valid address.
> +	 */
> +	if (hw->mem_resource[4].addr)
> +		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
> +	else
> +		hw->mq_cfg = NULL;
> +
>  	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
>  			hw->isr == NULL || hw->dev_cfg == NULL) {
>  		DEBUGOUT("capability incomplete\n");
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index ef7697a..d16d9ab 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -50,6 +50,7 @@
> 
>  #define IFCVF_LM_CFG_SIZE		0x40
>  #define IFCVF_LM_RING_STATE_OFFSET	0x20
> +#define IFCVF_MQ_OFFSET			0x28
> 
>  #define IFCVF_LM_LOGGING_CTRL		0x0
> 
> @@ -149,6 +150,7 @@ struct ifcvf_hw {
>  	u16    *notify_base;
>  	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
>  	u8     *lm_cfg;
> +	u8     *mq_cfg;
>  	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
>  	u8 nr_vring;
>  	int device_type;
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index b4389a0..008cf89 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
>  		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
> +		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
>  		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
> 
>  #define VDPA_BLK_PROTOCOL_FEATURES \
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v4 2/8] vdpa/ifc: add multi-queue support

[Pei, Andy]

Hi Chenbo,

Thanks for your effort.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Monday, October 17, 2022 2:21 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v4 2/8] vdpa/ifc: add multi-queue support
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, October 13, 2022 4:44 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v4 2/8] vdpa/ifc: add multi-queue support
> >
> > Enable VHOST_USER_PROTOCOL_F_MQ feature.
> > Expose IFCVF_MQ_OFFSET register to enable multi-queue.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
> > drivers/vdpa/ifc/base/ifcvf.h | 2 ++  drivers/vdpa/ifc/ifcvf_vdpa.c |
> > 1 +
> >  3 files changed, 12 insertions(+)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index f1e1474..81c68c0 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -90,6 +90,15 @@
> >  	if (!hw->lm_cfg)
> >  		WARNINGOUT("HW support live migration not support!\n");
> >
> > +	/* For some hardware implementation, for example:
> > +	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
> > +	 * This code makes sure hw->mq_cfg is a valid address.
> > +	 */
> > +	if (hw->mem_resource[4].addr)
> > +		hw->mq_cfg = hw->mem_resource[4].addr +
> IFCVF_MQ_OFFSET;
> > +	else
> > +		hw->mq_cfg = NULL;
> > +
> >  	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
> >  			hw->isr == NULL || hw->dev_cfg == NULL) {
> >  		DEBUGOUT("capability incomplete\n"); diff --git
> > a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h index
> > ef7697a..d16d9ab 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -50,6 +50,7 @@
> >
> >  #define IFCVF_LM_CFG_SIZE		0x40
> >  #define IFCVF_LM_RING_STATE_OFFSET	0x20
> > +#define IFCVF_MQ_OFFSET			0x28
> >
> >  #define IFCVF_LM_LOGGING_CTRL		0x0
> >
> > @@ -149,6 +150,7 @@ struct ifcvf_hw {
> >  	u16    *notify_base;
> >  	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
> >  	u8     *lm_cfg;
> > +	u8     *mq_cfg;
> >  	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
> >  	u8 nr_vring;
> >  	int device_type;
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index b4389a0..008cf89 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
> >  		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
> > +		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
> >  		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
> >
> >  #define VDPA_BLK_PROTOCOL_FEATURES \
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec

[Andy Pei]

Set max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues
in struct virtio_blk_config
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg,
get the queue pair number using num_queues and set max_queues to it.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index d16d9ab..1e133c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
 #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 008cf89..5a24204 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,12 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MIN() cannot be used since braced-group within expression allowed
+ * only inside a function.
+ */
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		/*
+		 * ifc device always has CTRL_VQ,
+		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
+		 */
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

RE: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 4:44 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec
> 
> Set max_queues according to virtio spec.
> For virtio BLK device, set max_queues to the value of num_queues
> in struct virtio_blk_config

Missing '.'

With this fixed:

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

> For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg,
> get the queue pair number using num_queues and set max_queues to it.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.h |  2 +-
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
>  2 files changed, 19 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
> index d16d9ab..1e133c0 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.h
> +++ b/drivers/vdpa/ifc/base/ifcvf.h
> @@ -21,7 +21,7 @@
>  #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
>  #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> 
> -#define IFCVF_MAX_QUEUES		1
> +#define IFCVF_MAX_QUEUES		32
> 
>  #ifndef VIRTIO_F_IOMMU_PLATFORM
>  #define VIRTIO_F_IOMMU_PLATFORM		33
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 008cf89..5a24204 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -26,6 +26,12 @@
> 
>  #include "base/ifcvf.h"
> 
> +/*
> + * RTE_MIN() cannot be used since braced-group within expression allowed
> + * only inside a function.
> + */
> +#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
> +
>  RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
>  #define DRV_LOG(level, fmt, args...) \
>  	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
> @@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	uint64_t capacity = 0;
>  	uint8_t *byte;
>  	uint32_t i;
> +	uint16_t queue_pairs;
> 
>  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
>  		return 0;
> @@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	}
> 
>  	internal->configured = 0;
> -	internal->max_queues = IFCVF_MAX_QUEUES;
>  	features = ifcvf_get_features(&internal->hw);
> 
>  	device_id = ifcvf_pci_get_device_type(pci_dev);
> @@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
> 
>  	if (device_id == VIRTIO_ID_NET) {
>  		internal->hw.device_type = IFCVF_NET;
> +		/*
> +		 * ifc device always has CTRL_VQ,
> +		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
> +		 */
> +		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
> +		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
> +			queue_pairs);
> +		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
>  		internal->features = features &
>  					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
>  		internal->features |= dev_info[IFCVF_NET].features;
> @@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
>  			internal->hw.blk_cfg->geometry.sectors);
>  		DRV_LOG(DEBUG, "num_queues: 0x%08x",
>  			internal->hw.blk_cfg->num_queues);
> +
> +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> +			internal->hw.blk_cfg->num_queues);
>  	}
> 
>  	list->internal = internal;
> --
> 1.8.3.1

RE: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec

[Pei, Andy]

Hi Chenbo,

Thanks for your effort.


> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Monday, October 17, 2022 2:22 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, October 13, 2022 4:44 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v4 3/8] vdpa/ifc: set max queues based on virtio spec
> >
> > Set max_queues according to virtio spec.
> > For virtio BLK device, set max_queues to the value of num_queues in
> > struct virtio_blk_config
> 
> Missing '.'
> 
I will fix in next version.

> With this fixed:
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
> 
> > For virtio NET device, read num_queues from struct
> > ifcvf_pci_common_cfg, get the queue pair number using num_queues and
> set max_queues to it.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.h |  2 +-  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 19 ++++++++++++++++++-
> >  2 files changed, 19 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.h
> > b/drivers/vdpa/ifc/base/ifcvf.h index d16d9ab..1e133c0 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.h
> > +++ b/drivers/vdpa/ifc/base/ifcvf.h
> > @@ -21,7 +21,7 @@
> >  #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
> >  #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
> >
> > -#define IFCVF_MAX_QUEUES		1
> > +#define IFCVF_MAX_QUEUES		32
> >
> >  #ifndef VIRTIO_F_IOMMU_PLATFORM
> >  #define VIRTIO_F_IOMMU_PLATFORM		33
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 008cf89..5a24204 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -26,6 +26,12 @@
> >
> >  #include "base/ifcvf.h"
> >
> > +/*
> > + * RTE_MIN() cannot be used since braced-group within expression
> > +allowed
> > + * only inside a function.
> > + */
> > +#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
> > +
> >  RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
> > #define DRV_LOG(level, fmt, args...) \
> >  	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \ @@ -1512,6
> +1518,7
> > @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	uint64_t capacity = 0;
> >  	uint8_t *byte;
> >  	uint32_t i;
> > +	uint16_t queue_pairs;
> >
> >  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> >  		return 0;
> > @@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	}
> >
> >  	internal->configured = 0;
> > -	internal->max_queues = IFCVF_MAX_QUEUES;
> >  	features = ifcvf_get_features(&internal->hw);
> >
> >  	device_id = ifcvf_pci_get_device_type(pci_dev);
> > @@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
> >
> >  	if (device_id == VIRTIO_ID_NET) {
> >  		internal->hw.device_type = IFCVF_NET;
> > +		/*
> > +		 * ifc device always has CTRL_VQ,
> > +		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
> > +		 */
> > +		queue_pairs = (internal->hw.common_cfg->num_queues - 1)
> / 2;
> > +		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev-
> >name,
> > +			queue_pairs);
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> queue_pairs);
> >  		internal->features = features &
> >  					~(1ULL <<
> VIRTIO_F_IOMMU_PLATFORM);
> >  		internal->features |= dev_info[IFCVF_NET].features; @@ -
> 1609,6
> > +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  			internal->hw.blk_cfg->geometry.sectors);
> >  		DRV_LOG(DEBUG, "num_queues: 0x%08x",
> >  			internal->hw.blk_cfg->num_queues);
> > +
> > +		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
> > +			internal->hw.blk_cfg->num_queues);
> >  	}
> >
> >  	list->internal = internal;
> > --
> > 1.8.3.1

[PATCH v4 4/8] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi-queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 81c68c0..60c7017 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -202,6 +202,37 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_mq(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	u8 qid;
+	int nr_queue = 0;
+
+	for (qid = 0; qid < hw->nr_vring; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
+		nr_queue++;
+	}
+
+	if (nr_queue == 0) {
+		WARNINGOUT("no enabled vring\n");
+		return;
+	}
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			*(u32 *)mq_cfg = nr_queue;
+			RTE_LOG(INFO, PMD, "%d queue are enabled\n", nr_queue);
+		} else {
+			*(u32 *)mq_cfg = nr_queue / 2;
+			RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+				nr_queue / 2);
+		}
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -219,6 +250,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

RE: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ register

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 4:45 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ register
> 
> Write queue count to IFCVF_MQ_OFFSET register
> to enable multi-queue feature.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
>  1 file changed, 32 insertions(+)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 81c68c0..60c7017 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -202,6 +202,37 @@
>  	IFCVF_WRITE_REG32(val >> 32, hi);
>  }
> 
> +STATIC void
> +ifcvf_enable_mq(struct ifcvf_hw *hw)
> +{
> +	u8 *mq_cfg;
> +	u8 qid;
> +	int nr_queue = 0;
> +
> +	for (qid = 0; qid < hw->nr_vring; qid++) {
> +		if (!hw->vring[qid].enable)
> +			continue;
> +		nr_queue++;
> +	}
> +
> +	if (nr_queue == 0) {
> +		WARNINGOUT("no enabled vring\n");
> +		return;
> +	}
> +
> +	mq_cfg = hw->mq_cfg;
> +	if (mq_cfg) {
> +		if (hw->device_type == IFCVF_BLK) {
> +			*(u32 *)mq_cfg = nr_queue;
> +			RTE_LOG(INFO, PMD, "%d queue are enabled\n", nr_queue);

queue -> queues

With this fixed:

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

> +		} else {
> +			*(u32 *)mq_cfg = nr_queue / 2;
> +			RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
> +				nr_queue / 2);
> +		}
> +	}
> +}
> +
>  STATIC int
>  ifcvf_hw_enable(struct ifcvf_hw *hw)
>  {
> @@ -219,6 +250,7 @@
>  		return -1;
>  	}
> 
> +	ifcvf_enable_mq(hw);
>  	for (i = 0; i < hw->nr_vring; i++) {
>  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
>  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> --
> 1.8.3.1

RE: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ register

[Pei, Andy]

Hi Chenbo,

Thanks for your effort.
I will send a new version to fix this.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Monday, October 17, 2022 2:24 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ register
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, October 13, 2022 4:45 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v4 4/8] vdpa/ifc: write queue count to MQ register
> >
> > Write queue count to IFCVF_MQ_OFFSET register to enable multi-queue
> > feature.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
> >  1 file changed, 32 insertions(+)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 81c68c0..60c7017 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -202,6 +202,37 @@
> >  	IFCVF_WRITE_REG32(val >> 32, hi);
> >  }
> >
> > +STATIC void
> > +ifcvf_enable_mq(struct ifcvf_hw *hw)
> > +{
> > +	u8 *mq_cfg;
> > +	u8 qid;
> > +	int nr_queue = 0;
> > +
> > +	for (qid = 0; qid < hw->nr_vring; qid++) {
> > +		if (!hw->vring[qid].enable)
> > +			continue;
> > +		nr_queue++;
> > +	}
> > +
> > +	if (nr_queue == 0) {
> > +		WARNINGOUT("no enabled vring\n");
> > +		return;
> > +	}
> > +
> > +	mq_cfg = hw->mq_cfg;
> > +	if (mq_cfg) {
> > +		if (hw->device_type == IFCVF_BLK) {
> > +			*(u32 *)mq_cfg = nr_queue;
> > +			RTE_LOG(INFO, PMD, "%d queue are enabled\n",
> nr_queue);
> 
> queue -> queues
> 
> With this fixed:
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
> 
> > +		} else {
> > +			*(u32 *)mq_cfg = nr_queue / 2;
> > +			RTE_LOG(INFO, PMD, "%d queue pairs are
> enabled\n",
> > +				nr_queue / 2);
> > +		}
> > +	}
> > +}
> > +
> >  STATIC int
> >  ifcvf_hw_enable(struct ifcvf_hw *hw)
> >  {
> > @@ -219,6 +250,7 @@
> >  		return -1;
> >  	}
> >
> > +	ifcvf_enable_mq(hw);
> >  	for (i = 0; i < hw->nr_vring; i++) {
> >  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
> >  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> > --
> > 1.8.3.1

[PATCH v4 5/8] vdpa/ifc: only configure enabled queue

[Andy Pei]

When configuring the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  3 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 60c7017..4d85911 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -252,6 +252,9 @@
 
 	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 5a24204..0c3407a 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

RE: [PATCH v4 5/8] vdpa/ifc: only configure enabled queue

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 4:45 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v4 5/8] vdpa/ifc: only configure enabled queue
> 
> When configuring the hardware queue, we only configure queues which
> have been enabled by vhost.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c |  3 +++
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
>  2 files changed, 17 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
> index 60c7017..4d85911 100644
> --- a/drivers/vdpa/ifc/base/ifcvf.c
> +++ b/drivers/vdpa/ifc/base/ifcvf.c
> @@ -252,6 +252,9 @@
> 
>  	ifcvf_enable_mq(hw);
>  	for (i = 0; i < hw->nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
> +
>  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
>  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
>  				&cfg->queue_desc_hi);
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 5a24204..0c3407a 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
>  	rte_vhost_get_negotiated_features(vid, &hw->req_features);
> 
>  	for (i = 0; i < nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
>  		rte_vhost_get_vhost_vring(vid, i, &vq);
>  		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
>  		if (gpa == 0) {
> @@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
> 
>  	vring.kickfd = -1;
>  	for (qid = 0; qid < q_num; qid++) {
> +		if (!hw->vring[qid].enable)
> +			continue;
>  		ev.events = EPOLLIN | EPOLLPRI;
>  		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
>  		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
> @@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
>  	struct rte_vdpa_device *vdev;
>  	struct internal_list *list;
>  	struct ifcvf_internal *internal;
> +	struct ifcvf_hw *hw;
> +	uint16_t i;
> 
>  	vdev = rte_vhost_get_vdpa_device(vid);
>  	list = find_internal_resource_by_vdev(vdev);
> @@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
>  	rte_atomic32_set(&internal->dev_attached, 1);
>  	update_datapath(internal);
> 
> -	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) !=
> 0)
> -		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
> +	hw = &internal->hw;
> +	for (i = 0; i < hw->nr_vring; i++) {
> +		if (!hw->vring[i].enable)
> +			continue;
> +		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
> +			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
>  				vdev->device->name);
> +	}
> 
>  	internal->configured = 1;
> +	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
>  	return 0;
>  }
> 
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v4 5/8] vdpa/ifc: only configure enabled queue

[Pei, Andy]

Hi  Chenbo,
Thanks for your efforts.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Monday, October 17, 2022 2:24 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v4 5/8] vdpa/ifc: only configure enabled queue
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, October 13, 2022 4:45 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v4 5/8] vdpa/ifc: only configure enabled queue
> >
> > When configuring the hardware queue, we only configure queues which
> > have been enabled by vhost.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c |  3 +++  drivers/vdpa/ifc/ifcvf_vdpa.c
> > | 16 ++++++++++++++--
> >  2 files changed, 17 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/base/ifcvf.c
> > b/drivers/vdpa/ifc/base/ifcvf.c index 60c7017..4d85911 100644
> > --- a/drivers/vdpa/ifc/base/ifcvf.c
> > +++ b/drivers/vdpa/ifc/base/ifcvf.c
> > @@ -252,6 +252,9 @@
> >
> >  	ifcvf_enable_mq(hw);
> >  	for (i = 0; i < hw->nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> > +
> >  		IFCVF_WRITE_REG16(i, &cfg->queue_select);
> >  		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
> >  				&cfg->queue_desc_hi);
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 5a24204..0c3407a 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
> >  	rte_vhost_get_negotiated_features(vid, &hw->req_features);
> >
> >  	for (i = 0; i < nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> >  		rte_vhost_get_vhost_vring(vid, i, &vq);
> >  		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
> >  		if (gpa == 0) {
> > @@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
> >
> >  	vring.kickfd = -1;
> >  	for (qid = 0; qid < q_num; qid++) {
> > +		if (!hw->vring[qid].enable)
> > +			continue;
> >  		ev.events = EPOLLIN | EPOLLPRI;
> >  		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
> >  		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32; @@ -1058,6
> > +1062,8 @@ struct rte_vdpa_dev_info {
> >  	struct rte_vdpa_device *vdev;
> >  	struct internal_list *list;
> >  	struct ifcvf_internal *internal;
> > +	struct ifcvf_hw *hw;
> > +	uint16_t i;
> >
> >  	vdev = rte_vhost_get_vdpa_device(vid);
> >  	list = find_internal_resource_by_vdev(vdev);
> > @@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
> >  	rte_atomic32_set(&internal->dev_attached, 1);
> >  	update_datapath(internal);
> >
> > -	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) !=
> > 0)
> > -		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
> > +	hw = &internal->hw;
> > +	for (i = 0; i < hw->nr_vring; i++) {
> > +		if (!hw->vring[i].enable)
> > +			continue;
> > +		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
> > +			DRV_LOG(NOTICE, "vDPA (%s): software relay is
> used.",
> >  				vdev->device->name);
> > +	}
> >
> >  	internal->configured = 1;
> > +	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device-
> >name);
> >  	return 0;
> >  }
> >
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |   6 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c |  93 ++++++++++++++++++++++++++++++++-------
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 4d85911..0ecd58b 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -233,6 +233,106 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (i >= (int)hw->nr_vring)
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_mq(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			RTE_LOG(ERR, PMD, "queue %d, msix vec alloc failed\n",
+				i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (i >= (int)hw->nr_vring)
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		} else {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		}
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 1e133c0..3726da7 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 0c3407a..9c49f9c 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

RE: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 4:45 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> From: Huang Wei <wei.huang@intel.com>
> 
> Support dynamic enable or disable queue.
> For front end, like QEMU, user can use ethtool to configure queue.
> For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> 
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/base/ifcvf.c | 100
> ++++++++++++++++++++++++++++++++++++++++++
>  drivers/vdpa/ifc/base/ifcvf.h |   6 +++
>  drivers/vdpa/ifc/ifcvf_vdpa.c |  93 ++++++++++++++++++++++++++++++++-----
> --
>  3 files changed, 184 insertions(+), 15 deletions(-)
> 

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue

[Pei, Andy]

Hi Chenbo,

Thanks for your efforts.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Monday, October 17, 2022 2:26 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, October 13, 2022 4:45 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v4 6/8] vdpa/ifc: support dynamic enable/disable queue
> >
> > From: Huang Wei <wei.huang@intel.com>
> >
> > Support dynamic enable or disable queue.
> > For front end, like QEMU, user can use ethtool to configure queue.
> > For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.
> >
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/base/ifcvf.c | 100
> > ++++++++++++++++++++++++++++++++++++++++++
> >  drivers/vdpa/ifc/base/ifcvf.h |   6 +++
> >  drivers/vdpa/ifc/ifcvf_vdpa.c |  93
> > ++++++++++++++++++++++++++++++++-----
> > --
> >  3 files changed, 184 insertions(+), 15 deletions(-)
> >
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Andy Pei]

When boot from virtio blk device, seabios in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end configure device
when any queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 49 ++++++++++++++++++++++++++++++-------------------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index cd65257..0509025 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1441,9 +1441,10 @@
 }
 
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
+#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
 
 static int
-virtio_is_ready(struct virtio_net *dev)
+virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
 {
 	struct vhost_virtqueue *vq;
 	uint32_t i, nr_vring = dev->nr_vring;
@@ -1454,13 +1455,16 @@
 	if (!dev->nr_vring)
 		return 0;
 
-	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
-		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
-
-		if (dev->nr_vring < nr_vring)
-			return 0;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
+	} else {
+		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
+			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
 	}
 
+	if (dev->nr_vring < nr_vring)
+		return 0;
+
 	for (i = 0; i < nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
@@ -2958,7 +2962,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	int ret;
 	int unlock_required = 0;
 	bool handled;
-	uint32_t vdpa_type = 0;
+	uint32_t vdpa_type = -1;
 	uint32_t request;
 	uint32_t i;
 
@@ -3152,7 +3156,25 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (unlock_required)
 		vhost_user_unlock_all_queue_pairs(dev);
 
-	if (ret != 0 || !virtio_is_ready(dev))
+	if (ret != 0)
+		goto out;
+
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev) {
+		if (vdpa_dev->ops->get_dev_type) {
+			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
+			if (ret) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+					"failed to get vdpa dev type.\n");
+				ret = -1;
+				goto out;
+			}
+		} else {
+			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+		}
+	}
+
+	if (!virtio_is_ready(dev, vdpa_type))
 		goto out;
 
 	/*
@@ -3166,20 +3188,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
 			dev->flags |= VIRTIO_DEV_RUNNING;
 	}
 
-	vdpa_dev = dev->vdpa_dev;
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
 	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
-- 
1.8.3.1

RE: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 4:45 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue
> is ready

This title does not match to the code. You mean first queue?

> 
> When boot from virtio blk device, seabios in QEMU only enables one queue.
> To work in this scenario, vDPA BLK device back-end configure device
> when any queue is ready.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 49 ++++++++++++++++++++++++++++++---------------
> ----
>  1 file changed, 30 insertions(+), 19 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index cd65257..0509025 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -1441,9 +1441,10 @@
>  }
> 
>  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> 
>  static int
> -virtio_is_ready(struct virtio_net *dev)
> +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)

I agree with Maxime's v3 comment. We don't need a new parameter.

Thanks,
Chenbo

>  {
>  	struct vhost_virtqueue *vq;
>  	uint32_t i, nr_vring = dev->nr_vring;
> @@ -1454,13 +1455,16 @@
>  	if (!dev->nr_vring)
>  		return 0;
> 
> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> -
> -		if (dev->nr_vring < nr_vring)
> -			return 0;
> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> +	} else {
> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> +			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>  	}
> 
> +	if (dev->nr_vring < nr_vring)
> +		return 0;
> +
>  	for (i = 0; i < nr_vring; i++) {
>  		vq = dev->virtqueue[i];
> 
> @@ -2958,7 +2962,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	int ret;
>  	int unlock_required = 0;
>  	bool handled;
> -	uint32_t vdpa_type = 0;
> +	uint32_t vdpa_type = -1;
>  	uint32_t request;
>  	uint32_t i;
> 
> @@ -3152,7 +3156,25 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	if (unlock_required)
>  		vhost_user_unlock_all_queue_pairs(dev);
> 
> -	if (ret != 0 || !virtio_is_ready(dev))
> +	if (ret != 0)
> +		goto out;
> +
> +	vdpa_dev = dev->vdpa_dev;
> +	if (vdpa_dev) {
> +		if (vdpa_dev->ops->get_dev_type) {
> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> +			if (ret) {
> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> +					"failed to get vdpa dev type.\n");
> +				ret = -1;
> +				goto out;
> +			}
> +		} else {
> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> +		}
> +	}
> +
> +	if (!virtio_is_ready(dev, vdpa_type))
>  		goto out;
> 
>  	/*
> @@ -3166,20 +3188,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  			dev->flags |= VIRTIO_DEV_RUNNING;
>  	}
> 
> -	vdpa_dev = dev->vdpa_dev;
>  	if (!vdpa_dev)
>  		goto out;
> 
> -	if (vdpa_dev->ops->get_dev_type) {
> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> -		if (ret) {
> -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa
> dev type.\n");
> -			ret = -1;
> -			goto out;
> -		}
> -	} else {
> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> -	}
>  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
>  		&& request != VHOST_USER_SET_VRING_CALL)
>  		goto out;
> --
> 1.8.3.1

RE: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any queue is ready

[Pei, Andy]

Hi Chenbo,

Thanks for your efforts, my reply is inline.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Monday, October 17, 2022 2:34 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any
> queue is ready
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, October 13, 2022 4:45 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v4 7/8] vhost: vDPA blk device gets ready when any
> > queue is ready
> 
> This title does not match to the code. You mean first queue?
> 
Yes, I think I can make the title the first queue.
> >
> > When boot from virtio blk device, seabios in QEMU only enables one queue.
> > To work in this scenario, vDPA BLK device back-end configure device
> > when any queue is ready.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  lib/vhost/vhost_user.c | 49
> > ++++++++++++++++++++++++++++++---------------
> > ----
> >  1 file changed, 30 insertions(+), 19 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > cd65257..0509025 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -1441,9 +1441,10 @@
> >  }
> >
> >  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> > +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> >
> >  static int
> > -virtio_is_ready(struct virtio_net *dev)
> > +virtio_is_ready(struct virtio_net *dev, uint32_t vdpa_type)
> 
> I agree with Maxime's v3 comment. We don't need a new parameter.
> 
OK, I will send a new version to fix this.
> Thanks,
> Chenbo
> 
> >  {
> >  	struct vhost_virtqueue *vq;
> >  	uint32_t i, nr_vring = dev->nr_vring; @@ -1454,13 +1455,16 @@
> >  	if (!dev->nr_vring)
> >  		return 0;
> >
> > -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> > -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > -
> > -		if (dev->nr_vring < nr_vring)
> > -			return 0;
> > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> > +	} else {
> > +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> > +			nr_vring =
> VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> >  	}
> >
> > +	if (dev->nr_vring < nr_vring)
> > +		return 0;
> > +
> >  	for (i = 0; i < nr_vring; i++) {
> >  		vq = dev->virtqueue[i];
> >
> > @@ -2958,7 +2962,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	int ret;
> >  	int unlock_required = 0;
> >  	bool handled;
> > -	uint32_t vdpa_type = 0;
> > +	uint32_t vdpa_type = -1;
> >  	uint32_t request;
> >  	uint32_t i;
> >
> > @@ -3152,7 +3156,25 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	if (unlock_required)
> >  		vhost_user_unlock_all_queue_pairs(dev);
> >
> > -	if (ret != 0 || !virtio_is_ready(dev))
> > +	if (ret != 0)
> > +		goto out;
> > +
> > +	vdpa_dev = dev->vdpa_dev;
> > +	if (vdpa_dev) {
> > +		if (vdpa_dev->ops->get_dev_type) {
> > +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev,
> &vdpa_type);
> > +			if (ret) {
> > +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> > +					"failed to get vdpa dev type.\n");
> > +				ret = -1;
> > +				goto out;
> > +			}
> > +		} else {
> > +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > +		}
> > +	}
> > +
> > +	if (!virtio_is_ready(dev, vdpa_type))
> >  		goto out;
> >
> >  	/*
> > @@ -3166,20 +3188,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  			dev->flags |= VIRTIO_DEV_RUNNING;
> >  	}
> >
> > -	vdpa_dev = dev->vdpa_dev;
> >  	if (!vdpa_dev)
> >  		goto out;
> >
> > -	if (vdpa_dev->ops->get_dev_type) {
> > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > -		if (ret) {
> > -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to
> get vdpa
> > dev type.\n");
> > -			ret = -1;
> > -			goto out;
> > -		}
> > -	} else {
> > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > -	}
> >  	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> >  		&& request != VHOST_USER_SET_VRING_CALL)
> >  		goto out;
> > --
> > 1.8.3.1

[PATCH v4 8/8] vhost: improve vDPA blk device configure condition

[Andy Pei]

To support multi-queue, configure device 
after call fd of all queues are set.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index 0509025..3ede83f 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2965,6 +2965,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	uint32_t vdpa_type = -1;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3191,9 +3192,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

RE: [PATCH v4 8/8] vhost: improve vDPA blk device configure condition

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Thursday, October 13, 2022 4:45 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v4 8/8] vhost: improve vDPA blk device configure condition
> 
> To support multi-queue, configure device
> after call fd of all queues are set.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 13 ++++++++++---
>  1 file changed, 10 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index 0509025..3ede83f 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -2965,6 +2965,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	uint32_t vdpa_type = -1;
>  	uint32_t request;
>  	uint32_t i;
> +	uint16_t blk_call_fd;
> 
>  	dev = get_device(vid);
>  	if (dev == NULL)
> @@ -3191,9 +3192,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	if (!vdpa_dev)
>  		goto out;
> 
> -	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> -		&& request != VHOST_USER_SET_VRING_CALL)
> -		goto out;
> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +		if (request == VHOST_USER_SET_VRING_CALL) {
> +			blk_call_fd = ctx.msg.payload.u64 &
> VHOST_USER_VRING_IDX_MASK;
> +			if (blk_call_fd != dev->nr_vring - 1)
> +				goto out;
> +		} else {
> +			goto out;
> +		}
> +	}
> 
>  	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
>  		if (vdpa_dev->ops->dev_conf(dev->vid))
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v4 8/8] vhost: improve vDPA blk device configure condition

[Pei, Andy]

Hi Chenbo,

Thanks for your efforts.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Monday, October 17, 2022 2:35 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v4 8/8] vhost: improve vDPA blk device configure
> condition
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Thursday, October 13, 2022 4:45 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v4 8/8] vhost: improve vDPA blk device configure
> > condition
> >
> > To support multi-queue, configure device after call fd of all queues
> > are set.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  lib/vhost/vhost_user.c | 13 ++++++++++---
> >  1 file changed, 10 insertions(+), 3 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > 0509025..3ede83f 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -2965,6 +2965,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	uint32_t vdpa_type = -1;
> >  	uint32_t request;
> >  	uint32_t i;
> > +	uint16_t blk_call_fd;
> >
> >  	dev = get_device(vid);
> >  	if (dev == NULL)
> > @@ -3191,9 +3192,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	if (!vdpa_dev)
> >  		goto out;
> >
> > -	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > -		&& request != VHOST_USER_SET_VRING_CALL)
> > -		goto out;
> > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > +		if (request == VHOST_USER_SET_VRING_CALL) {
> > +			blk_call_fd = ctx.msg.payload.u64 &
> > VHOST_USER_VRING_IDX_MASK;
> > +			if (blk_call_fd != dev->nr_vring - 1)
> > +				goto out;
> > +		} else {
> > +			goto out;
> > +		}
> > +	}
> >
> >  	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
> >  		if (vdpa_dev->ops->dev_conf(dev->vid))
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v5 0/8] vdpa/ifc: add multi queue support

[Andy Pei]

v5:
 fix some commit message.
 rework some code logic.

v4:
 fix some commit message.
 add some commets to code.
 fix some code to reduce confusion.

v3:
 rename device ID macro name.
 fix some patch title and commit message.
 delete some used marco.
 rework some code logic.

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (6):
  vdpa/ifc: add multi-queue support
  vdpa/ifc: set max queues based on virtio spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vhost: vDPA blk device gets ready when the first queue is ready
  vhost: improve vDPA blk device configure condition

Huang Wei (2):
  vdpa/ifc: add new device ID for legacy network device
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  16 ++++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++------
 lib/vhost/vhost_user.c        |  44 +++++++++++--
 4 files changed, 315 insertions(+), 31 deletions(-)

-- 
1.8.3.1

[PATCH v5 1/8] vdpa/ifc: add new device ID for legacy network device

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..ef7697a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,12 +12,14 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
-#define IFCVF_BLK_DEVICE_ID                 0x0002
+#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
+#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
 #define IFCVF_MAX_QUEUES		1
 
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index d5ac583..b4389a0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .vendor_id = 0, /* sentinel */
-- 
1.8.3.1

[PATCH v5 2/8] vdpa/ifc: add multi-queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
Expose IFCVF_MQ_OFFSET register to enable multi-queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 12 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..81c68c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,15 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	/* For some hardware implementation, for example:
+	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
+	 * This code makes sure hw->mq_cfg is a valid address.
+	 */
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ef7697a..d16d9ab 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index b4389a0..008cf89 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

[PATCH v5 3/8] vdpa/ifc: set max queues based on virtio spec

[Andy Pei]

Set max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues
in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg,
get the queue pair number using num_queues and set max_queues to it.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index d16d9ab..1e133c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
 #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 008cf89..5a24204 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,12 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MIN() cannot be used since braced-group within expression allowed
+ * only inside a function.
+ */
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		/*
+		 * ifc device always has CTRL_VQ,
+		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
+		 */
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

[PATCH v5 4/8] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi-queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 81c68c0..b377126 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -202,6 +202,37 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_mq(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	u8 qid;
+	int nr_queue = 0;
+
+	for (qid = 0; qid < hw->nr_vring; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
+		nr_queue++;
+	}
+
+	if (nr_queue == 0) {
+		WARNINGOUT("no enabled vring\n");
+		return;
+	}
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			*(u32 *)mq_cfg = nr_queue;
+			RTE_LOG(INFO, PMD, "%d queues are enabled\n", nr_queue);
+		} else {
+			*(u32 *)mq_cfg = nr_queue / 2;
+			RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+				nr_queue / 2);
+		}
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -219,6 +250,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

[PATCH v5 5/8] vdpa/ifc: only configure enabled queue

[Andy Pei]

When configuring the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  3 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index b377126..30bb8cb 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -252,6 +252,9 @@
 
 	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 5a24204..0c3407a 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v5 6/8] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |   6 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c |  93 ++++++++++++++++++++++++++++++++-------
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 30bb8cb..869ddd6 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -233,6 +233,106 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (i >= (int)hw->nr_vring)
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_mq(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			RTE_LOG(ERR, PMD, "queue %d, msix vec alloc failed\n",
+				i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (i >= (int)hw->nr_vring)
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		} else {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		}
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 1e133c0..3726da7 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 0c3407a..9c49f9c 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready

[Andy Pei]

When boot from virtio blk device, seabios in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end configure device
when any queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index cd65257..f5206dd 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1441,11 +1441,15 @@
 }
 
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
+#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
 
 static int
 virtio_is_ready(struct virtio_net *dev)
 {
+	struct rte_vdpa_device *vdpa_dev;
 	struct vhost_virtqueue *vq;
+	uint32_t vdpa_type;
+	int ret = 0;
 	uint32_t i, nr_vring = dev->nr_vring;
 
 	if (dev->flags & VIRTIO_DEV_READY)
@@ -1454,13 +1458,32 @@
 	if (!dev->nr_vring)
 		return 0;
 
-	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
-		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev) {
+		if (vdpa_dev->ops->get_dev_type) {
+			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
+			if (ret) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+					"failed to get vdpa dev type.\n");
+				return -1;
+			}
+		} else {
+			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+		}
+	} else {
+		vdpa_type = -1;
+	}
 
-		if (dev->nr_vring < nr_vring)
-			return 0;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
+	} else {
+		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
+			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
 	}
 
+	if (dev->nr_vring < nr_vring)
+		return 0;
+
 	for (i = 0; i < nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
-- 
1.8.3.1

RE: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Monday, October 17, 2022 3:14 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first
> queue is ready
> 
> When boot from virtio blk device, seabios in QEMU only enables one queue.
> To work in this scenario, vDPA BLK device back-end configure device
> when any queue is ready.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++----
>  1 file changed, 27 insertions(+), 4 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index cd65257..f5206dd 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -1441,11 +1441,15 @@
>  }
> 
>  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> 
>  static int
>  virtio_is_ready(struct virtio_net *dev)
>  {
> +	struct rte_vdpa_device *vdpa_dev;
>  	struct vhost_virtqueue *vq;
> +	uint32_t vdpa_type;
> +	int ret = 0;
>  	uint32_t i, nr_vring = dev->nr_vring;
> 
>  	if (dev->flags & VIRTIO_DEV_READY)
> @@ -1454,13 +1458,32 @@
>  	if (!dev->nr_vring)
>  		return 0;
> 
> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> +	vdpa_dev = dev->vdpa_dev;
> +	if (vdpa_dev) {
> +		if (vdpa_dev->ops->get_dev_type) {
> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> +			if (ret) {
> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> +					"failed to get vdpa dev type.\n");
> +				return -1;
> +			}
> +		} else {
> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> +		}
> +	} else {
> +		vdpa_type = -1;
> +	}

Looking at it again, instead of getting the device type again and again,
Should we just get and set the type into vdpa_dev when registering a vdpa device?

What do you think?

Thanks,
Chenbo

> 
> -		if (dev->nr_vring < nr_vring)
> -			return 0;
> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> +	} else {
> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> +			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>  	}
> 
> +	if (dev->nr_vring < nr_vring)
> +		return 0;
> +
>  	for (i = 0; i < nr_vring; i++) {
>  		vq = dev->virtqueue[i];
> 
> --
> 1.8.3.1

Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready

[Maxime Coquelin]

On 10/17/22 09:54, Xia, Chenbo wrote:
>> -----Original Message-----
>> From: Pei, Andy <andy.pei@intel.com>
>> Sent: Monday, October 17, 2022 3:14 PM
>> To: dev@dpdk.org
>> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
>> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
>> maxime.coquelin@redhat.com
>> Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first
>> queue is ready
>>
>> When boot from virtio blk device, seabios in QEMU only enables one queue.
>> To work in this scenario, vDPA BLK device back-end configure device
>> when any queue is ready.
>>
>> Signed-off-by: Andy Pei <andy.pei@intel.com>
>> Signed-off-by: Huang Wei <wei.huang@intel.com>
>> ---
>>   lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++----
>>   1 file changed, 27 insertions(+), 4 deletions(-)
>>
>> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
>> index cd65257..f5206dd 100644
>> --- a/lib/vhost/vhost_user.c
>> +++ b/lib/vhost/vhost_user.c
>> @@ -1441,11 +1441,15 @@
>>   }
>>
>>   #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
>> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
>>
>>   static int
>>   virtio_is_ready(struct virtio_net *dev)
>>   {
>> +	struct rte_vdpa_device *vdpa_dev;
>>   	struct vhost_virtqueue *vq;
>> +	uint32_t vdpa_type;
>> +	int ret = 0;
>>   	uint32_t i, nr_vring = dev->nr_vring;
>>
>>   	if (dev->flags & VIRTIO_DEV_READY)
>> @@ -1454,13 +1458,32 @@
>>   	if (!dev->nr_vring)
>>   		return 0;
>>
>> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
>> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>> +	vdpa_dev = dev->vdpa_dev;
>> +	if (vdpa_dev) {
>> +		if (vdpa_dev->ops->get_dev_type) {
>> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
>> +			if (ret) {
>> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
>> +					"failed to get vdpa dev type.\n");
>> +				return -1;
>> +			}
>> +		} else {
>> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
>> +		}
>> +	} else {
>> +		vdpa_type = -1;
>> +	}
> 
> Looking at it again, instead of getting the device type again and again,
> Should we just get and set the type into vdpa_dev when registering a vdpa device?
> 
> What do you think?

I think this would be preferable.

If it is to be in struct virtio_net, please take care to place it
properly not to create new holes. Also, it does not have to be placed in
hot cachelines.

Thanks,
Maxime
> Thanks,
> Chenbo
> 
>>
>> -		if (dev->nr_vring < nr_vring)
>> -			return 0;
>> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
>> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
>> +	} else {
>> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
>> +			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>>   	}
>>
>> +	if (dev->nr_vring < nr_vring)
>> +		return 0;
>> +
>>   	for (i = 0; i < nr_vring; i++) {
>>   		vq = dev->virtqueue[i];
>>
>> --
>> 1.8.3.1
> 

RE: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready

[Xia, Chenbo]

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Monday, October 17, 2022 4:37 PM
> To: Xia, Chenbo <chenbo.xia@intel.com>; Pei, Andy <andy.pei@intel.com>;
> dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
> Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the
> first queue is ready
> 
> 
> 
> On 10/17/22 09:54, Xia, Chenbo wrote:
> >> -----Original Message-----
> >> From: Pei, Andy <andy.pei@intel.com>
> >> Sent: Monday, October 17, 2022 3:14 PM
> >> To: dev@dpdk.org
> >> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> >> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> >> maxime.coquelin@redhat.com
> >> Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the
> first
> >> queue is ready
> >>
> >> When boot from virtio blk device, seabios in QEMU only enables one
> queue.
> >> To work in this scenario, vDPA BLK device back-end configure device
> >> when any queue is ready.
> >>
> >> Signed-off-by: Andy Pei <andy.pei@intel.com>
> >> Signed-off-by: Huang Wei <wei.huang@intel.com>
> >> ---
> >>   lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++----
> >>   1 file changed, 27 insertions(+), 4 deletions(-)
> >>
> >> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> >> index cd65257..f5206dd 100644
> >> --- a/lib/vhost/vhost_user.c
> >> +++ b/lib/vhost/vhost_user.c
> >> @@ -1441,11 +1441,15 @@
> >>   }
> >>
> >>   #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> >> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> >>
> >>   static int
> >>   virtio_is_ready(struct virtio_net *dev)
> >>   {
> >> +	struct rte_vdpa_device *vdpa_dev;
> >>   	struct vhost_virtqueue *vq;
> >> +	uint32_t vdpa_type;
> >> +	int ret = 0;
> >>   	uint32_t i, nr_vring = dev->nr_vring;
> >>
> >>   	if (dev->flags & VIRTIO_DEV_READY)
> >> @@ -1454,13 +1458,32 @@
> >>   	if (!dev->nr_vring)
> >>   		return 0;
> >>
> >> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> >> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> >> +	vdpa_dev = dev->vdpa_dev;
> >> +	if (vdpa_dev) {
> >> +		if (vdpa_dev->ops->get_dev_type) {
> >> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> >> +			if (ret) {
> >> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
> >> +					"failed to get vdpa dev type.\n");
> >> +				return -1;
> >> +			}
> >> +		} else {
> >> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> >> +		}
> >> +	} else {
> >> +		vdpa_type = -1;
> >> +	}
> >
> > Looking at it again, instead of getting the device type again and again,
> > Should we just get and set the type into vdpa_dev when registering a
> vdpa device?
> >
> > What do you think?
> 
> I think this would be preferable.
> 
> If it is to be in struct virtio_net, please take care to place it
> properly not to create new holes. Also, it does not have to be placed in
> hot cachelines.

I am thinking struct rte_vdpa_device. Because if no vdpa device, there will
be no vdpa device type. What do you think?

Thanks,
Chenbo

> 
> Thanks,
> Maxime
> > Thanks,
> > Chenbo
> >
> >>
> >> -		if (dev->nr_vring < nr_vring)
> >> -			return 0;
> >> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> >> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> >> +	} else {
> >> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> >> +			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> >>   	}
> >>
> >> +	if (dev->nr_vring < nr_vring)
> >> +		return 0;
> >> +
> >>   	for (i = 0; i < nr_vring; i++) {
> >>   		vq = dev->virtqueue[i];
> >>
> >> --
> >> 1.8.3.1
> >

Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the first queue is ready

[Maxime Coquelin]

On 10/17/22 10:42, Xia, Chenbo wrote:
>> -----Original Message-----
>> From: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Sent: Monday, October 17, 2022 4:37 PM
>> To: Xia, Chenbo <chenbo.xia@intel.com>; Pei, Andy <andy.pei@intel.com>;
>> dev@dpdk.org
>> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao,
>> Gang <gang.cao@intel.com>
>> Subject: Re: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the
>> first queue is ready
>>
>>
>>
>> On 10/17/22 09:54, Xia, Chenbo wrote:
>>>> -----Original Message-----
>>>> From: Pei, Andy <andy.pei@intel.com>
>>>> Sent: Monday, October 17, 2022 3:14 PM
>>>> To: dev@dpdk.org
>>>> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
>>>> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
>>>> maxime.coquelin@redhat.com
>>>> Subject: [PATCH v5 7/8] vhost: vDPA blk device gets ready when the
>> first
>>>> queue is ready
>>>>
>>>> When boot from virtio blk device, seabios in QEMU only enables one
>> queue.
>>>> To work in this scenario, vDPA BLK device back-end configure device
>>>> when any queue is ready.
>>>>
>>>> Signed-off-by: Andy Pei <andy.pei@intel.com>
>>>> Signed-off-by: Huang Wei <wei.huang@intel.com>
>>>> ---
>>>>    lib/vhost/vhost_user.c | 31 +++++++++++++++++++++++++++----
>>>>    1 file changed, 27 insertions(+), 4 deletions(-)
>>>>
>>>> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
>>>> index cd65257..f5206dd 100644
>>>> --- a/lib/vhost/vhost_user.c
>>>> +++ b/lib/vhost/vhost_user.c
>>>> @@ -1441,11 +1441,15 @@
>>>>    }
>>>>
>>>>    #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
>>>> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
>>>>
>>>>    static int
>>>>    virtio_is_ready(struct virtio_net *dev)
>>>>    {
>>>> +	struct rte_vdpa_device *vdpa_dev;
>>>>    	struct vhost_virtqueue *vq;
>>>> +	uint32_t vdpa_type;
>>>> +	int ret = 0;
>>>>    	uint32_t i, nr_vring = dev->nr_vring;
>>>>
>>>>    	if (dev->flags & VIRTIO_DEV_READY)
>>>> @@ -1454,13 +1458,32 @@
>>>>    	if (!dev->nr_vring)
>>>>    		return 0;
>>>>
>>>> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
>>>> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>>>> +	vdpa_dev = dev->vdpa_dev;
>>>> +	if (vdpa_dev) {
>>>> +		if (vdpa_dev->ops->get_dev_type) {
>>>> +			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
>>>> +			if (ret) {
>>>> +				VHOST_LOG_CONFIG(dev->ifname, ERR,
>>>> +					"failed to get vdpa dev type.\n");
>>>> +				return -1;
>>>> +			}
>>>> +		} else {
>>>> +			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
>>>> +		}
>>>> +	} else {
>>>> +		vdpa_type = -1;
>>>> +	}
>>>
>>> Looking at it again, instead of getting the device type again and again,
>>> Should we just get and set the type into vdpa_dev when registering a
>> vdpa device?
>>>
>>> What do you think?
>>
>> I think this would be preferable.
>>
>> If it is to be in struct virtio_net, please take care to place it
>> properly not to create new holes. Also, it does not have to be placed in
>> hot cachelines.
> 
> I am thinking struct rte_vdpa_device. Because if no vdpa device, there will
> be no vdpa device type. What do you think?

Just looked at the code and indeed, struct rte_vdpa_device is the right
place.

Thanks,
Maxime

> Thanks,
> Chenbo
> 
>>
>> Thanks,
>> Maxime
>>> Thanks,
>>> Chenbo
>>>
>>>>
>>>> -		if (dev->nr_vring < nr_vring)
>>>> -			return 0;
>>>> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
>>>> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
>>>> +	} else {
>>>> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
>>>> +			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>>>>    	}
>>>>
>>>> +	if (dev->nr_vring < nr_vring)
>>>> +		return 0;
>>>> +
>>>>    	for (i = 0; i < nr_vring; i++) {
>>>>    		vq = dev->virtqueue[i];
>>>>
>>>> --
>>>> 1.8.3.1
>>>
> 

[PATCH v5 8/8] vhost: improve vDPA blk device configure condition

[Andy Pei]

To support multi-queue, configure device
after call fd of all queues are set.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index f5206dd..6b5f89a 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2984,6 +2984,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	uint32_t vdpa_type = 0;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3203,9 +3204,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	} else {
 		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
 	}
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

[PATCH v6 0/8] vdpa/ifc: add multi queue support

[Andy Pei]

v6:
 Add vdpa_device_type to rte_vdpa_device to store vDPA device type.

v5:
 fix some commit message.
 rework some code logic.

v4:
 fix some commit message.
 add some commets to code.
 fix some code to reduce confusion.

v3:
 rename device ID macro name.
 fix some patch title and commit message.
 delete some used marco.
 rework some code logic.

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (6):
  vdpa/ifc: add multi-queue support
  vdpa/ifc: set max queues based on virtio spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vhost: vDPA blk device gets ready when the first queue is ready
  vhost: improve vDPA blk device configure condition

Huang Wei (2):
  vdpa/ifc: add new device ID for legacy network device
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  16 ++++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 142 +++++++++++++++++++++++++++++++++++------
 lib/vhost/vdpa_driver.h       |   2 +
 lib/vhost/vhost_user.c        |  62 +++++++++++++-----
 5 files changed, 325 insertions(+), 41 deletions(-)

-- 
1.8.3.1

[PATCH v6 1/8] vdpa/ifc: add new device ID for legacy network device

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..ef7697a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,12 +12,14 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
-#define IFCVF_BLK_DEVICE_ID                 0x0002
+#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
+#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
 #define IFCVF_MAX_QUEUES		1
 
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index d5ac583..b4389a0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .vendor_id = 0, /* sentinel */
-- 
1.8.3.1

[PATCH v6 2/8] vdpa/ifc: add multi-queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
Expose IFCVF_MQ_OFFSET register to enable multi-queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 12 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..81c68c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,15 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	/* For some hardware implementation, for example:
+	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
+	 * This code makes sure hw->mq_cfg is a valid address.
+	 */
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ef7697a..d16d9ab 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index b4389a0..008cf89 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

[PATCH v6 3/8] vdpa/ifc: set max queues based on virtio spec

[Andy Pei]

Set max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues
in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg,
get the queue pair number using num_queues and set max_queues to it.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index d16d9ab..1e133c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
 #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 008cf89..5a24204 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,12 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MIN() cannot be used since braced-group within expression allowed
+ * only inside a function.
+ */
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		/*
+		 * ifc device always has CTRL_VQ,
+		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
+		 */
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

[PATCH v6 4/8] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi-queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 81c68c0..b377126 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -202,6 +202,37 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_mq(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	u8 qid;
+	int nr_queue = 0;
+
+	for (qid = 0; qid < hw->nr_vring; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
+		nr_queue++;
+	}
+
+	if (nr_queue == 0) {
+		WARNINGOUT("no enabled vring\n");
+		return;
+	}
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			*(u32 *)mq_cfg = nr_queue;
+			RTE_LOG(INFO, PMD, "%d queues are enabled\n", nr_queue);
+		} else {
+			*(u32 *)mq_cfg = nr_queue / 2;
+			RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+				nr_queue / 2);
+		}
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -219,6 +250,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

[PATCH v6 5/8] vdpa/ifc: only configure enabled queue

[Andy Pei]

When configuring the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  3 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index b377126..30bb8cb 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -252,6 +252,9 @@
 
 	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 5a24204..0c3407a 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v6 6/8] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |   6 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c |  93 ++++++++++++++++++++++++++++++++-------
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 30bb8cb..869ddd6 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -233,6 +233,106 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (i >= (int)hw->nr_vring)
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_mq(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			RTE_LOG(ERR, PMD, "queue %d, msix vec alloc failed\n",
+				i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (i >= (int)hw->nr_vring)
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		} else {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		}
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 1e133c0..3726da7 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 0c3407a..9c49f9c 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v6 7/8] vhost: vDPA blk device gets ready when the first queue is ready

[Andy Pei]

When boot from virtio blk device, seabios in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end configure device
when any queue is ready.
Add vdpa_device_type to rte_vdpa_device to store vDPA device type.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vdpa_driver.h |  2 ++
 lib/vhost/vhost_user.c  | 55 ++++++++++++++++++++++++++++++++++---------------
 2 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h
index 8b88a53..c4ec222 100644
--- a/lib/vhost/vdpa_driver.h
+++ b/lib/vhost/vdpa_driver.h
@@ -92,6 +92,8 @@ struct rte_vdpa_device {
 	struct rte_device *device;
 	/** vdpa device operations */
 	struct rte_vdpa_dev_ops *ops;
+	/** vdpa device type: net, blk... */
+	uint32_t vdpa_device_type;
 };
 
 /**
diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index cd65257..53806fa 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1441,11 +1441,14 @@
 }
 
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
+#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
 
 static int
 virtio_is_ready(struct virtio_net *dev)
 {
+	struct rte_vdpa_device *vdpa_dev;
 	struct vhost_virtqueue *vq;
+	uint32_t vdpa_type;
 	uint32_t i, nr_vring = dev->nr_vring;
 
 	if (dev->flags & VIRTIO_DEV_READY)
@@ -1454,13 +1457,22 @@
 	if (!dev->nr_vring)
 		return 0;
 
-	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
-		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev)
+		vdpa_type = vdpa_dev->vdpa_device_type;
+	else
+		vdpa_type = -1;
 
-		if (dev->nr_vring < nr_vring)
-			return 0;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
+	} else {
+		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
+			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
 	}
 
+	if (dev->nr_vring < nr_vring)
+		return 0;
+
 	for (i = 0; i < nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
@@ -2958,7 +2970,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	int ret;
 	int unlock_required = 0;
 	bool handled;
-	uint32_t vdpa_type = 0;
+	uint32_t vdpa_type = -1;
 	uint32_t request;
 	uint32_t i;
 
@@ -3152,7 +3164,27 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (unlock_required)
 		vhost_user_unlock_all_queue_pairs(dev);
 
-	if (ret != 0 || !virtio_is_ready(dev))
+	if (ret != 0)
+		goto out;
+
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev) {
+		if (vdpa_dev->ops->get_dev_type) {
+			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
+			if (ret) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+					"failed to get vdpa dev type.\n");
+				ret = -1;
+				goto out;
+			}
+		} else {
+			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+		}
+
+		vdpa_dev->vdpa_device_type = vdpa_type;
+	}
+
+	if (!virtio_is_ready(dev))
 		goto out;
 
 	/*
@@ -3166,20 +3198,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
 			dev->flags |= VIRTIO_DEV_RUNNING;
 	}
 
-	vdpa_dev = dev->vdpa_dev;
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
 	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
-- 
1.8.3.1

[PATCH v6 8/8] vhost: improve vDPA blk device configure condition

[Andy Pei]

To support multi-queue, configure device
after call fd of all queues are set.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index 53806fa..2c50d13 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2973,6 +2973,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	uint32_t vdpa_type = -1;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3201,9 +3202,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

[PATCH v7 00/12] vdpa/ifc: add multi queue support

[Andy Pei]

v7:
 Fill vdpa_device_type in vdpa device registration.

v6:
 Add vdpa_device_type to rte_vdpa_device to store vDPA device type.

v5:
 fix some commit message.
 rework some code logic.

v4:
 fix some commit message.
 add some commets to code.
 fix some code to reduce confusion.

v3:
 rename device ID macro name.
 fix some patch title and commit message.
 delete some used marco.
 rework some code logic.

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (10):
  vdpa/ifc: add multi-queue support
  vdpa/ifc: set max queues based on virtio spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vdpa/ifc: change internal function name
  vdpa/ifc: add internal API to get device.
  vdpa/ifc: change some driver logic
  vhost: add vdpa device type to rte vdpa device
  vhost: vDPA blk device gets ready when the first queue is ready
  vhost: improve vDPA blk device configure condition

Huang Wei (2):
  vdpa/ifc: add new device ID for legacy network device
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  16 +++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++-------
 lib/vhost/socket.c            |  15 +---
 lib/vhost/vdpa.c              |  17 ++++
 lib/vhost/vdpa_driver.h       |   2 +
 lib/vhost/vhost_user.c        |  40 +++++----
 7 files changed, 358 insertions(+), 61 deletions(-)

-- 
1.8.3.1

[PATCH v7 01/12] vdpa/ifc: add new device ID for legacy network device

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..ef7697a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,12 +12,14 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
-#define IFCVF_BLK_DEVICE_ID                 0x0002
+#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
+#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
 #define IFCVF_MAX_QUEUES		1
 
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index d5ac583..b4389a0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .vendor_id = 0, /* sentinel */
-- 
1.8.3.1

[PATCH v7 02/12] vdpa/ifc: add multi-queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
Expose IFCVF_MQ_OFFSET register to enable multi-queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 12 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..81c68c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,15 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	/* For some hardware implementation, for example:
+	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
+	 * This code makes sure hw->mq_cfg is a valid address.
+	 */
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ef7697a..d16d9ab 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index b4389a0..008cf89 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

[PATCH v7 03/12] vdpa/ifc: set max queues based on virtio spec

[Andy Pei]

Set max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues
in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg,
get the queue pair number using num_queues and set max_queues to it.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index d16d9ab..1e133c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
 #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 008cf89..5a24204 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,12 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MIN() cannot be used since braced-group within expression allowed
+ * only inside a function.
+ */
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		/*
+		 * ifc device always has CTRL_VQ,
+		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
+		 */
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

[PATCH v7 04/12] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi-queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 81c68c0..b377126 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -202,6 +202,37 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_mq(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	u8 qid;
+	int nr_queue = 0;
+
+	for (qid = 0; qid < hw->nr_vring; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
+		nr_queue++;
+	}
+
+	if (nr_queue == 0) {
+		WARNINGOUT("no enabled vring\n");
+		return;
+	}
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			*(u32 *)mq_cfg = nr_queue;
+			RTE_LOG(INFO, PMD, "%d queues are enabled\n", nr_queue);
+		} else {
+			*(u32 *)mq_cfg = nr_queue / 2;
+			RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+				nr_queue / 2);
+		}
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -219,6 +250,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

[PATCH v7 05/12] vdpa/ifc: only configure enabled queue

[Andy Pei]

When configuring the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  3 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index b377126..30bb8cb 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -252,6 +252,9 @@
 
 	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 5a24204..0c3407a 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v7 06/12] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |   6 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c |  93 ++++++++++++++++++++++++++++++++-------
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 30bb8cb..869ddd6 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -233,6 +233,106 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (i >= (int)hw->nr_vring)
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_mq(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			RTE_LOG(ERR, PMD, "queue %d, msix vec alloc failed\n",
+				i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (i >= (int)hw->nr_vring)
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		} else {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		}
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 1e133c0..3726da7 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 0c3407a..9c49f9c 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v7 07/12] vdpa/ifc: change internal function name

[Andy Pei]

Change internal function name "find_internal_resource_by_dev"
to "find_internal_resource_by_pci_dev".

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 9c49f9c..73d04ed 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info {
 }
 
 static struct internal_list *
-find_internal_resource_by_dev(struct rte_pci_device *pdev)
+find_internal_resource_by_pci_dev(struct rte_pci_device *pdev)
 {
 	int found = 0;
 	struct internal_list *list;
@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
-	list = find_internal_resource_by_dev(pci_dev);
+	list = find_internal_resource_by_pci_dev(pci_dev);
 	if (list == NULL) {
 		DRV_LOG(ERR, "Invalid device: %s", pci_dev->name);
 		return -1;
-- 
1.8.3.1

[PATCH v7 08/12] vdpa/ifc: add internal API to get device

[Andy Pei]

Add new internal API "find_internal_resource_by_rte_dev"
to get device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 73d04ed..c16e263 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {
 	return list;
 }
 
+static struct internal_list *
+find_internal_resource_by_rte_dev(struct rte_device *rte_dev)
+{
+	int found = 0;
+	struct internal_list *list;
+
+	pthread_mutex_lock(&internal_list_lock);
+
+	TAILQ_FOREACH(list, &internal_list, next) {
+		if (rte_dev == &list->internal->pdev->device) {
+			found = 1;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&internal_list_lock);
+
+	if (!found)
+		return NULL;
+
+	return list;
+}
+
 static int
 ifcvf_vfio_setup(struct ifcvf_internal *internal)
 {
@@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info {
 {
 	struct ifcvf_internal *internal;
 	struct internal_list *list;
+	struct rte_device *rte_dev = vdev->device;
 
-	list = find_internal_resource_by_vdev(vdev);
+	list = find_internal_resource_by_rte_dev(rte_dev);
 	if (list == NULL) {
-		DRV_LOG(ERR, "Invalid vDPA device: %p", vdev);
+		DRV_LOG(ERR, "Invalid rte device: %p", rte_dev);
 		return -1;
 	}
 
-- 
1.8.3.1

[PATCH v7 09/12] vdpa/ifc: change some driver logic

[Andy Pei]

Insert internal list element to internal list before
register vdpa device, in order to call vdpa ops during
vdpa device registration.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index c16e263..8dfd493 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 	internal->sw_lm = sw_fallback_lm;
 
+	pthread_mutex_lock(&internal_list_lock);
+	TAILQ_INSERT_TAIL(&internal_list, list, next);
+	pthread_mutex_unlock(&internal_list_lock);
+
 	internal->vdev = rte_vdpa_register_device(&pci_dev->device,
 				dev_info[internal->hw.device_type].ops);
 	if (internal->vdev == NULL) {
 		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
+		pthread_mutex_lock(&internal_list_lock);
+		TAILQ_REMOVE(&internal_list, list, next);
+		pthread_mutex_unlock(&internal_list_lock);
 		goto error;
 	}
 
-	pthread_mutex_lock(&internal_list_lock);
-	TAILQ_INSERT_TAIL(&internal_list, list, next);
-	pthread_mutex_unlock(&internal_list_lock);
-
 	rte_atomic32_set(&internal->started, 1);
 	update_datapath(internal);
 
-- 
1.8.3.1

[PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device

[Andy Pei]

Add vdpa_device_type to rte_vdpa_device to store device type.
Call vdpa ops get_dev_type to fill vdpa_device_type
when register vdpa device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 lib/vhost/socket.c      | 15 +--------------
 lib/vhost/vdpa.c        | 17 +++++++++++++++++
 lib/vhost/vdpa_driver.h |  2 ++
 3 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c
index 608ae57..f768114 100644
--- a/lib/vhost/socket.c
+++ b/lib/vhost/socket.c
@@ -627,7 +627,6 @@ struct rte_vdpa_device *
 {
 	struct vhost_user_socket *vsocket;
 	struct rte_vdpa_device *vdpa_dev;
-	uint32_t vdpa_type = 0;
 	int ret = 0;
 
 	pthread_mutex_lock(&vhost_user.mutex);
@@ -644,19 +643,7 @@ struct rte_vdpa_device *
 		goto unlock_exit;
 	}
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(path, ERR,
-				"failed to get vdpa dev type for socket file.\n");
-			ret = -1;
-			goto unlock_exit;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
-
-	*type = vdpa_type;
+	*type = vdpa_dev->vdpa_device_type;
 
 unlock_exit:
 	pthread_mutex_unlock(&vhost_user.mutex);
diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c
index bb82857..b487f4d 100644
--- a/lib/vhost/vdpa.c
+++ b/lib/vhost/vdpa.c
@@ -73,6 +73,8 @@ struct rte_vdpa_device *
 		struct rte_vdpa_dev_ops *ops)
 {
 	struct rte_vdpa_device *dev;
+	uint32_t vdpa_type = -1;
+	int ret = 0;
 
 	if (ops == NULL)
 		return NULL;
@@ -101,6 +103,21 @@ struct rte_vdpa_device *
 
 	dev->device = rte_dev;
 	dev->ops = ops;
+
+	if (ops->get_dev_type) {
+		ret = ops->get_dev_type(dev, &vdpa_type);
+		if (ret) {
+			VHOST_LOG_CONFIG(rte_dev->name, ERR,
+					 "Failed to get vdpa dev type.\n");
+			ret = -1;
+			goto out_unlock;
+		}
+	} else {
+		/** by default, we assume vdpa device is a net device */
+		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+	}
+	dev->vdpa_device_type = vdpa_type;
+
 	TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
 out_unlock:
 	rte_spinlock_unlock(&vdpa_device_list_lock);
diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h
index 8b88a53..c4ec222 100644
--- a/lib/vhost/vdpa_driver.h
+++ b/lib/vhost/vdpa_driver.h
@@ -92,6 +92,8 @@ struct rte_vdpa_device {
 	struct rte_device *device;
 	/** vdpa device operations */
 	struct rte_vdpa_dev_ops *ops;
+	/** vdpa device type: net, blk... */
+	uint32_t vdpa_device_type;
 };
 
 /**
-- 
1.8.3.1

Re: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device

[Maxime Coquelin]

On 10/18/22 08:19, Andy Pei wrote:
> Add vdpa_device_type to rte_vdpa_device to store device type.
> Call vdpa ops get_dev_type to fill vdpa_device_type
> when register vdpa device.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   lib/vhost/socket.c      | 15 +--------------
>   lib/vhost/vdpa.c        | 17 +++++++++++++++++
>   lib/vhost/vdpa_driver.h |  2 ++
>   3 files changed, 20 insertions(+), 14 deletions(-)
> 
> diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c
> index 608ae57..f768114 100644
> --- a/lib/vhost/socket.c
> +++ b/lib/vhost/socket.c
> @@ -627,7 +627,6 @@ struct rte_vdpa_device *
>   {
>   	struct vhost_user_socket *vsocket;
>   	struct rte_vdpa_device *vdpa_dev;
> -	uint32_t vdpa_type = 0;
>   	int ret = 0;
>   
>   	pthread_mutex_lock(&vhost_user.mutex);
> @@ -644,19 +643,7 @@ struct rte_vdpa_device *
>   		goto unlock_exit;
>   	}
>   
> -	if (vdpa_dev->ops->get_dev_type) {
> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> -		if (ret) {
> -			VHOST_LOG_CONFIG(path, ERR,
> -				"failed to get vdpa dev type for socket file.\n");
> -			ret = -1;
> -			goto unlock_exit;
> -		}
> -	} else {
> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> -	}
> -
> -	*type = vdpa_type;
> +	*type = vdpa_dev->vdpa_device_type;
>   
>   unlock_exit:
>   	pthread_mutex_unlock(&vhost_user.mutex);
> diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c
> index bb82857..b487f4d 100644
> --- a/lib/vhost/vdpa.c
> +++ b/lib/vhost/vdpa.c
> @@ -73,6 +73,8 @@ struct rte_vdpa_device *
>   		struct rte_vdpa_dev_ops *ops)
>   {
>   	struct rte_vdpa_device *dev;
> +	uint32_t vdpa_type = -1;
> +	int ret = 0;
>   
>   	if (ops == NULL)
>   		return NULL;
> @@ -101,6 +103,21 @@ struct rte_vdpa_device *
>   
>   	dev->device = rte_dev;
>   	dev->ops = ops;
> +
> +	if (ops->get_dev_type) {
> +		ret = ops->get_dev_type(dev, &vdpa_type);
> +		if (ret) {
> +			VHOST_LOG_CONFIG(rte_dev->name, ERR,
> +					 "Failed to get vdpa dev type.\n");
> +			ret = -1;
> +			goto out_unlock;
> +		}
> +	} else {
> +		/** by default, we assume vdpa device is a net device */
> +		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> +	}
> +	dev->vdpa_device_type = vdpa_type;
> +
>   	TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
>   out_unlock:
>   	rte_spinlock_unlock(&vdpa_device_list_lock);
> diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h
> index 8b88a53..c4ec222 100644
> --- a/lib/vhost/vdpa_driver.h
> +++ b/lib/vhost/vdpa_driver.h
> @@ -92,6 +92,8 @@ struct rte_vdpa_device {
>   	struct rte_device *device;
>   	/** vdpa device operations */
>   	struct rte_vdpa_dev_ops *ops;
> +	/** vdpa device type: net, blk... */
> +	uint32_t vdpa_device_type;

I would name it simply "type".
It is part of rte_vdpa_device struct, no need to duplicate information.

Doing that, you can use it directly in later patches using
'vdpa_dev->type'.

Maxime

>   };
>   
>   /**

RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device

[Pei, Andy]

Hi Maxime,

Thanks for your reply.
I think your suggestion is good, I will send a new version to do it.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Tuesday, October 18, 2022 3:15 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device
> 
> 
> 
> On 10/18/22 08:19, Andy Pei wrote:
> > Add vdpa_device_type to rte_vdpa_device to store device type.
> > Call vdpa ops get_dev_type to fill vdpa_device_type when register vdpa
> > device.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >   lib/vhost/socket.c      | 15 +--------------
> >   lib/vhost/vdpa.c        | 17 +++++++++++++++++
> >   lib/vhost/vdpa_driver.h |  2 ++
> >   3 files changed, 20 insertions(+), 14 deletions(-)
> >
> > diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c index
> > 608ae57..f768114 100644
> > --- a/lib/vhost/socket.c
> > +++ b/lib/vhost/socket.c
> > @@ -627,7 +627,6 @@ struct rte_vdpa_device *
> >   {
> >   	struct vhost_user_socket *vsocket;
> >   	struct rte_vdpa_device *vdpa_dev;
> > -	uint32_t vdpa_type = 0;
> >   	int ret = 0;
> >
> >   	pthread_mutex_lock(&vhost_user.mutex);
> > @@ -644,19 +643,7 @@ struct rte_vdpa_device *
> >   		goto unlock_exit;
> >   	}
> >
> > -	if (vdpa_dev->ops->get_dev_type) {
> > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > -		if (ret) {
> > -			VHOST_LOG_CONFIG(path, ERR,
> > -				"failed to get vdpa dev type for socket
> file.\n");
> > -			ret = -1;
> > -			goto unlock_exit;
> > -		}
> > -	} else {
> > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > -	}
> > -
> > -	*type = vdpa_type;
> > +	*type = vdpa_dev->vdpa_device_type;
> >
> >   unlock_exit:
> >   	pthread_mutex_unlock(&vhost_user.mutex);
> > diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c index
> > bb82857..b487f4d 100644
> > --- a/lib/vhost/vdpa.c
> > +++ b/lib/vhost/vdpa.c
> > @@ -73,6 +73,8 @@ struct rte_vdpa_device *
> >   		struct rte_vdpa_dev_ops *ops)
> >   {
> >   	struct rte_vdpa_device *dev;
> > +	uint32_t vdpa_type = -1;
> > +	int ret = 0;
> >
> >   	if (ops == NULL)
> >   		return NULL;
> > @@ -101,6 +103,21 @@ struct rte_vdpa_device *
> >
> >   	dev->device = rte_dev;
> >   	dev->ops = ops;
> > +
> > +	if (ops->get_dev_type) {
> > +		ret = ops->get_dev_type(dev, &vdpa_type);
> > +		if (ret) {
> > +			VHOST_LOG_CONFIG(rte_dev->name, ERR,
> > +					 "Failed to get vdpa dev type.\n");
> > +			ret = -1;
> > +			goto out_unlock;
> > +		}
> > +	} else {
> > +		/** by default, we assume vdpa device is a net device */
> > +		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > +	}
> > +	dev->vdpa_device_type = vdpa_type;
> > +
> >   	TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
> >   out_unlock:
> >   	rte_spinlock_unlock(&vdpa_device_list_lock);
> > diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h index
> > 8b88a53..c4ec222 100644
> > --- a/lib/vhost/vdpa_driver.h
> > +++ b/lib/vhost/vdpa_driver.h
> > @@ -92,6 +92,8 @@ struct rte_vdpa_device {
> >   	struct rte_device *device;
> >   	/** vdpa device operations */
> >   	struct rte_vdpa_dev_ops *ops;
> > +	/** vdpa device type: net, blk... */
> > +	uint32_t vdpa_device_type;
> 
> I would name it simply "type".
> It is part of rte_vdpa_device struct, no need to duplicate information.
> 
> Doing that, you can use it directly in later patches using 'vdpa_dev->type'.
> 
> Maxime
> 
> >   };
> >
> >   /**

RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Tuesday, October 18, 2022 2:20 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device
> 
> Add vdpa_device_type to rte_vdpa_device to store device type.
> Call vdpa ops get_dev_type to fill vdpa_device_type
> when register vdpa device.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  lib/vhost/socket.c      | 15 +--------------
>  lib/vhost/vdpa.c        | 17 +++++++++++++++++
>  lib/vhost/vdpa_driver.h |  2 ++
>  3 files changed, 20 insertions(+), 14 deletions(-)
> 
> diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c
> index 608ae57..f768114 100644
> --- a/lib/vhost/socket.c
> +++ b/lib/vhost/socket.c
> @@ -627,7 +627,6 @@ struct rte_vdpa_device *
>  {
>  	struct vhost_user_socket *vsocket;
>  	struct rte_vdpa_device *vdpa_dev;
> -	uint32_t vdpa_type = 0;
>  	int ret = 0;
> 
>  	pthread_mutex_lock(&vhost_user.mutex);
> @@ -644,19 +643,7 @@ struct rte_vdpa_device *
>  		goto unlock_exit;
>  	}
> 
> -	if (vdpa_dev->ops->get_dev_type) {
> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> -		if (ret) {
> -			VHOST_LOG_CONFIG(path, ERR,
> -				"failed to get vdpa dev type for socket file.\n");
> -			ret = -1;
> -			goto unlock_exit;
> -		}
> -	} else {
> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> -	}
> -
> -	*type = vdpa_type;
> +	*type = vdpa_dev->vdpa_device_type;
> 
>  unlock_exit:
>  	pthread_mutex_unlock(&vhost_user.mutex);
> diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c
> index bb82857..b487f4d 100644
> --- a/lib/vhost/vdpa.c
> +++ b/lib/vhost/vdpa.c
> @@ -73,6 +73,8 @@ struct rte_vdpa_device *
>  		struct rte_vdpa_dev_ops *ops)
>  {
>  	struct rte_vdpa_device *dev;
> +	uint32_t vdpa_type = -1;
> +	int ret = 0;
> 
>  	if (ops == NULL)
>  		return NULL;
> @@ -101,6 +103,21 @@ struct rte_vdpa_device *
> 
>  	dev->device = rte_dev;
>  	dev->ops = ops;
> +
> +	if (ops->get_dev_type) {
> +		ret = ops->get_dev_type(dev, &vdpa_type);
> +		if (ret) {
> +			VHOST_LOG_CONFIG(rte_dev->name, ERR,
> +					 "Failed to get vdpa dev type.\n");
> +			ret = -1;
> +			goto out_unlock;
> +		}
> +	} else {
> +		/** by default, we assume vdpa device is a net device */
> +		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> +	}
> +	dev->vdpa_device_type = vdpa_type;
> +
>  	TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
>  out_unlock:
>  	rte_spinlock_unlock(&vdpa_device_list_lock);
> diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h
> index 8b88a53..c4ec222 100644
> --- a/lib/vhost/vdpa_driver.h
> +++ b/lib/vhost/vdpa_driver.h
> @@ -92,6 +92,8 @@ struct rte_vdpa_device {
>  	struct rte_device *device;
>  	/** vdpa device operations */
>  	struct rte_vdpa_dev_ops *ops;
> +	/** vdpa device type: net, blk... */
> +	uint32_t vdpa_device_type;
>  };
> 
>  /**
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device

[Pei, Andy]

Hi Chebo,

Thanks for your review.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 19, 2022 5:14 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa device
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Tuesday, October 18, 2022 2:20 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v7 10/12] vhost: add vdpa device type to rte vdpa
> > device
> >
> > Add vdpa_device_type to rte_vdpa_device to store device type.
> > Call vdpa ops get_dev_type to fill vdpa_device_type when register vdpa
> > device.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  lib/vhost/socket.c      | 15 +--------------
> >  lib/vhost/vdpa.c        | 17 +++++++++++++++++
> >  lib/vhost/vdpa_driver.h |  2 ++
> >  3 files changed, 20 insertions(+), 14 deletions(-)
> >
> > diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c index
> > 608ae57..f768114 100644
> > --- a/lib/vhost/socket.c
> > +++ b/lib/vhost/socket.c
> > @@ -627,7 +627,6 @@ struct rte_vdpa_device *  {
> >  	struct vhost_user_socket *vsocket;
> >  	struct rte_vdpa_device *vdpa_dev;
> > -	uint32_t vdpa_type = 0;
> >  	int ret = 0;
> >
> >  	pthread_mutex_lock(&vhost_user.mutex);
> > @@ -644,19 +643,7 @@ struct rte_vdpa_device *
> >  		goto unlock_exit;
> >  	}
> >
> > -	if (vdpa_dev->ops->get_dev_type) {
> > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > -		if (ret) {
> > -			VHOST_LOG_CONFIG(path, ERR,
> > -				"failed to get vdpa dev type for socket
> file.\n");
> > -			ret = -1;
> > -			goto unlock_exit;
> > -		}
> > -	} else {
> > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > -	}
> > -
> > -	*type = vdpa_type;
> > +	*type = vdpa_dev->vdpa_device_type;
> >
> >  unlock_exit:
> >  	pthread_mutex_unlock(&vhost_user.mutex);
> > diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c index
> > bb82857..b487f4d 100644
> > --- a/lib/vhost/vdpa.c
> > +++ b/lib/vhost/vdpa.c
> > @@ -73,6 +73,8 @@ struct rte_vdpa_device *
> >  		struct rte_vdpa_dev_ops *ops)
> >  {
> >  	struct rte_vdpa_device *dev;
> > +	uint32_t vdpa_type = -1;
> > +	int ret = 0;
> >
> >  	if (ops == NULL)
> >  		return NULL;
> > @@ -101,6 +103,21 @@ struct rte_vdpa_device *
> >
> >  	dev->device = rte_dev;
> >  	dev->ops = ops;
> > +
> > +	if (ops->get_dev_type) {
> > +		ret = ops->get_dev_type(dev, &vdpa_type);
> > +		if (ret) {
> > +			VHOST_LOG_CONFIG(rte_dev->name, ERR,
> > +					 "Failed to get vdpa dev type.\n");
> > +			ret = -1;
> > +			goto out_unlock;
> > +		}
> > +	} else {
> > +		/** by default, we assume vdpa device is a net device */
> > +		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > +	}
> > +	dev->vdpa_device_type = vdpa_type;
> > +
> >  	TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
> >  out_unlock:
> >  	rte_spinlock_unlock(&vdpa_device_list_lock);
> > diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h index
> > 8b88a53..c4ec222 100644
> > --- a/lib/vhost/vdpa_driver.h
> > +++ b/lib/vhost/vdpa_driver.h
> > @@ -92,6 +92,8 @@ struct rte_vdpa_device {
> >  	struct rte_device *device;
> >  	/** vdpa device operations */
> >  	struct rte_vdpa_dev_ops *ops;
> > +	/** vdpa device type: net, blk... */
> > +	uint32_t vdpa_device_type;
> >  };
> >
> >  /**
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v7 11/12] vhost: vDPA blk device gets ready when the first queue is ready

[Andy Pei]

When boot from virtio blk device, seabios in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end configure device
when the first queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index cd65257..d5dbd9b 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1441,11 +1441,14 @@
 }
 
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
+#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
 
 static int
 virtio_is_ready(struct virtio_net *dev)
 {
+	struct rte_vdpa_device *vdpa_dev;
 	struct vhost_virtqueue *vq;
+	uint32_t vdpa_type;
 	uint32_t i, nr_vring = dev->nr_vring;
 
 	if (dev->flags & VIRTIO_DEV_READY)
@@ -1454,13 +1457,22 @@
 	if (!dev->nr_vring)
 		return 0;
 
-	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
-		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev)
+		vdpa_type = vdpa_dev->vdpa_device_type;
+	else
+		vdpa_type = -1;
 
-		if (dev->nr_vring < nr_vring)
-			return 0;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
+	} else {
+		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
+			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
 	}
 
+	if (dev->nr_vring < nr_vring)
+		return 0;
+
 	for (i = 0; i < nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
@@ -2958,7 +2970,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	int ret;
 	int unlock_required = 0;
 	bool handled;
-	uint32_t vdpa_type = 0;
+	uint32_t vdpa_type;
 	uint32_t request;
 	uint32_t i;
 
@@ -3170,16 +3182,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
+	vdpa_type = vdpa_dev->vdpa_device_type;
 	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
-- 
1.8.3.1

[PATCH v7 12/12] vhost: improve vDPA blk device configure condition

[Andy Pei]

To support multi-queue, configure device
after call fd of all queues are set.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index d5dbd9b..96383b9 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2973,6 +2973,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	uint32_t vdpa_type;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3183,9 +3184,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 		goto out;
 
 	vdpa_type = vdpa_dev->vdpa_device_type;
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

[PATCH v8 00/12] vdpa/ifc: add multi queue support

[Andy Pei]

v8:
 change "vdpa_device_type" in "rte_vdpa_device" to "type".

v7:
 Fill vdpa_device_type in vdpa device registration.

v6:
 Add vdpa_device_type to rte_vdpa_device to store vDPA device type.

v5:
 fix some commit message.
 rework some code logic.

v4:
 fix some commit message.
 add some commets to code.
 fix some code to reduce confusion.

v3:
 rename device ID macro name.
 fix some patch title and commit message.
 delete some used marco.
 rework some code logic.

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (10):
  vdpa/ifc: add multi-queue support
  vdpa/ifc: set max queues based on virtio spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vdpa/ifc: change internal function name
  vdpa/ifc: add internal API to get device
  vdpa/ifc: change some driver logic
  vhost: add type to rte vdpa device
  vhost: vDPA blk device gets ready when the first queue is ready
  vhost: improve vDPA blk device configure condition

Huang Wei (2):
  vdpa/ifc: add new device ID for legacy network device
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  16 +++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++-------
 lib/vhost/socket.c            |  15 +---
 lib/vhost/vdpa.c              |  15 ++++
 lib/vhost/vdpa_driver.h       |   2 +
 lib/vhost/vhost_user.c        |  38 +++++----
 7 files changed, 354 insertions(+), 61 deletions(-)

-- 
1.8.3.1

[PATCH v8 01/12] vdpa/ifc: add new device ID for legacy network device

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..ef7697a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,12 +12,14 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
-#define IFCVF_BLK_DEVICE_ID                 0x0002
+#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
+#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
 #define IFCVF_MAX_QUEUES		1
 
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index d5ac583..b4389a0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .vendor_id = 0, /* sentinel */
-- 
1.8.3.1

[PATCH v8 02/12] vdpa/ifc: add multi-queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
Expose IFCVF_MQ_OFFSET register to enable multi-queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 12 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..81c68c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,15 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	/* For some hardware implementation, for example:
+	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
+	 * This code makes sure hw->mq_cfg is a valid address.
+	 */
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ef7697a..d16d9ab 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index b4389a0..008cf89 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

[PATCH v8 03/12] vdpa/ifc: set max queues based on virtio spec

[Andy Pei]

Set max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues
in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg,
get the queue pair number using num_queues and set max_queues to it.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index d16d9ab..1e133c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
 #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 008cf89..5a24204 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,12 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MIN() cannot be used since braced-group within expression allowed
+ * only inside a function.
+ */
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		/*
+		 * ifc device always has CTRL_VQ,
+		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
+		 */
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

[PATCH v8 04/12] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi-queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 81c68c0..b377126 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -202,6 +202,37 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_mq(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	u8 qid;
+	int nr_queue = 0;
+
+	for (qid = 0; qid < hw->nr_vring; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
+		nr_queue++;
+	}
+
+	if (nr_queue == 0) {
+		WARNINGOUT("no enabled vring\n");
+		return;
+	}
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			*(u32 *)mq_cfg = nr_queue;
+			RTE_LOG(INFO, PMD, "%d queues are enabled\n", nr_queue);
+		} else {
+			*(u32 *)mq_cfg = nr_queue / 2;
+			RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+				nr_queue / 2);
+		}
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -219,6 +250,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

[PATCH v8 05/12] vdpa/ifc: only configure enabled queue

[Andy Pei]

When configuring the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  3 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index b377126..30bb8cb 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -252,6 +252,9 @@
 
 	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 5a24204..0c3407a 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v8 06/12] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |   6 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c |  93 ++++++++++++++++++++++++++++++++-------
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 30bb8cb..869ddd6 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -233,6 +233,106 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (i >= (int)hw->nr_vring)
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_mq(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			RTE_LOG(ERR, PMD, "queue %d, msix vec alloc failed\n",
+				i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (i >= (int)hw->nr_vring)
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		} else {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		}
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 1e133c0..3726da7 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 0c3407a..9c49f9c 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v8 07/12] vdpa/ifc: change internal function name

[Andy Pei]

Change internal function name "find_internal_resource_by_dev"
to "find_internal_resource_by_pci_dev".

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 9c49f9c..73d04ed 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info {
 }
 
 static struct internal_list *
-find_internal_resource_by_dev(struct rte_pci_device *pdev)
+find_internal_resource_by_pci_dev(struct rte_pci_device *pdev)
 {
 	int found = 0;
 	struct internal_list *list;
@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
-	list = find_internal_resource_by_dev(pci_dev);
+	list = find_internal_resource_by_pci_dev(pci_dev);
 	if (list == NULL) {
 		DRV_LOG(ERR, "Invalid device: %s", pci_dev->name);
 		return -1;
-- 
1.8.3.1

Re: [PATCH v8 07/12] vdpa/ifc: change internal function name

[Maxime Coquelin]

On 10/18/22 14:07, Andy Pei wrote:
> Change internal function name "find_internal_resource_by_dev"
> to "find_internal_resource_by_pci_dev".
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++--
>   1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 9c49f9c..73d04ed 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -121,7 +121,7 @@ struct rte_vdpa_dev_info {
>   }
>   
>   static struct internal_list *
> -find_internal_resource_by_dev(struct rte_pci_device *pdev)
> +find_internal_resource_by_pci_dev(struct rte_pci_device *pdev)
>   {
>   	int found = 0;
>   	struct internal_list *list;
> @@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {
>   	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
>   		return 0;
>   
> -	list = find_internal_resource_by_dev(pci_dev);
> +	list = find_internal_resource_by_pci_dev(pci_dev);
>   	if (list == NULL) {
>   		DRV_LOG(ERR, "Invalid device: %s", pci_dev->name);
>   		return -1;

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks,
Maxime

RE: [PATCH v8 07/12] vdpa/ifc: change internal function name

[Pei, Andy]

Hi Maxime,

Thanks for your efforts.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Tuesday, October 18, 2022 9:45 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v8 07/12] vdpa/ifc: change internal function name
> 
> 
> 
> On 10/18/22 14:07, Andy Pei wrote:
> > Change internal function name "find_internal_resource_by_dev"
> > to "find_internal_resource_by_pci_dev".
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >   drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++--
> >   1 file changed, 2 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 9c49f9c..73d04ed 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -121,7 +121,7 @@ struct rte_vdpa_dev_info {
> >   }
> >
> >   static struct internal_list *
> > -find_internal_resource_by_dev(struct rte_pci_device *pdev)
> > +find_internal_resource_by_pci_dev(struct rte_pci_device *pdev)
> >   {
> >   	int found = 0;
> >   	struct internal_list *list;
> > @@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {
> >   	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> >   		return 0;
> >
> > -	list = find_internal_resource_by_dev(pci_dev);
> > +	list = find_internal_resource_by_pci_dev(pci_dev);
> >   	if (list == NULL) {
> >   		DRV_LOG(ERR, "Invalid device: %s", pci_dev->name);
> >   		return -1;
> 
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> 
> Thanks,
> Maxime

RE: [PATCH v8 07/12] vdpa/ifc: change internal function name

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Tuesday, October 18, 2022 8:08 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v8 07/12] vdpa/ifc: change internal function name
> 
> Change internal function name "find_internal_resource_by_dev"
> to "find_internal_resource_by_pci_dev".
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 9c49f9c..73d04ed 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -121,7 +121,7 @@ struct rte_vdpa_dev_info {
>  }
> 
>  static struct internal_list *
> -find_internal_resource_by_dev(struct rte_pci_device *pdev)
> +find_internal_resource_by_pci_dev(struct rte_pci_device *pdev)
>  {
>  	int found = 0;
>  	struct internal_list *list;
> @@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
>  		return 0;
> 
> -	list = find_internal_resource_by_dev(pci_dev);
> +	list = find_internal_resource_by_pci_dev(pci_dev);
>  	if (list == NULL) {
>  		DRV_LOG(ERR, "Invalid device: %s", pci_dev->name);
>  		return -1;
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v8 07/12] vdpa/ifc: change internal function name

[Pei, Andy]

HI Chenbo,

Thanks for your review.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 19, 2022 2:59 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v8 07/12] vdpa/ifc: change internal function name
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Tuesday, October 18, 2022 8:08 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v8 07/12] vdpa/ifc: change internal function name
> >
> > Change internal function name "find_internal_resource_by_dev"
> > to "find_internal_resource_by_pci_dev".
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++--
> >  1 file changed, 2 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 9c49f9c..73d04ed 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -121,7 +121,7 @@ struct rte_vdpa_dev_info {  }
> >
> >  static struct internal_list *
> > -find_internal_resource_by_dev(struct rte_pci_device *pdev)
> > +find_internal_resource_by_pci_dev(struct rte_pci_device *pdev)
> >  {
> >  	int found = 0;
> >  	struct internal_list *list;
> > @@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> >  		return 0;
> >
> > -	list = find_internal_resource_by_dev(pci_dev);
> > +	list = find_internal_resource_by_pci_dev(pci_dev);
> >  	if (list == NULL) {
> >  		DRV_LOG(ERR, "Invalid device: %s", pci_dev->name);
> >  		return -1;
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v8 08/12] vdpa/ifc: add internal API to get device

[Andy Pei]

Add new internal API "find_internal_resource_by_rte_dev"
to get device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 73d04ed..c16e263 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {
 	return list;
 }
 
+static struct internal_list *
+find_internal_resource_by_rte_dev(struct rte_device *rte_dev)
+{
+	int found = 0;
+	struct internal_list *list;
+
+	pthread_mutex_lock(&internal_list_lock);
+
+	TAILQ_FOREACH(list, &internal_list, next) {
+		if (rte_dev == &list->internal->pdev->device) {
+			found = 1;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&internal_list_lock);
+
+	if (!found)
+		return NULL;
+
+	return list;
+}
+
 static int
 ifcvf_vfio_setup(struct ifcvf_internal *internal)
 {
@@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info {
 {
 	struct ifcvf_internal *internal;
 	struct internal_list *list;
+	struct rte_device *rte_dev = vdev->device;
 
-	list = find_internal_resource_by_vdev(vdev);
+	list = find_internal_resource_by_rte_dev(rte_dev);
 	if (list == NULL) {
-		DRV_LOG(ERR, "Invalid vDPA device: %p", vdev);
+		DRV_LOG(ERR, "Invalid rte device: %p", rte_dev);
 		return -1;
 	}
 
-- 
1.8.3.1

Re: [PATCH v8 08/12] vdpa/ifc: add internal API to get device

[Maxime Coquelin]

On 10/18/22 14:07, Andy Pei wrote:
> Add new internal API "find_internal_resource_by_rte_dev"
> to get device.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++--
>   1 file changed, 26 insertions(+), 2 deletions(-)
> 

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks,
Maxime

RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get device

[Pei, Andy]

Hi Maxime,

Thanks for your efforts.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Tuesday, October 18, 2022 9:48 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
> 
> 
> 
> On 10/18/22 14:07, Andy Pei wrote:
> > Add new internal API "find_internal_resource_by_rte_dev"
> > to get device.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >   drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++--
> >   1 file changed, 26 insertions(+), 2 deletions(-)
> >
> 
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> 
> Thanks,
> Maxime

RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get device

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Tuesday, October 18, 2022 8:08 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
> 
> Add new internal API "find_internal_resource_by_rte_dev"
> to get device.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++--
>  1 file changed, 26 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 73d04ed..c16e263 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {
>  	return list;
>  }
> 
> +static struct internal_list *
> +find_internal_resource_by_rte_dev(struct rte_device *rte_dev)
> +{
> +	int found = 0;
> +	struct internal_list *list;
> +
> +	pthread_mutex_lock(&internal_list_lock);
> +
> +	TAILQ_FOREACH(list, &internal_list, next) {
> +		if (rte_dev == &list->internal->pdev->device) {
> +			found = 1;
> +			break;
> +		}
> +	}
> +
> +	pthread_mutex_unlock(&internal_list_lock);
> +
> +	if (!found)
> +		return NULL;
> +
> +	return list;
> +}
> +
>  static int
>  ifcvf_vfio_setup(struct ifcvf_internal *internal)
>  {
> @@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info {
>  {
>  	struct ifcvf_internal *internal;
>  	struct internal_list *list;
> +	struct rte_device *rte_dev = vdev->device;
> 
> -	list = find_internal_resource_by_vdev(vdev);
> +	list = find_internal_resource_by_rte_dev(rte_dev);
>  	if (list == NULL) {
> -		DRV_LOG(ERR, "Invalid vDPA device: %p", vdev);
> +		DRV_LOG(ERR, "Invalid rte device: %p", rte_dev);
>  		return -1;
>  	}
> 
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get device

[Pei, Andy]

Hi Chenbo, 
Thanks for your review.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 19, 2022 3:04 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Tuesday, October 18, 2022 8:08 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v8 08/12] vdpa/ifc: add internal API to get device
> >
> > Add new internal API "find_internal_resource_by_rte_dev"
> > to get device.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++--
> >  1 file changed, 26 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index 73d04ed..c16e263 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {
> >  	return list;
> >  }
> >
> > +static struct internal_list *
> > +find_internal_resource_by_rte_dev(struct rte_device *rte_dev) {
> > +	int found = 0;
> > +	struct internal_list *list;
> > +
> > +	pthread_mutex_lock(&internal_list_lock);
> > +
> > +	TAILQ_FOREACH(list, &internal_list, next) {
> > +		if (rte_dev == &list->internal->pdev->device) {
> > +			found = 1;
> > +			break;
> > +		}
> > +	}
> > +
> > +	pthread_mutex_unlock(&internal_list_lock);
> > +
> > +	if (!found)
> > +		return NULL;
> > +
> > +	return list;
> > +}
> > +
> >  static int
> >  ifcvf_vfio_setup(struct ifcvf_internal *internal)  { @@ -1398,10
> > +1421,11 @@ struct rte_vdpa_dev_info {  {
> >  	struct ifcvf_internal *internal;
> >  	struct internal_list *list;
> > +	struct rte_device *rte_dev = vdev->device;
> >
> > -	list = find_internal_resource_by_vdev(vdev);
> > +	list = find_internal_resource_by_rte_dev(rte_dev);
> >  	if (list == NULL) {
> > -		DRV_LOG(ERR, "Invalid vDPA device: %p", vdev);
> > +		DRV_LOG(ERR, "Invalid rte device: %p", rte_dev);
> >  		return -1;
> >  	}
> >
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v8 09/12] vdpa/ifc: change some driver logic

[Andy Pei]

Insert internal list element to internal list before
register vdpa device, in order to call vdpa ops during
vdpa device registration.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index c16e263..8dfd493 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 	internal->sw_lm = sw_fallback_lm;
 
+	pthread_mutex_lock(&internal_list_lock);
+	TAILQ_INSERT_TAIL(&internal_list, list, next);
+	pthread_mutex_unlock(&internal_list_lock);
+
 	internal->vdev = rte_vdpa_register_device(&pci_dev->device,
 				dev_info[internal->hw.device_type].ops);
 	if (internal->vdev == NULL) {
 		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
+		pthread_mutex_lock(&internal_list_lock);
+		TAILQ_REMOVE(&internal_list, list, next);
+		pthread_mutex_unlock(&internal_list_lock);
 		goto error;
 	}
 
-	pthread_mutex_lock(&internal_list_lock);
-	TAILQ_INSERT_TAIL(&internal_list, list, next);
-	pthread_mutex_unlock(&internal_list_lock);
-
 	rte_atomic32_set(&internal->started, 1);
 	update_datapath(internal);
 
-- 
1.8.3.1

Re: [PATCH v8 09/12] vdpa/ifc: change some driver logic

[Maxime Coquelin]

On 10/18/22 14:07, Andy Pei wrote:
> Insert internal list element to internal list before
> register vdpa device, in order to call vdpa ops during
> vdpa device registration.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
>   1 file changed, 7 insertions(+), 4 deletions(-)
> 

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks,
Maxime

RE: [PATCH v8 09/12] vdpa/ifc: change some driver logic

[Pei, Andy]

Hi Maxime,

Thanks.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Tuesday, October 18, 2022 9:57 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v8 09/12] vdpa/ifc: change some driver logic
> 
> 
> 
> On 10/18/22 14:07, Andy Pei wrote:
> > Insert internal list element to internal list before register vdpa
> > device, in order to call vdpa ops during vdpa device registration.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >   drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
> >   1 file changed, 7 insertions(+), 4 deletions(-)
> >
> 
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> 
> Thanks,
> Maxime

RE: [PATCH v8 09/12] vdpa/ifc: change some driver logic

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Tuesday, October 18, 2022 8:08 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v8 09/12] vdpa/ifc: change some driver logic
> 
> Insert internal list element to internal list before
> register vdpa device, in order to call vdpa ops during
> vdpa device registration.

Patch seems good but title may be too general..

Maybe: improve internal list logic ?

Thanks,
Chenbo

> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
>  1 file changed, 7 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index c16e263..8dfd493 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	}
>  	internal->sw_lm = sw_fallback_lm;
> 
> +	pthread_mutex_lock(&internal_list_lock);
> +	TAILQ_INSERT_TAIL(&internal_list, list, next);
> +	pthread_mutex_unlock(&internal_list_lock);
> +
>  	internal->vdev = rte_vdpa_register_device(&pci_dev->device,
>  				dev_info[internal->hw.device_type].ops);
>  	if (internal->vdev == NULL) {
>  		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
> +		pthread_mutex_lock(&internal_list_lock);
> +		TAILQ_REMOVE(&internal_list, list, next);
> +		pthread_mutex_unlock(&internal_list_lock);
>  		goto error;
>  	}
> 
> -	pthread_mutex_lock(&internal_list_lock);
> -	TAILQ_INSERT_TAIL(&internal_list, list, next);
> -	pthread_mutex_unlock(&internal_list_lock);
> -
>  	rte_atomic32_set(&internal->started, 1);
>  	update_datapath(internal);
> 
> --
> 1.8.3.1

RE: [PATCH v8 09/12] vdpa/ifc: change some driver logic

[Pei, Andy]

Hi Chenbo,

Thanks for your review, I will send v9 to do this.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 19, 2022 5:13 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v8 09/12] vdpa/ifc: change some driver logic
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Tuesday, October 18, 2022 8:08 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v8 09/12] vdpa/ifc: change some driver logic
> >
> > Insert internal list element to internal list before register vdpa
> > device, in order to call vdpa ops during vdpa device registration.
> 
> Patch seems good but title may be too general..
> 
> Maybe: improve internal list logic ?
> 
> Thanks,
> Chenbo
> 
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
> >  1 file changed, 7 insertions(+), 4 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index c16e263..8dfd493 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	}
> >  	internal->sw_lm = sw_fallback_lm;
> >
> > +	pthread_mutex_lock(&internal_list_lock);
> > +	TAILQ_INSERT_TAIL(&internal_list, list, next);
> > +	pthread_mutex_unlock(&internal_list_lock);
> > +
> >  	internal->vdev = rte_vdpa_register_device(&pci_dev->device,
> >  				dev_info[internal->hw.device_type].ops);
> >  	if (internal->vdev == NULL) {
> >  		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
> > +		pthread_mutex_lock(&internal_list_lock);
> > +		TAILQ_REMOVE(&internal_list, list, next);
> > +		pthread_mutex_unlock(&internal_list_lock);
> >  		goto error;
> >  	}
> >
> > -	pthread_mutex_lock(&internal_list_lock);
> > -	TAILQ_INSERT_TAIL(&internal_list, list, next);
> > -	pthread_mutex_unlock(&internal_list_lock);
> > -
> >  	rte_atomic32_set(&internal->started, 1);
> >  	update_datapath(internal);
> >
> > --
> > 1.8.3.1

[PATCH v8 10/12] vhost: add type to rte vdpa device

[Andy Pei]

Add type to rte_vdpa_device to store device type.
Call vdpa ops get_dev_type to fill type when register
vdpa device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 lib/vhost/socket.c      | 15 +--------------
 lib/vhost/vdpa.c        | 15 +++++++++++++++
 lib/vhost/vdpa_driver.h |  2 ++
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c
index 608ae57..863a6f6 100644
--- a/lib/vhost/socket.c
+++ b/lib/vhost/socket.c
@@ -627,7 +627,6 @@ struct rte_vdpa_device *
 {
 	struct vhost_user_socket *vsocket;
 	struct rte_vdpa_device *vdpa_dev;
-	uint32_t vdpa_type = 0;
 	int ret = 0;
 
 	pthread_mutex_lock(&vhost_user.mutex);
@@ -644,19 +643,7 @@ struct rte_vdpa_device *
 		goto unlock_exit;
 	}
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(path, ERR,
-				"failed to get vdpa dev type for socket file.\n");
-			ret = -1;
-			goto unlock_exit;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
-
-	*type = vdpa_type;
+	*type = vdpa_dev->type;
 
 unlock_exit:
 	pthread_mutex_unlock(&vhost_user.mutex);
diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c
index bb82857..577cb00 100644
--- a/lib/vhost/vdpa.c
+++ b/lib/vhost/vdpa.c
@@ -73,6 +73,7 @@ struct rte_vdpa_device *
 		struct rte_vdpa_dev_ops *ops)
 {
 	struct rte_vdpa_device *dev;
+	int ret = 0;
 
 	if (ops == NULL)
 		return NULL;
@@ -101,6 +102,20 @@ struct rte_vdpa_device *
 
 	dev->device = rte_dev;
 	dev->ops = ops;
+
+	if (ops->get_dev_type) {
+		ret = ops->get_dev_type(dev, &dev->type);
+		if (ret) {
+			VHOST_LOG_CONFIG(rte_dev->name, ERR,
+					 "Failed to get vdpa dev type.\n");
+			ret = -1;
+			goto out_unlock;
+		}
+	} else {
+		/** by default, we assume vdpa device is a net device */
+		dev->type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+	}
+
 	TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
 out_unlock:
 	rte_spinlock_unlock(&vdpa_device_list_lock);
diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h
index 8b88a53..8db4ab9 100644
--- a/lib/vhost/vdpa_driver.h
+++ b/lib/vhost/vdpa_driver.h
@@ -92,6 +92,8 @@ struct rte_vdpa_device {
 	struct rte_device *device;
 	/** vdpa device operations */
 	struct rte_vdpa_dev_ops *ops;
+	/** vdpa device type: net, blk... */
+	uint32_t type;
 };
 
 /**
-- 
1.8.3.1

Re: [PATCH v8 10/12] vhost: add type to rte vdpa device

[Maxime Coquelin]

On 10/18/22 14:07, Andy Pei wrote:
> Add type to rte_vdpa_device to store device type.
> Call vdpa ops get_dev_type to fill type when register
> vdpa device.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>   lib/vhost/socket.c      | 15 +--------------
>   lib/vhost/vdpa.c        | 15 +++++++++++++++
>   lib/vhost/vdpa_driver.h |  2 ++
>   3 files changed, 18 insertions(+), 14 deletions(-)
> 

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks for doing the change!
Maxime

RE: [PATCH v8 10/12] vhost: add type to rte vdpa device

[Pei, Andy]

Hi Maxime,

Thanks for your efforts.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Tuesday, October 18, 2022 9:15 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v8 10/12] vhost: add type to rte vdpa device
> 
> 
> 
> On 10/18/22 14:07, Andy Pei wrote:
> > Add type to rte_vdpa_device to store device type.
> > Call vdpa ops get_dev_type to fill type when register vdpa device.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >   lib/vhost/socket.c      | 15 +--------------
> >   lib/vhost/vdpa.c        | 15 +++++++++++++++
> >   lib/vhost/vdpa_driver.h |  2 ++
> >   3 files changed, 18 insertions(+), 14 deletions(-)
> >
> 
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> 
> Thanks for doing the change!
> Maxime

[PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready

[Andy Pei]

When boot from virtio blk device, seabios in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end configure device
when the first queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index cd65257..e0ff79d 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1441,11 +1441,14 @@
 }
 
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
+#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
 
 static int
 virtio_is_ready(struct virtio_net *dev)
 {
+	struct rte_vdpa_device *vdpa_dev;
 	struct vhost_virtqueue *vq;
+	uint32_t vdpa_type;
 	uint32_t i, nr_vring = dev->nr_vring;
 
 	if (dev->flags & VIRTIO_DEV_READY)
@@ -1454,13 +1457,22 @@
 	if (!dev->nr_vring)
 		return 0;
 
-	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
-		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev)
+		vdpa_type = vdpa_dev->type;
+	else
+		vdpa_type = -1;
 
-		if (dev->nr_vring < nr_vring)
-			return 0;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
+	} else {
+		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
+			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
 	}
 
+	if (dev->nr_vring < nr_vring)
+		return 0;
+
 	for (i = 0; i < nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
@@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	int ret;
 	int unlock_required = 0;
 	bool handled;
-	uint32_t vdpa_type = 0;
 	uint32_t request;
 	uint32_t i;
 
@@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
+	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
 
-- 
1.8.3.1

Re: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready

[Maxime Coquelin]

On 10/18/22 14:07, Andy Pei wrote:
> When boot from virtio blk device, seabios in QEMU only enables one queue.
> To work in this scenario, vDPA BLK device back-end configure device
> when the first queue is ready.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>   lib/vhost/vhost_user.c | 33 +++++++++++++++++----------------
>   1 file changed, 17 insertions(+), 16 deletions(-)
> 

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks,
Maxime

RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready

[Pei, Andy]

HI Maxime,

Thanks for your review.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Tuesday, October 18, 2022 10:09 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the
> first queue is ready
> 
> 
> 
> On 10/18/22 14:07, Andy Pei wrote:
> > When boot from virtio blk device, seabios in QEMU only enables one queue.
> > To work in this scenario, vDPA BLK device back-end configure device
> > when the first queue is ready.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >   lib/vhost/vhost_user.c | 33 +++++++++++++++++----------------
> >   1 file changed, 17 insertions(+), 16 deletions(-)
> >
> 
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> 
> Thanks,
> Maxime

RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Tuesday, October 18, 2022 8:08 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first
> queue is ready
> 
> When boot from virtio blk device, seabios in QEMU only enables one queue.
> To work in this scenario, vDPA BLK device back-end configure device
> when the first queue is ready.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 33 +++++++++++++++++----------------
>  1 file changed, 17 insertions(+), 16 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index cd65257..e0ff79d 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -1441,11 +1441,14 @@
>  }
> 
>  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> 
>  static int
>  virtio_is_ready(struct virtio_net *dev)
>  {
> +	struct rte_vdpa_device *vdpa_dev;
>  	struct vhost_virtqueue *vq;
> +	uint32_t vdpa_type;
>  	uint32_t i, nr_vring = dev->nr_vring;
> 
>  	if (dev->flags & VIRTIO_DEV_READY)
> @@ -1454,13 +1457,22 @@
>  	if (!dev->nr_vring)
>  		return 0;
> 
> -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> +	vdpa_dev = dev->vdpa_dev;
> +	if (vdpa_dev)
> +		vdpa_type = vdpa_dev->type;
> +	else
> +		vdpa_type = -1;
> 
> -		if (dev->nr_vring < nr_vring)
> -			return 0;
> +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> +	} else {
> +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> +			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
>  	}
> 
> +	if (dev->nr_vring < nr_vring)
> +		return 0;
> +
>  	for (i = 0; i < nr_vring; i++) {
>  		vq = dev->virtqueue[i];
> 
> @@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	int ret;
>  	int unlock_required = 0;
>  	bool handled;
> -	uint32_t vdpa_type = 0;
>  	uint32_t request;
>  	uint32_t i;
> 
> @@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	if (!vdpa_dev)
>  		goto out;
> 
> -	if (vdpa_dev->ops->get_dev_type) {
> -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> -		if (ret) {
> -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa
> dev type.\n");
> -			ret = -1;
> -			goto out;
> -		}
> -	} else {
> -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> -	}
> -	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> +	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
>  		&& request != VHOST_USER_SET_VRING_CALL)
>  		goto out;
> 
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the first queue is ready

[Pei, Andy]

Hi Chenbo,

Thanks for your review.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 19, 2022 5:15 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the
> first queue is ready
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Tuesday, October 18, 2022 8:08 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v8 11/12] vhost: vDPA blk device gets ready when the
> > first queue is ready
> >
> > When boot from virtio blk device, seabios in QEMU only enables one queue.
> > To work in this scenario, vDPA BLK device back-end configure device
> > when the first queue is ready.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  lib/vhost/vhost_user.c | 33 +++++++++++++++++----------------
> >  1 file changed, 17 insertions(+), 16 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > cd65257..e0ff79d 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -1441,11 +1441,14 @@
> >  }
> >
> >  #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
> > +#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
> >
> >  static int
> >  virtio_is_ready(struct virtio_net *dev)  {
> > +	struct rte_vdpa_device *vdpa_dev;
> >  	struct vhost_virtqueue *vq;
> > +	uint32_t vdpa_type;
> >  	uint32_t i, nr_vring = dev->nr_vring;
> >
> >  	if (dev->flags & VIRTIO_DEV_READY)
> > @@ -1454,13 +1457,22 @@
> >  	if (!dev->nr_vring)
> >  		return 0;
> >
> > -	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
> > -		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> > +	vdpa_dev = dev->vdpa_dev;
> > +	if (vdpa_dev)
> > +		vdpa_type = vdpa_dev->type;
> > +	else
> > +		vdpa_type = -1;
> >
> > -		if (dev->nr_vring < nr_vring)
> > -			return 0;
> > +	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > +		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
> > +	} else {
> > +		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
> > +			nr_vring =
> VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
> >  	}
> >
> > +	if (dev->nr_vring < nr_vring)
> > +		return 0;
> > +
> >  	for (i = 0; i < nr_vring; i++) {
> >  		vq = dev->virtqueue[i];
> >
> > @@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	int ret;
> >  	int unlock_required = 0;
> >  	bool handled;
> > -	uint32_t vdpa_type = 0;
> >  	uint32_t request;
> >  	uint32_t i;
> >
> > @@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	if (!vdpa_dev)
> >  		goto out;
> >
> > -	if (vdpa_dev->ops->get_dev_type) {
> > -		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
> > -		if (ret) {
> > -			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to
> get vdpa
> > dev type.\n");
> > -			ret = -1;
> > -			goto out;
> > -		}
> > -	} else {
> > -		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
> > -	}
> > -	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > +	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> >  		&& request != VHOST_USER_SET_VRING_CALL)
> >  		goto out;
> >
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v8 12/12] vhost: improve vDPA blk device configure condition

[Andy Pei]

To support multi-queue, configure device
after call fd of all queues are set.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index e0ff79d..9902ae9 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	bool handled;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

Re: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition

[Maxime Coquelin]

On 10/18/22 14:07, Andy Pei wrote:
> To support multi-queue, configure device
> after call fd of all queues are set.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>   lib/vhost/vhost_user.c | 13 ++++++++++---
>   1 file changed, 10 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index e0ff79d..9902ae9 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
>   	bool handled;
>   	uint32_t request;
>   	uint32_t i;
> +	uint16_t blk_call_fd;
>   
>   	dev = get_device(vid);
>   	if (dev == NULL)
> @@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
>   	if (!vdpa_dev)
>   		goto out;
>   
> -	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> -		&& request != VHOST_USER_SET_VRING_CALL)
> -		goto out;
> +	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +		if (request == VHOST_USER_SET_VRING_CALL) {
> +			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
> +			if (blk_call_fd != dev->nr_vring - 1)
> +				goto out;
> +		} else {
> +			goto out;
> +		}
> +	}
>   
>   	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
>   		if (vdpa_dev->ops->dev_conf(dev->vid))

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks,
Maxime

RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition

[Pei, Andy]

Hi  Maxime,

Thanks for your review.

> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Tuesday, October 18, 2022 10:15 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xia, Chenbo <Chenbo.Xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>
> Subject: Re: [PATCH v8 12/12] vhost: improve vDPA blk device configure
> condition
> 
> 
> 
> On 10/18/22 14:07, Andy Pei wrote:
> > To support multi-queue, configure device after call fd of all queues
> > are set.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >   lib/vhost/vhost_user.c | 13 ++++++++++---
> >   1 file changed, 10 insertions(+), 3 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > e0ff79d..9902ae9 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >   	bool handled;
> >   	uint32_t request;
> >   	uint32_t i;
> > +	uint16_t blk_call_fd;
> >
> >   	dev = get_device(vid);
> >   	if (dev == NULL)
> > @@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >   	if (!vdpa_dev)
> >   		goto out;
> >
> > -	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > -		&& request != VHOST_USER_SET_VRING_CALL)
> > -		goto out;
> > +	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > +		if (request == VHOST_USER_SET_VRING_CALL) {
> > +			blk_call_fd = ctx.msg.payload.u64 &
> VHOST_USER_VRING_IDX_MASK;
> > +			if (blk_call_fd != dev->nr_vring - 1)
> > +				goto out;
> > +		} else {
> > +			goto out;
> > +		}
> > +	}
> >
> >   	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
> >   		if (vdpa_dev->ops->dev_conf(dev->vid))
> 
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> 
> Thanks,
> Maxime

RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Tuesday, October 18, 2022 8:08 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v8 12/12] vhost: improve vDPA blk device configure
> condition
> 
> To support multi-queue, configure device
> after call fd of all queues are set.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Signed-off-by: Huang Wei <wei.huang@intel.com>
> ---
>  lib/vhost/vhost_user.c | 13 ++++++++++---
>  1 file changed, 10 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index e0ff79d..9902ae9 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	bool handled;
>  	uint32_t request;
>  	uint32_t i;
> +	uint16_t blk_call_fd;
> 
>  	dev = get_device(vid);
>  	if (dev == NULL)
> @@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
>  	if (!vdpa_dev)
>  		goto out;
> 
> -	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> -		&& request != VHOST_USER_SET_VRING_CALL)
> -		goto out;
> +	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> +		if (request == VHOST_USER_SET_VRING_CALL) {
> +			blk_call_fd = ctx.msg.payload.u64 &
> VHOST_USER_VRING_IDX_MASK;
> +			if (blk_call_fd != dev->nr_vring - 1)
> +				goto out;
> +		} else {
> +			goto out;
> +		}
> +	}
> 
>  	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
>  		if (vdpa_dev->ops->dev_conf(dev->vid))
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure condition

[Pei, Andy]

Hi Chenbo,

Thanks for your review.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 19, 2022 5:15 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v8 12/12] vhost: improve vDPA blk device configure
> condition
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Tuesday, October 18, 2022 8:08 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v8 12/12] vhost: improve vDPA blk device configure
> > condition
> >
> > To support multi-queue, configure device after call fd of all queues
> > are set.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Signed-off-by: Huang Wei <wei.huang@intel.com>
> > ---
> >  lib/vhost/vhost_user.c | 13 ++++++++++---
> >  1 file changed, 10 insertions(+), 3 deletions(-)
> >
> > diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index
> > e0ff79d..9902ae9 100644
> > --- a/lib/vhost/vhost_user.c
> > +++ b/lib/vhost/vhost_user.c
> > @@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	bool handled;
> >  	uint32_t request;
> >  	uint32_t i;
> > +	uint16_t blk_call_fd;
> >
> >  	dev = get_device(vid);
> >  	if (dev == NULL)
> > @@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
> >  	if (!vdpa_dev)
> >  		goto out;
> >
> > -	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
> > -		&& request != VHOST_USER_SET_VRING_CALL)
> > -		goto out;
> > +	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
> > +		if (request == VHOST_USER_SET_VRING_CALL) {
> > +			blk_call_fd = ctx.msg.payload.u64 &
> > VHOST_USER_VRING_IDX_MASK;
> > +			if (blk_call_fd != dev->nr_vring - 1)
> > +				goto out;
> > +		} else {
> > +			goto out;
> > +		}
> > +	}
> >
> >  	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
> >  		if (vdpa_dev->ops->dev_conf(dev->vid))
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v9 00/12] vdpa/ifc: add multi queue support

[Andy Pei]

v9:
 fix some commit message.

v8:
 change "vdpa_device_type" in "rte_vdpa_device" to "type".

v7:
 Fill vdpa_device_type in vdpa device registration.

v6:
 Add vdpa_device_type to rte_vdpa_device to store vDPA device type.

v5:
 fix some commit message.
 rework some code logic.

v4:
 fix some commit message.
 add some commets to code.
 fix some code to reduce confusion.

v3:
 rename device ID macro name.
 fix some patch title and commit message.
 delete some used marco.
 rework some code logic.

v2:
 fix some coding style issue.
 support dynamic enable/disable queue at run time.

Andy Pei (10):
  vdpa/ifc: add multi-queue support
  vdpa/ifc: set max queues based on virtio spec
  vdpa/ifc: write queue count to MQ register
  vdpa/ifc: only configure enabled queue
  vdpa/ifc: change internal function name
  vdpa/ifc: add internal API to get device
  vdpa/ifc: improve internal list logic
  vhost: add type to rte vdpa device
  vhost: vDPA blk device gets ready when the first queue is ready
  vhost: improve vDPA blk device configure condition

Huang Wei (2):
  vdpa/ifc: add new device ID for legacy network device
  vdpa/ifc: support dynamic enable/disable queue

 drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |  16 +++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++-------
 lib/vhost/socket.c            |  15 +---
 lib/vhost/vdpa.c              |  15 ++++
 lib/vhost/vdpa_driver.h       |   2 +
 lib/vhost/vhost_user.c        |  38 +++++----
 7 files changed, 354 insertions(+), 61 deletions(-)

-- 
1.8.3.1

[PATCH v9 01/12] vdpa/ifc: add new device ID for legacy network device

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Add new device id to support IFCVF_NET_TRANSITIONAL_DEVICE_ID (0x1000).
Rename macro from "IFCVF_BLK_DEVICE_ID" to "IFCVF_SUBSYS_BLK_DEVICE_ID".

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  6 ++++--
 drivers/vdpa/ifc/ifcvf_vdpa.c | 13 ++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 9d95aac..ef7697a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -12,12 +12,14 @@
 #define IFCVF_BLK	1
 
 #define IFCVF_VENDOR_ID                     0x1AF4
-#define IFCVF_NET_DEVICE_ID                 0x1041
+#define IFCVF_NET_MODERN_DEVICE_ID          0x1041
 #define IFCVF_BLK_MODERN_DEVICE_ID          0x1042
+#define IFCVF_NET_TRANSITIONAL_DEVICE_ID    0x1000
 #define IFCVF_BLK_TRANSITIONAL_DEVICE_ID    0x1001
 #define IFCVF_SUBSYS_VENDOR_ID              0x8086
 #define IFCVF_SUBSYS_DEVICE_ID              0x001A
-#define IFCVF_BLK_DEVICE_ID                 0x0002
+#define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
+#define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
 #define IFCVF_MAX_QUEUES		1
 
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index d5ac583..b4389a0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1684,23 +1684,30 @@ struct rte_vdpa_dev_info dev_info[] = {
 static const struct rte_pci_id pci_id_ifcvf_map[] = {
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
-	  .device_id = IFCVF_NET_DEVICE_ID,
+	  .device_id = IFCVF_NET_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
 	  .subsystem_device_id = IFCVF_SUBSYS_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
+	  .device_id = IFCVF_NET_TRANSITIONAL_DEVICE_ID,
+	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_NET_DEVICE_ID,
+	},
+
+	{ .class_id = RTE_CLASS_ANY_ID,
+	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_TRANSITIONAL_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .class_id = RTE_CLASS_ANY_ID,
 	  .vendor_id = IFCVF_VENDOR_ID,
 	  .device_id = IFCVF_BLK_MODERN_DEVICE_ID,
 	  .subsystem_vendor_id = IFCVF_SUBSYS_VENDOR_ID,
-	  .subsystem_device_id = IFCVF_BLK_DEVICE_ID,
+	  .subsystem_device_id = IFCVF_SUBSYS_BLK_DEVICE_ID,
 	},
 
 	{ .vendor_id = 0, /* sentinel */
-- 
1.8.3.1

[PATCH v9 02/12] vdpa/ifc: add multi-queue support

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
Expose IFCVF_MQ_OFFSET register to enable multi-queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 9 +++++++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 12 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..81c68c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,15 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	/* For some hardware implementation, for example:
+	 * the BAR 4 of PF is NULL, while BAR 4 of VF is not.
+	 * This code makes sure hw->mq_cfg is a valid address.
+	 */
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ef7697a..d16d9ab 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index b4389a0..008cf89 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

[PATCH v9 03/12] vdpa/ifc: set max queues based on virtio spec

[Andy Pei]

Set max_queues according to virtio spec.
For virtio BLK device, set max_queues to the value of num_queues
in struct virtio_blk_config.
For virtio NET device, read num_queues from struct ifcvf_pci_common_cfg,
get the queue pair number using num_queues and set max_queues to it.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index d16d9ab..1e133c0 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_SUBSYS_NET_DEVICE_ID          0x0001
 #define IFCVF_SUBSYS_BLK_DEVICE_ID          0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 008cf89..5a24204 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,12 @@
 
 #include "base/ifcvf.h"
 
+/*
+ * RTE_MIN() cannot be used since braced-group within expression allowed
+ * only inside a function.
+ */
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1512,6 +1518,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	uint64_t capacity = 0;
 	uint8_t *byte;
 	uint32_t i;
+	uint16_t queue_pairs;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
@@ -1559,7 +1566,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1576,14 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		/*
+		 * ifc device always has CTRL_VQ,
+		 * and supports VIRTIO_NET_F_CTRL_VQ feature.
+		 */
+		queue_pairs = (internal->hw.common_cfg->num_queues - 1) / 2;
+		DRV_LOG(INFO, "%s support %u queue pairs", pci_dev->name,
+			queue_pairs);
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1623,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

[PATCH v9 04/12] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi-queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 81c68c0..b377126 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -202,6 +202,37 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_mq(struct ifcvf_hw *hw)
+{
+	u8 *mq_cfg;
+	u8 qid;
+	int nr_queue = 0;
+
+	for (qid = 0; qid < hw->nr_vring; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
+		nr_queue++;
+	}
+
+	if (nr_queue == 0) {
+		WARNINGOUT("no enabled vring\n");
+		return;
+	}
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			*(u32 *)mq_cfg = nr_queue;
+			RTE_LOG(INFO, PMD, "%d queues are enabled\n", nr_queue);
+		} else {
+			*(u32 *)mq_cfg = nr_queue / 2;
+			RTE_LOG(INFO, PMD, "%d queue pairs are enabled\n",
+				nr_queue / 2);
+		}
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -219,6 +250,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

[PATCH v9 05/12] vdpa/ifc: only configure enabled queue

[Andy Pei]

When configuring the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c |  3 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 16 ++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index b377126..30bb8cb 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -252,6 +252,9 @@
 
 	ifcvf_enable_mq(hw);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 5a24204..0c3407a 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -284,6 +284,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -499,6 +501,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
@@ -1058,6 +1062,8 @@ struct rte_vdpa_dev_info {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
+	struct ifcvf_hw *hw;
+	uint16_t i;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1071,11 +1077,17 @@ struct rte_vdpa_dev_info {
 	rte_atomic32_set(&internal->dev_attached, 1);
 	update_datapath(internal);
 
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
+	hw = &internal->hw;
+	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
+		if (rte_vhost_host_notifier_ctrl(vid, i, true) != 0)
+			DRV_LOG(NOTICE, "vDPA (%s): software relay is used.",
 				vdev->device->name);
+	}
 
 	internal->configured = 1;
+	DRV_LOG(INFO, "vDPA device %s is configured", vdev->device->name);
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v9 06/12] vdpa/ifc: support dynamic enable/disable queue

[Andy Pei]

From: Huang Wei <wei.huang@intel.com>

Support dynamic enable or disable queue.
For front end, like QEMU, user can use ethtool to configure queue.
For example, "ethtool -L eth0 combined 3" to enable 3 queues pairs.

Signed-off-by: Huang Wei <wei.huang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/ifc/base/ifcvf.h |   6 +++
 drivers/vdpa/ifc/ifcvf_vdpa.c |  93 ++++++++++++++++++++++++++++++++-------
 3 files changed, 184 insertions(+), 15 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 30bb8cb..869ddd6 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -233,6 +233,106 @@
 	}
 }
 
+int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u16 notify_off;
+	int msix_vector;
+
+	if (i >= (int)hw->nr_vring)
+		return -1;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return -1;
+	}
+
+	ifcvf_enable_mq(hw);
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+	if (msix_vector != (i + 1)) {
+		IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector);
+		msix_vector = IFCVF_READ_REG16(&cfg->queue_msix_vector);
+		if (msix_vector == IFCVF_MSI_NO_VECTOR) {
+			RTE_LOG(ERR, PMD, "queue %d, msix vec alloc failed\n",
+				i);
+			return -1;
+		}
+	}
+
+	io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+			&cfg->queue_desc_hi);
+	io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+			&cfg->queue_avail_hi);
+	io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+			&cfg->queue_used_hi);
+	IFCVF_WRITE_REG16(hw->vring[i].size, &cfg->queue_size);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK)
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				i * IFCVF_LM_CFG_SIZE) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+		else
+			*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE +
+				(i % 2) * 4) =
+				(u32)hw->vring[i].last_avail_idx |
+				((u32)hw->vring[i].last_used_idx << 16);
+	}
+
+	notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
+	hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+			notify_off * hw->notify_off_multiplier);
+	IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+
+	return 0;
+}
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i)
+{
+	struct ifcvf_pci_common_cfg *cfg;
+	u32 ring_state;
+	u8 *lm_cfg;
+
+	if (i >= (int)hw->nr_vring)
+		return;
+
+	cfg = hw->common_cfg;
+	if (!cfg) {
+		RTE_LOG(ERR, PMD, "common_cfg in HW is NULL.\n");
+		return;
+	}
+
+	IFCVF_WRITE_REG16(i, &cfg->queue_select);
+	IFCVF_WRITE_REG16(0, &cfg->queue_enable);
+
+	lm_cfg = hw->lm_cfg;
+	if (lm_cfg) {
+		if (hw->device_type == IFCVF_BLK) {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					i * IFCVF_LM_CFG_SIZE);
+			hw->vring[i].last_avail_idx =
+				(u16)(ring_state & IFCVF_16_BIT_MASK);
+		} else {
+			ring_state = *(u32 *)(lm_cfg +
+					IFCVF_LM_RING_STATE_OFFSET +
+					(i / 2) * IFCVF_LM_CFG_SIZE +
+					(i % 2) * 4);
+			hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
+		}
+		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
+	}
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 1e133c0..3726da7 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -164,6 +164,12 @@ struct ifcvf_hw {
 ifcvf_get_features(struct ifcvf_hw *hw);
 
 int
+ifcvf_enable_vring_hw(struct ifcvf_hw *hw, int i);
+
+void
+ifcvf_disable_vring_hw(struct ifcvf_hw *hw, int i);
+
+int
 ifcvf_start_hw(struct ifcvf_hw *hw);
 
 void
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 0c3407a..9c49f9c 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1282,13 +1282,59 @@ struct rte_vdpa_dev_info {
 }
 
 static int
+ifcvf_config_vring(struct ifcvf_internal *internal, int vring)
+{
+	struct ifcvf_hw *hw = &internal->hw;
+	int vid = internal->vid;
+	struct rte_vhost_vring vq;
+	uint64_t gpa;
+
+	if (hw->vring[vring].enable) {
+		rte_vhost_get_vhost_vring(vid, vring, &vq);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
+			return -1;
+		}
+		hw->vring[vring].desc = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for available ring.");
+			return -1;
+		}
+		hw->vring[vring].avail = gpa;
+
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		if (gpa == 0) {
+			DRV_LOG(ERR, "Fail to get GPA for used ring.");
+			return -1;
+		}
+		hw->vring[vring].used = gpa;
+
+		hw->vring[vring].size = vq.size;
+		rte_vhost_get_vring_base(vid, vring,
+				&hw->vring[vring].last_avail_idx,
+				&hw->vring[vring].last_used_idx);
+		ifcvf_enable_vring_hw(&internal->hw, vring);
+	} else {
+		ifcvf_disable_vring_hw(&internal->hw, vring);
+		rte_vhost_set_vring_base(vid, vring,
+				hw->vring[vring].last_avail_idx,
+				hw->vring[vring].last_used_idx);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_set_vring_state(int vid, int vring, int state)
 {
 	struct rte_vdpa_device *vdev;
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
+	bool enable = !!state;
 	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
@@ -1298,6 +1344,9 @@ struct rte_vdpa_dev_info {
 		return -1;
 	}
 
+	DRV_LOG(INFO, "%s queue %d of vDPA device %s",
+		enable ? "enable" : "disable", vring, vdev->device->name);
+
 	internal = list->internal;
 	if (vring < 0 || vring >= internal->max_queues * 2) {
 		DRV_LOG(ERR, "Vring index %d not correct", vring);
@@ -1305,27 +1354,41 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
+	hw->vring[vring].enable = enable;
+
 	if (!internal->configured)
-		goto exit;
+		return 0;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	unset_notify_relay(internal);
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
+	ret = vdpa_enable_vfio_intr(internal, false);
+	if (ret) {
+		DRV_LOG(ERR, "failed to set vfio interrupt of vDPA device %s",
+			vdev->device->name);
+		return ret;
 	}
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
+	ret = ifcvf_config_vring(internal, vring);
+	if (ret) {
+		DRV_LOG(ERR, "failed to configure queue %d of vDPA device %s",
+			vring, vdev->device->name);
+		return ret;
+	}
+
+	ret = setup_notify_relay(internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to setup notify relay of vDPA device %s",
+			vdev->device->name);
+		return ret;
+	}
+
+	ret = rte_vhost_host_notifier_ctrl(vid, vring, enable);
+	if (ret) {
+		DRV_LOG(ERR, "vDPA device %s queue %d host notifier ctrl fail",
+			vdev->device->name, vring);
+		return ret;
 	}
 
-exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH v9 07/12] vdpa/ifc: change internal function name

[Andy Pei]

Change internal function name "find_internal_resource_by_dev"
to "find_internal_resource_by_pci_dev".

Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 9c49f9c..73d04ed 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -121,7 +121,7 @@ struct rte_vdpa_dev_info {
 }
 
 static struct internal_list *
-find_internal_resource_by_dev(struct rte_pci_device *pdev)
+find_internal_resource_by_pci_dev(struct rte_pci_device *pdev)
 {
 	int found = 0;
 	struct internal_list *list;
@@ -1746,7 +1746,7 @@ struct rte_vdpa_dev_info dev_info[] = {
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
-	list = find_internal_resource_by_dev(pci_dev);
+	list = find_internal_resource_by_pci_dev(pci_dev);
 	if (list == NULL) {
 		DRV_LOG(ERR, "Invalid device: %s", pci_dev->name);
 		return -1;
-- 
1.8.3.1

[PATCH v9 08/12] vdpa/ifc: add internal API to get device

[Andy Pei]

Add new internal API "find_internal_resource_by_rte_dev"
to get device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 73d04ed..c16e263 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -144,6 +144,29 @@ struct rte_vdpa_dev_info {
 	return list;
 }
 
+static struct internal_list *
+find_internal_resource_by_rte_dev(struct rte_device *rte_dev)
+{
+	int found = 0;
+	struct internal_list *list;
+
+	pthread_mutex_lock(&internal_list_lock);
+
+	TAILQ_FOREACH(list, &internal_list, next) {
+		if (rte_dev == &list->internal->pdev->device) {
+			found = 1;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&internal_list_lock);
+
+	if (!found)
+		return NULL;
+
+	return list;
+}
+
 static int
 ifcvf_vfio_setup(struct ifcvf_internal *internal)
 {
@@ -1398,10 +1421,11 @@ struct rte_vdpa_dev_info {
 {
 	struct ifcvf_internal *internal;
 	struct internal_list *list;
+	struct rte_device *rte_dev = vdev->device;
 
-	list = find_internal_resource_by_vdev(vdev);
+	list = find_internal_resource_by_rte_dev(rte_dev);
 	if (list == NULL) {
-		DRV_LOG(ERR, "Invalid vDPA device: %p", vdev);
+		DRV_LOG(ERR, "Invalid rte device: %p", rte_dev);
 		return -1;
 	}
 
-- 
1.8.3.1

[PATCH v9 09/12] vdpa/ifc: improve internal list logic

[Andy Pei]

Insert internal list element to internal list before
register vdpa device, in order to call vdpa ops during
vdpa device registration.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index c16e263..8dfd493 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 	internal->sw_lm = sw_fallback_lm;
 
+	pthread_mutex_lock(&internal_list_lock);
+	TAILQ_INSERT_TAIL(&internal_list, list, next);
+	pthread_mutex_unlock(&internal_list_lock);
+
 	internal->vdev = rte_vdpa_register_device(&pci_dev->device,
 				dev_info[internal->hw.device_type].ops);
 	if (internal->vdev == NULL) {
 		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
+		pthread_mutex_lock(&internal_list_lock);
+		TAILQ_REMOVE(&internal_list, list, next);
+		pthread_mutex_unlock(&internal_list_lock);
 		goto error;
 	}
 
-	pthread_mutex_lock(&internal_list_lock);
-	TAILQ_INSERT_TAIL(&internal_list, list, next);
-	pthread_mutex_unlock(&internal_list_lock);
-
 	rte_atomic32_set(&internal->started, 1);
 	update_datapath(internal);
 
-- 
1.8.3.1

RE: [PATCH v9 09/12] vdpa/ifc: improve internal list logic

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Wednesday, October 19, 2022 4:41 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v9 09/12] vdpa/ifc: improve internal list logic
> 
> Insert internal list element to internal list before
> register vdpa device, in order to call vdpa ops during
> vdpa device registration.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
>  1 file changed, 7 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index c16e263..8dfd493 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = {
>  	}
>  	internal->sw_lm = sw_fallback_lm;
> 
> +	pthread_mutex_lock(&internal_list_lock);
> +	TAILQ_INSERT_TAIL(&internal_list, list, next);
> +	pthread_mutex_unlock(&internal_list_lock);
> +
>  	internal->vdev = rte_vdpa_register_device(&pci_dev->device,
>  				dev_info[internal->hw.device_type].ops);
>  	if (internal->vdev == NULL) {
>  		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
> +		pthread_mutex_lock(&internal_list_lock);
> +		TAILQ_REMOVE(&internal_list, list, next);
> +		pthread_mutex_unlock(&internal_list_lock);
>  		goto error;
>  	}
> 
> -	pthread_mutex_lock(&internal_list_lock);
> -	TAILQ_INSERT_TAIL(&internal_list, list, next);
> -	pthread_mutex_unlock(&internal_list_lock);
> -
>  	rte_atomic32_set(&internal->started, 1);
>  	update_datapath(internal);
> 
> --
> 1.8.3.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

RE: [PATCH v9 09/12] vdpa/ifc: improve internal list logic

[Pei, Andy]

Hi Chenbo,

Thanks.

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Thursday, October 20, 2022 11:22 AM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v9 09/12] vdpa/ifc: improve internal list logic
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Wednesday, October 19, 2022 4:41 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v9 09/12] vdpa/ifc: improve internal list logic
> >
> > Insert internal list element to internal list before register vdpa
> > device, in order to call vdpa ops during vdpa device registration.
> >
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> > ---
> >  drivers/vdpa/ifc/ifcvf_vdpa.c | 11 +++++++----
> >  1 file changed, 7 insertions(+), 4 deletions(-)
> >
> > diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > b/drivers/vdpa/ifc/ifcvf_vdpa.c index c16e263..8dfd493 100644
> > --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> > +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> > @@ -1737,17 +1737,20 @@ struct rte_vdpa_dev_info dev_info[] = {
> >  	}
> >  	internal->sw_lm = sw_fallback_lm;
> >
> > +	pthread_mutex_lock(&internal_list_lock);
> > +	TAILQ_INSERT_TAIL(&internal_list, list, next);
> > +	pthread_mutex_unlock(&internal_list_lock);
> > +
> >  	internal->vdev = rte_vdpa_register_device(&pci_dev->device,
> >  				dev_info[internal->hw.device_type].ops);
> >  	if (internal->vdev == NULL) {
> >  		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
> > +		pthread_mutex_lock(&internal_list_lock);
> > +		TAILQ_REMOVE(&internal_list, list, next);
> > +		pthread_mutex_unlock(&internal_list_lock);
> >  		goto error;
> >  	}
> >
> > -	pthread_mutex_lock(&internal_list_lock);
> > -	TAILQ_INSERT_TAIL(&internal_list, list, next);
> > -	pthread_mutex_unlock(&internal_list_lock);
> > -
> >  	rte_atomic32_set(&internal->started, 1);
> >  	update_datapath(internal);
> >
> > --
> > 1.8.3.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

[PATCH v9 10/12] vhost: add type to rte vdpa device

[Andy Pei]

Add type to rte_vdpa_device to store device type.
Call vdpa ops get_dev_type to fill type when register
vdpa device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 lib/vhost/socket.c      | 15 +--------------
 lib/vhost/vdpa.c        | 15 +++++++++++++++
 lib/vhost/vdpa_driver.h |  2 ++
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c
index 608ae57..863a6f6 100644
--- a/lib/vhost/socket.c
+++ b/lib/vhost/socket.c
@@ -627,7 +627,6 @@ struct rte_vdpa_device *
 {
 	struct vhost_user_socket *vsocket;
 	struct rte_vdpa_device *vdpa_dev;
-	uint32_t vdpa_type = 0;
 	int ret = 0;
 
 	pthread_mutex_lock(&vhost_user.mutex);
@@ -644,19 +643,7 @@ struct rte_vdpa_device *
 		goto unlock_exit;
 	}
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(path, ERR,
-				"failed to get vdpa dev type for socket file.\n");
-			ret = -1;
-			goto unlock_exit;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
-
-	*type = vdpa_type;
+	*type = vdpa_dev->type;
 
 unlock_exit:
 	pthread_mutex_unlock(&vhost_user.mutex);
diff --git a/lib/vhost/vdpa.c b/lib/vhost/vdpa.c
index bb82857..577cb00 100644
--- a/lib/vhost/vdpa.c
+++ b/lib/vhost/vdpa.c
@@ -73,6 +73,7 @@ struct rte_vdpa_device *
 		struct rte_vdpa_dev_ops *ops)
 {
 	struct rte_vdpa_device *dev;
+	int ret = 0;
 
 	if (ops == NULL)
 		return NULL;
@@ -101,6 +102,20 @@ struct rte_vdpa_device *
 
 	dev->device = rte_dev;
 	dev->ops = ops;
+
+	if (ops->get_dev_type) {
+		ret = ops->get_dev_type(dev, &dev->type);
+		if (ret) {
+			VHOST_LOG_CONFIG(rte_dev->name, ERR,
+					 "Failed to get vdpa dev type.\n");
+			ret = -1;
+			goto out_unlock;
+		}
+	} else {
+		/** by default, we assume vdpa device is a net device */
+		dev->type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+	}
+
 	TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
 out_unlock:
 	rte_spinlock_unlock(&vdpa_device_list_lock);
diff --git a/lib/vhost/vdpa_driver.h b/lib/vhost/vdpa_driver.h
index 8b88a53..8db4ab9 100644
--- a/lib/vhost/vdpa_driver.h
+++ b/lib/vhost/vdpa_driver.h
@@ -92,6 +92,8 @@ struct rte_vdpa_device {
 	struct rte_device *device;
 	/** vdpa device operations */
 	struct rte_vdpa_dev_ops *ops;
+	/** vdpa device type: net, blk... */
+	uint32_t type;
 };
 
 /**
-- 
1.8.3.1

[PATCH v9 11/12] vhost: vDPA blk device gets ready when the first queue is ready

[Andy Pei]

When boot from virtio blk device, seabios in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end configure device
when the first queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 lib/vhost/vhost_user.c | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index cd65257..e0ff79d 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1441,11 +1441,14 @@
 }
 
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
+#define VIRTIO_BLK_NUM_VQS_TO_BE_READY 1u
 
 static int
 virtio_is_ready(struct virtio_net *dev)
 {
+	struct rte_vdpa_device *vdpa_dev;
 	struct vhost_virtqueue *vq;
+	uint32_t vdpa_type;
 	uint32_t i, nr_vring = dev->nr_vring;
 
 	if (dev->flags & VIRTIO_DEV_READY)
@@ -1454,13 +1457,22 @@
 	if (!dev->nr_vring)
 		return 0;
 
-	if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET) {
-		nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev)
+		vdpa_type = vdpa_dev->type;
+	else
+		vdpa_type = -1;
 
-		if (dev->nr_vring < nr_vring)
-			return 0;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		nr_vring = VIRTIO_BLK_NUM_VQS_TO_BE_READY;
+	} else {
+		if (dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET)
+			nr_vring = VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY;
 	}
 
+	if (dev->nr_vring < nr_vring)
+		return 0;
+
 	for (i = 0; i < nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
@@ -2958,7 +2970,6 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	int ret;
 	int unlock_required = 0;
 	bool handled;
-	uint32_t vdpa_type = 0;
 	uint32_t request;
 	uint32_t i;
 
@@ -3170,17 +3181,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
+	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
 
-- 
1.8.3.1

[PATCH v9 12/12] vhost: improve vDPA blk device configure condition

[Andy Pei]

To support multi-queue, configure device
after call fd of all queues are set.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei.huang@intel.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index e0ff79d..9902ae9 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2972,6 +2972,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	bool handled;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3181,9 +3182,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_dev->type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1

RE: [PATCH v9 00/12] vdpa/ifc: add multi queue support

[Xia, Chenbo]

> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Wednesday, October 19, 2022 4:41 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> Huang, Wei <wei.huang@intel.com>; Cao, Gang <gang.cao@intel.com>;
> maxime.coquelin@redhat.com
> Subject: [PATCH v9 00/12] vdpa/ifc: add multi queue support
> 
> v9:
>  fix some commit message.
> 
> v8:
>  change "vdpa_device_type" in "rte_vdpa_device" to "type".
> 
> v7:
>  Fill vdpa_device_type in vdpa device registration.
> 
> v6:
>  Add vdpa_device_type to rte_vdpa_device to store vDPA device type.
> 
> v5:
>  fix some commit message.
>  rework some code logic.
> 
> v4:
>  fix some commit message.
>  add some commets to code.
>  fix some code to reduce confusion.
> 
> v3:
>  rename device ID macro name.
>  fix some patch title and commit message.
>  delete some used marco.
>  rework some code logic.
> 
> v2:
>  fix some coding style issue.
>  support dynamic enable/disable queue at run time.
> 
> Andy Pei (10):
>   vdpa/ifc: add multi-queue support
>   vdpa/ifc: set max queues based on virtio spec
>   vdpa/ifc: write queue count to MQ register
>   vdpa/ifc: only configure enabled queue
>   vdpa/ifc: change internal function name
>   vdpa/ifc: add internal API to get device
>   vdpa/ifc: improve internal list logic
>   vhost: add type to rte vdpa device
>   vhost: vDPA blk device gets ready when the first queue is ready
>   vhost: improve vDPA blk device configure condition
> 
> Huang Wei (2):
>   vdpa/ifc: add new device ID for legacy network device
>   vdpa/ifc: support dynamic enable/disable queue
> 
>  drivers/vdpa/ifc/base/ifcvf.c | 144 ++++++++++++++++++++++++++++++++
>  drivers/vdpa/ifc/base/ifcvf.h |  16 +++-
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 185 +++++++++++++++++++++++++++++++++++--
> -----
>  lib/vhost/socket.c            |  15 +---
>  lib/vhost/vdpa.c              |  15 ++++
>  lib/vhost/vdpa_driver.h       |   2 +
>  lib/vhost/vhost_user.c        |  38 +++++----
>  7 files changed, 354 insertions(+), 61 deletions(-)
> 
> --
> 1.8.3.1

Change title ' vhost: add type to rte vdpa device ' to ' vhost: add type to vDPA device '

Series applied to next-virtio/main, Thanks

RE: [PATCH v9 00/12] vdpa/ifc: add multi queue support

[Pei, Andy]

> -----Original Message-----
> From: Xia, Chenbo <chenbo.xia@intel.com>
> Sent: Wednesday, October 26, 2022 5:00 PM
> To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>;
> Cao, Gang <gang.cao@intel.com>; maxime.coquelin@redhat.com
> Subject: RE: [PATCH v9 00/12] vdpa/ifc: add multi queue support
> 
> > -----Original Message-----
> > From: Pei, Andy <andy.pei@intel.com>
> > Sent: Wednesday, October 19, 2022 4:41 PM
> > To: dev@dpdk.org
> > Cc: Xia, Chenbo <chenbo.xia@intel.com>; Xu, Rosen
> > <rosen.xu@intel.com>; Huang, Wei <wei.huang@intel.com>; Cao, Gang
> > <gang.cao@intel.com>; maxime.coquelin@redhat.com
> > Subject: [PATCH v9 00/12] vdpa/ifc: add multi queue support
> >
> > v9:
> >  fix some commit message.
> >
> > v8:
> >  change "vdpa_device_type" in "rte_vdpa_device" to "type".
> >
> > v7:
> >  Fill vdpa_device_type in vdpa device registration.
> >
> > v6:
> >  Add vdpa_device_type to rte_vdpa_device to store vDPA device type.
> >
> > v5:
> >  fix some commit message.
> >  rework some code logic.
> >
> > v4:
> >  fix some commit message.
> >  add some commets to code.
> >  fix some code to reduce confusion.
> >
> > v3:
> >  rename device ID macro name.
> >  fix some patch title and commit message.
> >  delete some used marco.
> >  rework some code logic.
> >
> > v2:
> >  fix some coding style issue.
> >  support dynamic enable/disable queue at run time.
> >
> > Andy Pei (10):
> >   vdpa/ifc: add multi-queue support
> >   vdpa/ifc: set max queues based on virtio spec
> >   vdpa/ifc: write queue count to MQ register
> >   vdpa/ifc: only configure enabled queue
> >   vdpa/ifc: change internal function name
> >   vdpa/ifc: add internal API to get device
> >   vdpa/ifc: improve internal list logic
> >   vhost: add type to rte vdpa device
> >   vhost: vDPA blk device gets ready when the first queue is ready
> >   vhost: improve vDPA blk device configure condition
> >
> > Huang Wei (2):
> >   vdpa/ifc: add new device ID for legacy network device
> >   vdpa/ifc: support dynamic enable/disable queue
> >
> >  drivers/vdpa/ifc/base/ifcvf.c | 144
> ++++++++++++++++++++++++++++++++
> > drivers/vdpa/ifc/base/ifcvf.h |  16 +++-
> > drivers/vdpa/ifc/ifcvf_vdpa.c | 185
> > +++++++++++++++++++++++++++++++++++--
> > -----
> >  lib/vhost/socket.c            |  15 +---
> >  lib/vhost/vdpa.c              |  15 ++++
> >  lib/vhost/vdpa_driver.h       |   2 +
> >  lib/vhost/vhost_user.c        |  38 +++++----
> >  7 files changed, 354 insertions(+), 61 deletions(-)
> >
> > --
> > 1.8.3.1
> 
> Change title ' vhost: add type to rte vdpa device ' to ' vhost: add type to vDPA
> device '
> 
OK.  Thanks Chenbo.

> Series applied to next-virtio/main, Thanks

[PATCH 2/8] vdpa/ifc: add multi queue suppoort

[Andy Pei]

Enable VHOST_USER_PROTOCOL_F_MQ feature.
ExposeIFCVF_MQ_OFFSET register to enable multi queue.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 5 +++++
 drivers/vdpa/ifc/base/ifcvf.h | 2 ++
 drivers/vdpa/ifc/ifcvf_vdpa.c | 1 +
 3 files changed, 8 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index f1e1474..34c8226 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -90,6 +90,11 @@
 	if (!hw->lm_cfg)
 		WARNINGOUT("HW support live migration not support!\n");
 
+	if (hw->mem_resource[4].addr)
+		hw->mq_cfg = hw->mem_resource[4].addr + IFCVF_MQ_OFFSET;
+	else
+		hw->mq_cfg = NULL;
+
 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
 			hw->isr == NULL || hw->dev_cfg == NULL) {
 		DEBUGOUT("capability incomplete\n");
diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index 7ede738..ad505f1 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -50,6 +50,7 @@
 
 #define IFCVF_LM_CFG_SIZE		0x40
 #define IFCVF_LM_RING_STATE_OFFSET	0x20
+#define IFCVF_MQ_OFFSET			0x28
 
 #define IFCVF_LM_LOGGING_CTRL		0x0
 
@@ -149,6 +150,7 @@ struct ifcvf_hw {
 	u16    *notify_base;
 	u16    *notify_addr[IFCVF_MAX_QUEUES * 2];
 	u8     *lm_cfg;
+	u8     *mq_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES * 2];
 	u8 nr_vring;
 	int device_type;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 61d0250..2d165c0 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1248,6 +1248,7 @@ struct rte_vdpa_dev_info {
 		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
 		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_MQ | \
 		 1ULL << VHOST_USER_PROTOCOL_F_STATUS)
 
 #define VDPA_BLK_PROTOCOL_FEATURES \
-- 
1.8.3.1

[PATCH 3/8] vdpa/ifc: set max queues according to HW spec

[Andy Pei]

Set max_queues according to virtio HW spec.
For virtio BLK device, set max_queues to the value of "num_queues".
"num_queues" is element of struct virtio_blk_config.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.h |  2 +-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 18 +++++++++++++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.h b/drivers/vdpa/ifc/base/ifcvf.h
index ad505f1..c17bf2a 100644
--- a/drivers/vdpa/ifc/base/ifcvf.h
+++ b/drivers/vdpa/ifc/base/ifcvf.h
@@ -21,7 +21,7 @@
 #define IFCVF_NET_DEVICE_ID                 0x0001
 #define IFCVF_BLK_DEVICE_ID                 0x0002
 
-#define IFCVF_MAX_QUEUES		1
+#define IFCVF_MAX_QUEUES		32
 
 #ifndef VIRTIO_F_IOMMU_PLATFORM
 #define VIRTIO_F_IOMMU_PLATFORM		33
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 2d165c0..34aea6c 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -26,6 +26,18 @@
 
 #include "base/ifcvf.h"
 
+/**
+** RTE_MAX() and RTE_MIN() cannot be used since braced-group within
+** expression allowed only inside a function, but MAX() is used as
+** a number of elements in array.
+**/
+#ifndef MAX
+#define MAX(v1, v2)	((v1) > (v2) ? (v1) : (v2))
+#endif
+#ifndef MIN
+#define MIN(v1, v2)	((v1) < (v2) ? (v1) : (v2))
+#endif
+
 RTE_LOG_REGISTER(ifcvf_vdpa_logtype, pmd.vdpa.ifcvf, NOTICE);
 #define DRV_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, ifcvf_vdpa_logtype, \
@@ -1559,7 +1571,6 @@ struct rte_vdpa_dev_info dev_info[] = {
 	}
 
 	internal->configured = 0;
-	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 
 	device_id = ifcvf_pci_get_device_type(pci_dev);
@@ -1570,6 +1581,8 @@ struct rte_vdpa_dev_info dev_info[] = {
 
 	if (device_id == VIRTIO_ID_NET) {
 		internal->hw.device_type = IFCVF_NET;
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			(internal->hw.common_cfg->num_queues - 1)/2);
 		internal->features = features &
 					~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		internal->features |= dev_info[IFCVF_NET].features;
@@ -1609,6 +1622,9 @@ struct rte_vdpa_dev_info dev_info[] = {
 			internal->hw.blk_cfg->geometry.sectors);
 		DRV_LOG(DEBUG, "num_queues: 0x%08x",
 			internal->hw.blk_cfg->num_queues);
+
+		internal->max_queues = MIN(IFCVF_MAX_QUEUES,
+			internal->hw.blk_cfg->num_queues);
 	}
 
 	list->internal = internal;
-- 
1.8.3.1

[PATCH 4/8] vdpa/ifc: write queue count to MQ register

[Andy Pei]

Write queue count to IFCVF_MQ_OFFSET register
to enable multi queue feature.

Signed-off-by: Andy Pei <andy.pei@intel.com>
Signed-off-by: Huang Wei <wei_huang@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 34c8226..1b50df6 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -198,6 +198,19 @@
 	IFCVF_WRITE_REG32(val >> 32, hi);
 }
 
+STATIC void
+ifcvf_enable_multiqueue(struct ifcvf_hw *hw, u16 nr_queue_pair)
+{
+	u8 *mq_cfg;
+
+	if (hw->device_type == IFCVF_NET)
+		nr_queue_pair = (nr_queue_pair + 1) / 2;
+
+	mq_cfg = hw->mq_cfg;
+	if (mq_cfg)
+		*(u32 *)mq_cfg = nr_queue_pair;
+}
+
 STATIC int
 ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
@@ -215,6 +228,7 @@
 		return -1;
 	}
 
+	ifcvf_enable_multiqueue(hw, hw->nr_vring);
 	for (i = 0; i < hw->nr_vring; i++) {
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
-- 
1.8.3.1

[PATCH 5/8] vdpa/ifc: only configure enabled queue

[Andy Pei]

when configure the hardware queue, we only configure queues which
have been enabled by vhost.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/base/ifcvf.c | 5 ++++-
 drivers/vdpa/ifc/ifcvf_vdpa.c | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/drivers/vdpa/ifc/base/ifcvf.c b/drivers/vdpa/ifc/base/ifcvf.c
index 1b50df6..ca5f677 100644
--- a/drivers/vdpa/ifc/base/ifcvf.c
+++ b/drivers/vdpa/ifc/base/ifcvf.c
@@ -230,6 +230,8 @@
 
 	ifcvf_enable_multiqueue(hw, hw->nr_vring);
 	for (i = 0; i < hw->nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		IFCVF_WRITE_REG16(i, &cfg->queue_select);
 		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				&cfg->queue_desc_hi);
@@ -264,7 +266,8 @@
 		notify_off = IFCVF_READ_REG16(&cfg->queue_notify_off);
 		hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
 				notify_off * hw->notify_off_multiplier);
-		IFCVF_WRITE_REG16(1, &cfg->queue_enable);
+		if (hw->vring[i].enable)
+			IFCVF_WRITE_REG16(1, &cfg->queue_enable);
 	}
 
 	return 0;
diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 34aea6c..a62bcec 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -290,6 +290,8 @@ struct rte_vdpa_dev_info {
 	rte_vhost_get_negotiated_features(vid, &hw->req_features);
 
 	for (i = 0; i < nr_vring; i++) {
+		if (!hw->vring[i].enable)
+			continue;
 		rte_vhost_get_vhost_vring(vid, i, &vq);
 		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
@@ -505,6 +507,8 @@ struct rte_vdpa_dev_info {
 
 	vring.kickfd = -1;
 	for (qid = 0; qid < q_num; qid++) {
+		if (!hw->vring[qid].enable)
+			continue;
 		ev.events = EPOLLIN | EPOLLPRI;
 		rte_vhost_get_vhost_vring(internal->vid, qid, &vring);
 		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
-- 
1.8.3.1

[PATCH 6/8] vdpa/ifc: set vring state callback update data path

[Andy Pei]

To support multi queue, in the case that first queue is ready
and device is configured, when more queues need to be configured,
we just close and restart data path.
This also fix the situation that using set_vring_state callback
to disable one queue will cause all vfio interrupts being disabled.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index a62bcec..94c8ef1 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -1286,8 +1286,6 @@ struct rte_vdpa_dev_info {
 	struct internal_list *list;
 	struct ifcvf_internal *internal;
 	struct ifcvf_hw *hw;
-	struct ifcvf_pci_common_cfg *cfg;
-	int ret = 0;
 
 	vdev = rte_vhost_get_vdpa_device(vid);
 	list = find_internal_resource_by_vdev(vdev);
@@ -1303,27 +1301,20 @@ struct rte_vdpa_dev_info {
 	}
 
 	hw = &internal->hw;
-	if (!internal->configured)
-		goto exit;
 
-	cfg = hw->common_cfg;
-	IFCVF_WRITE_REG16(vring, &cfg->queue_select);
-	IFCVF_WRITE_REG16(!!state, &cfg->queue_enable);
+	hw->vring[vring].enable = !!state;
 
-	if (!state && hw->vring[vring].enable) {
-		ret = vdpa_disable_vfio_intr(internal);
-		if (ret)
-			return ret;
-	}
+	if (!internal->configured)
+		goto exit;
 
-	if (state && !hw->vring[vring].enable) {
-		ret = vdpa_enable_vfio_intr(internal, false);
-		if (ret)
-			return ret;
-	}
+	/* close data path */
+	rte_atomic32_set(&internal->dev_attached, 0);
+	update_datapath(internal);
 
+	/* restart data path */
+	rte_atomic32_set(&internal->dev_attached, 1);
+	update_datapath(internal);
 exit:
-	hw->vring[vring].enable = !!state;
 	return 0;
 }
 
-- 
1.8.3.1

[PATCH 7/8] vhost: configure device when any queue is ready for BLK device

[Andy Pei]

When boot from virtio blk device, seabois in QEMU only enables one queue.
To work in this scenario, vDPA BLK device back-end conf_dev when any
queue is ready.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 lib/vhost/vhost_user.c | 56 +++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 12 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index 4ad28ba..b65fba3 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -1451,6 +1451,25 @@
 #define VIRTIO_BUILTIN_NUM_VQS_TO_BE_READY 2u
 
 static int
+virtio_has_queue_ready(struct virtio_net *dev)
+{
+	struct vhost_virtqueue *vq;
+	uint32_t i, nr_vring = dev->nr_vring;
+
+	if (!dev->nr_vring)
+		return 0;
+
+	for (i = 0; i < nr_vring; i++) {
+		vq = dev->virtqueue[i];
+
+		if (vq_is_ready(dev, vq))
+			return 1;
+	}
+
+	return 0;
+}
+
+static int
 virtio_is_ready(struct virtio_net *dev)
 {
 	struct vhost_virtqueue *vq;
@@ -3167,9 +3186,33 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (unlock_required)
 		vhost_user_unlock_all_queue_pairs(dev);
 
-	if (ret != 0 || !virtio_is_ready(dev))
+	if (ret != 0)
 		goto out;
 
+	vdpa_dev = dev->vdpa_dev;
+	if (vdpa_dev) {
+		if (vdpa_dev->ops->get_dev_type) {
+			ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
+			if (ret) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+					"failed to get vdpa dev type.\n");
+				ret = -1;
+				goto out;
+			}
+		} else {
+			vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
+		}
+	}
+
+	if (!virtio_is_ready(dev)) {
+		if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+			if (!virtio_has_queue_ready(dev))
+				goto out;
+		} else {
+			goto out;
+		}
+	}
+
 	/*
 	 * Virtio is now ready. If not done already, it is time
 	 * to notify the application it can process the rings and
@@ -3181,20 +3224,9 @@ static int is_vring_iotlb(struct virtio_net *dev,
 			dev->flags |= VIRTIO_DEV_RUNNING;
 	}
 
-	vdpa_dev = dev->vdpa_dev;
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_dev->ops->get_dev_type) {
-		ret = vdpa_dev->ops->get_dev_type(vdpa_dev, &vdpa_type);
-		if (ret) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to get vdpa dev type.\n");
-			ret = -1;
-			goto out;
-		}
-	} else {
-		vdpa_type = RTE_VHOST_VDPA_DEVICE_TYPE_NET;
-	}
 	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
 		&& request != VHOST_USER_SET_VRING_CALL)
 		goto out;
-- 
1.8.3.1

[PATCH 8/8] vhost: vDPA BLK devices configure device when all queue callfds are set

[Andy Pei]

In the virtio blk vDPA live migration use case, before the live
migration process, QEMU will set call fd to vDPA back-end. QEMU
and vDPA back-end stand by until live migration starts.
During live migration process, QEMU sets kick fd and a new call
fd. However, after the kick fd is set to the vDPA back-end, the
vDPA back-end configures device and data path starts. The new
call fd will cause some kind of "re-configuration", this kind
of "re-configuration" cause IO drop.
After this patch, vDPA back-end configures device after kick fd
and call fd are well set and make sure no IO drops.
This patch only impact virtio blk vDPA device and does not impact
net device.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 lib/vhost/vhost_user.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index b65fba3..568030a 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -2994,6 +2994,7 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	uint32_t vdpa_type = 0;
 	uint32_t request;
 	uint32_t i;
+	uint16_t blk_call_fd;
 
 	dev = get_device(vid);
 	if (dev == NULL)
@@ -3227,9 +3228,15 @@ static int is_vring_iotlb(struct virtio_net *dev,
 	if (!vdpa_dev)
 		goto out;
 
-	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK
-		&& request != VHOST_USER_SET_VRING_CALL)
-		goto out;
+	if (vdpa_type == RTE_VHOST_VDPA_DEVICE_TYPE_BLK) {
+		if (request == VHOST_USER_SET_VRING_CALL) {
+			blk_call_fd = ctx.msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
+			if (blk_call_fd != dev->nr_vring - 1)
+				goto out;
+		} else {
+			goto out;
+		}
+	}
 
 	if (!(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED)) {
 		if (vdpa_dev->ops->dev_conf(dev->vid))
-- 
1.8.3.1