[PATCH 0/4] Packed virtqueue state support for vDPA

[PATCH 0/4] Packed virtqueue state support for vDPA

[Jason Wang]

Hi:

This series implements the packed virtqueue state support for
vDPA. This is done via extending the vdpa_vq_state to support packed
virtqueue.

For virtio-vDPA, an initial state required by the virtio spec is set.

For vhost-vDPA, the packed virtqueue support still needs to be done at
both vhost core and vhost-vDPA in the future.

Please review.

Eli Cohen (1):
  virtio/vdpa: clear the virtqueue state during probe

Jason Wang (3):
  vdpa: support packed virtqueue for set/get_vq_state()
  virtio-pci library: introduce vp_modern_get_driver_features()
  vp_vdpa: allow set vq state to initial state after reset

 drivers/vdpa/ifcvf/ifcvf_main.c        |  4 +--
 drivers/vdpa/vdpa_sim/vdpa_sim.c       |  4 +--
 drivers/vdpa/virtio_pci/vp_vdpa.c      | 42 ++++++++++++++++++++++++--
 drivers/vhost/vdpa.c                   |  4 +--
 drivers/virtio/virtio_pci_modern_dev.c | 21 +++++++++++++
 drivers/virtio/virtio_vdpa.c           | 15 +++++++++
 include/linux/vdpa.h                   | 25 +++++++++++++--
 include/linux/virtio_pci_modern.h      |  1 +
 8 files changed, 105 insertions(+), 11 deletions(-)

-- 
2.25.1

[PATCH 1/4] vdpa: support packed virtqueue for set/get_vq_state()

[Jason Wang]

This patch extends the vdpa_vq_state to support packed virtqueue
state which is basically the device/driver ring wrap counters and the
avail and used index. This will be used for the virito-vdpa support
for the packed virtqueue and the future vhost/vhost-vdpa support for
the packed virtqueue.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/vdpa/ifcvf/ifcvf_main.c  |  4 ++--
 drivers/vdpa/vdpa_sim/vdpa_sim.c |  4 ++--
 drivers/vhost/vdpa.c             |  4 ++--
 include/linux/vdpa.h             | 25 +++++++++++++++++++++++--
 4 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index ab0ab5cf0f6e..5d3891b1ca28 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -264,7 +264,7 @@ static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
 {
 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
 
-	state->avail_index = ifcvf_get_vq_state(vf, qid);
+	state->split.avail_index = ifcvf_get_vq_state(vf, qid);
 	return 0;
 }
 
@@ -273,7 +273,7 @@ static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
 {
 	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
 
-	return ifcvf_set_vq_state(vf, qid, state->avail_index);
+	return ifcvf_set_vq_state(vf, qid, state->split.avail_index);
 }
 
 static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 98f793bc9376..14e024de5cbf 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -374,7 +374,7 @@ static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
 	struct vringh *vrh = &vq->vring;
 
 	spin_lock(&vdpasim->lock);
-	vrh->last_avail_idx = state->avail_index;
+	vrh->last_avail_idx = state->split.avail_index;
 	spin_unlock(&vdpasim->lock);
 
 	return 0;
@@ -387,7 +387,7 @@ static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx,
 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
 	struct vringh *vrh = &vq->vring;
 
-	state->avail_index = vrh->last_avail_idx;
+	state->split.avail_index = vrh->last_avail_idx;
 	return 0;
 }
 
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index fb41db3da611..210ab35a7ebf 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -383,7 +383,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 		if (r)
 			return r;
 
-		vq->last_avail_idx = vq_state.avail_index;
+		vq->last_avail_idx = vq_state.split.avail_index;
 		break;
 	}
 
@@ -401,7 +401,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 		break;
 
 	case VHOST_SET_VRING_BASE:
-		vq_state.avail_index = vq->last_avail_idx;
+		vq_state.split.avail_index = vq->last_avail_idx;
 		if (ops->set_vq_state(vdpa, idx, &vq_state))
 			r = -EINVAL;
 		break;
diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
index f311d227aa1b..3357ac98878d 100644
--- a/include/linux/vdpa.h
+++ b/include/linux/vdpa.h
@@ -28,13 +28,34 @@ struct vdpa_notification_area {
 };
 
 /**
- * struct vdpa_vq_state - vDPA vq_state definition
+ * struct vdpa_vq_state_split - vDPA split virtqueue state
  * @avail_index: available index
  */
-struct vdpa_vq_state {
+struct vdpa_vq_state_split {
 	u16	avail_index;
 };
 
+/**
+ * struct vdpa_vq_state_packed - vDPA packed virtqueue state
+ * @last_avail_counter: last driver ring wrap counter observed by device
+ * @last_avail_idx: device available index
+ * @last_used_counter: device ring wrap counter
+ * @last_used_idx: used index
+ */
+struct vdpa_vq_state_packed {
+        u16	last_avail_counter:1;
+        u16	last_avail_idx:15;
+        u16	last_used_counter:1;
+        u16	last_used_idx:15;
+};
+
+struct vdpa_vq_state {
+     union {
+          struct vdpa_vq_state_split split;
+          struct vdpa_vq_state_packed packed;
+     };
+};
+
 struct vdpa_mgmt_dev;
 
 /**
-- 
2.25.1

[PATCH 2/4] virtio-pci library: introduce vp_modern_get_driver_features()

[Jason Wang]

This patch introduce a helper to get driver/guest features from the
device.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_pci_modern_dev.c | 21 +++++++++++++++++++++
 include/linux/virtio_pci_modern.h      |  1 +
 2 files changed, 22 insertions(+)

diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c
index 54f297028586..e11ed748e661 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -383,6 +383,27 @@ u64 vp_modern_get_features(struct virtio_pci_modern_device *mdev)
 }
 EXPORT_SYMBOL_GPL(vp_modern_get_features);
 
+/*
+ * vp_modern_get_driver_features - get driver features from device
+ * @mdev: the modern virtio-pci device
+ *
+ * Returns the driver features read from the device
+ */
+u64 vp_modern_get_driver_features(struct virtio_pci_modern_device *mdev)
+{
+	struct virtio_pci_common_cfg __iomem *cfg = mdev->common;
+
+	u64 features;
+
+	vp_iowrite32(0, &cfg->guest_feature_select);
+	features = vp_ioread32(&cfg->guest_feature);
+	vp_iowrite32(1, &cfg->guest_feature_select);
+	features |= ((u64)vp_ioread32(&cfg->guest_feature) << 32);
+
+	return features;
+}
+EXPORT_SYMBOL_GPL(vp_modern_get_driver_features);
+
 /*
  * vp_modern_set_features - set features to device
  * @mdev: the modern virtio-pci device
diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_modern.h
index 6a95b58fd0f4..eb2bd9b4077d 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -79,6 +79,7 @@ static inline void vp_iowrite64_twopart(u64 val,
 }
 
 u64 vp_modern_get_features(struct virtio_pci_modern_device *mdev);
+u64 vp_modern_get_driver_features(struct virtio_pci_modern_device *mdev);
 void vp_modern_set_features(struct virtio_pci_modern_device *mdev,
 		     u64 features);
 u32 vp_modern_generation(struct virtio_pci_modern_device *mdev);
-- 
2.25.1

[PATCH 3/4] vp_vdpa: allow set vq state to initial state after reset

[Jason Wang]

We used to fail the set_vq_state() since it was not supported yet by
the virtio spec. But if the bus tries to set the state which is equal
to the device initial state after reset, we can let it go.

This is a must for virtio_vdpa() to set vq state during probe which is
required for some vDPA parents.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/vdpa/virtio_pci/vp_vdpa.c | 42 ++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
index c76ebb531212..18bf4a422772 100644
--- a/drivers/vdpa/virtio_pci/vp_vdpa.c
+++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
@@ -210,13 +210,49 @@ static int vp_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid,
 	return -EOPNOTSUPP;
 }
 
+static int vp_vdpa_set_vq_state_split(struct vdpa_device *vdpa,
+				      const struct vdpa_vq_state *state)
+{
+	const struct vdpa_vq_state_split *split = &state->split;
+
+	if (split->avail_index == 0)
+		return 0;
+
+	return -EOPNOTSUPP;
+}
+
+static int vp_vdpa_set_vq_state_packed(struct vdpa_device *vdpa,
+				       const struct vdpa_vq_state *state)
+{
+	const struct vdpa_vq_state_packed *packed = &state->packed;
+
+	if (packed->last_avail_counter == 1 &&
+	    packed->last_avail_idx == 0 &&
+	    packed->last_used_counter == 1 &&
+	    packed->last_used_idx == 0)
+		return 0;
+
+	return -EOPNOTSUPP;
+}
+
 static int vp_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 qid,
 				const struct vdpa_vq_state *state)
 {
-	/* Note that this is not supported by virtio specification, so
-	 * we return -ENOPOTSUPP here. This means we can't support live
-	 * migration, vhost device start/stop.
+	struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
+
+	/* Note that this is not supported by virtio specification.
+	 * But if the state is by chance equal to the device initial
+	 * state, we can let it go.
 	 */
+	if ((vp_modern_get_status(mdev) & VIRTIO_CONFIG_S_FEATURES_OK) &&
+	    !vp_modern_get_queue_enable(mdev, qid)) {
+		if (vp_modern_get_driver_features(mdev) &
+		    BIT_ULL(VIRTIO_F_RING_PACKED))
+			return vp_vdpa_set_vq_state_packed(vdpa, state);
+		else
+			return vp_vdpa_set_vq_state_split(vdpa,	state);
+	}
+
 	return -EOPNOTSUPP;
 }
 
-- 
2.25.1

[PATCH 4/4] virtio/vdpa: clear the virtqueue state during probe

[Jason Wang]

From: Eli Cohen <elic@nvidia.com>

Clear the available index as part of the initialization process to
clear and values that might be left from previous usage of the device.
For example, if the device was previously used by vhost_vdpa and now
probed by vhost_vdpa, you want to start with indices.

Fixes: c043b4a8cf3b ("virtio: introduce a vDPA based transport")
Signed-off-by: Eli Cohen <elic@nvidia.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_vdpa.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
index e28acf482e0c..e1a141135992 100644
--- a/drivers/virtio/virtio_vdpa.c
+++ b/drivers/virtio/virtio_vdpa.c
@@ -142,6 +142,8 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 	struct vdpa_callback cb;
 	struct virtqueue *vq;
 	u64 desc_addr, driver_addr, device_addr;
+	/* Assume split virtqueue, switch to packed if necessary */
+	struct vdpa_vq_state state = {0};
 	unsigned long flags;
 	u32 align, num;
 	int err;
@@ -191,6 +193,19 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 		goto err_vq;
 	}
 
+	/* reset virtqueue state index */
+	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
+		struct vdpa_vq_state_packed *s = &state.packed;
+
+		s->last_avail_counter = 1;
+		s->last_avail_idx = 0;
+		s->last_used_counter = 1;
+		s->last_used_idx = 0;
+	}
+	err = ops->set_vq_state(vdpa, index, &state);
+	if (err)
+		goto err_vq;
+
 	ops->set_vq_ready(vdpa, index, 1);
 
 	vq->priv = info;
-- 
2.25.1

Re: [PATCH 1/4] vdpa: support packed virtqueue for set/get_vq_state()

[Eli Cohen]

On Tue, Jun 01, 2021 at 04:45:00PM +0800, Jason Wang wrote:
> This patch extends the vdpa_vq_state to support packed virtqueue
> state which is basically the device/driver ring wrap counters and the
> avail and used index. This will be used for the virito-vdpa support
> for the packed virtqueue and the future vhost/vhost-vdpa support for
> the packed virtqueue.
> 
> Signed-off-by: Jason Wang <jasowang@redhat.com>

You changed interface but did not modify mlx5. Does this compile on your
system?

> ---
>  drivers/vdpa/ifcvf/ifcvf_main.c  |  4 ++--
>  drivers/vdpa/vdpa_sim/vdpa_sim.c |  4 ++--
>  drivers/vhost/vdpa.c             |  4 ++--
>  include/linux/vdpa.h             | 25 +++++++++++++++++++++++--
>  4 files changed, 29 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> index ab0ab5cf0f6e..5d3891b1ca28 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> @@ -264,7 +264,7 @@ static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
>  {
>  	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>  
> -	state->avail_index = ifcvf_get_vq_state(vf, qid);
> +	state->split.avail_index = ifcvf_get_vq_state(vf, qid);
>  	return 0;
>  }
>  
> @@ -273,7 +273,7 @@ static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
>  {
>  	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>  
> -	return ifcvf_set_vq_state(vf, qid, state->avail_index);
> +	return ifcvf_set_vq_state(vf, qid, state->split.avail_index);
>  }
>  
>  static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> index 98f793bc9376..14e024de5cbf 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> @@ -374,7 +374,7 @@ static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
>  	struct vringh *vrh = &vq->vring;
>  
>  	spin_lock(&vdpasim->lock);
> -	vrh->last_avail_idx = state->avail_index;
> +	vrh->last_avail_idx = state->split.avail_index;
>  	spin_unlock(&vdpasim->lock);
>  
>  	return 0;
> @@ -387,7 +387,7 @@ static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx,
>  	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
>  	struct vringh *vrh = &vq->vring;
>  
> -	state->avail_index = vrh->last_avail_idx;
> +	state->split.avail_index = vrh->last_avail_idx;
>  	return 0;
>  }
>  
> diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
> index fb41db3da611..210ab35a7ebf 100644
> --- a/drivers/vhost/vdpa.c
> +++ b/drivers/vhost/vdpa.c
> @@ -383,7 +383,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
>  		if (r)
>  			return r;
>  
> -		vq->last_avail_idx = vq_state.avail_index;
> +		vq->last_avail_idx = vq_state.split.avail_index;
>  		break;
>  	}
>  
> @@ -401,7 +401,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
>  		break;
>  
>  	case VHOST_SET_VRING_BASE:
> -		vq_state.avail_index = vq->last_avail_idx;
> +		vq_state.split.avail_index = vq->last_avail_idx;
>  		if (ops->set_vq_state(vdpa, idx, &vq_state))
>  			r = -EINVAL;
>  		break;
> diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
> index f311d227aa1b..3357ac98878d 100644
> --- a/include/linux/vdpa.h
> +++ b/include/linux/vdpa.h
> @@ -28,13 +28,34 @@ struct vdpa_notification_area {
>  };
>  
>  /**
> - * struct vdpa_vq_state - vDPA vq_state definition
> + * struct vdpa_vq_state_split - vDPA split virtqueue state
>   * @avail_index: available index
>   */
> -struct vdpa_vq_state {
> +struct vdpa_vq_state_split {
>  	u16	avail_index;
>  };
>  
> +/**
> + * struct vdpa_vq_state_packed - vDPA packed virtqueue state
> + * @last_avail_counter: last driver ring wrap counter observed by device
> + * @last_avail_idx: device available index
> + * @last_used_counter: device ring wrap counter
> + * @last_used_idx: used index
> + */
> +struct vdpa_vq_state_packed {
> +        u16	last_avail_counter:1;
> +        u16	last_avail_idx:15;
> +        u16	last_used_counter:1;
> +        u16	last_used_idx:15;
> +};
> +
> +struct vdpa_vq_state {
> +     union {
> +          struct vdpa_vq_state_split split;
> +          struct vdpa_vq_state_packed packed;
> +     };
> +};
> +
>  struct vdpa_mgmt_dev;
>  
>  /**
> -- 
> 2.25.1
> 

Re: [PATCH 1/4] vdpa: support packed virtqueue for set/get_vq_state()

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 6033 bytes --]

Hi Jason,

I love your patch! Yet something to improve:

[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.13-rc4 next-20210601]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Jason-Wang/Packed-virtqueue-state-support-for-vDPA/20210601-164715
base:   https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next
config: alpha-allyesconfig (attached as .config)
compiler: alpha-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/0day-ci/linux/commit/eccc56e52d8c89dd93da5df0362931151417eb6a
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Jason-Wang/Packed-virtqueue-state-support-for-vDPA/20210601-164715
        git checkout eccc56e52d8c89dd93da5df0362931151417eb6a
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=alpha 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   drivers/vdpa/mlx5/net/mlx5_vnet.c: In function 'mlx5_vdpa_set_vq_state':
>> drivers/vdpa/mlx5/net/mlx5_vnet.c:1430:23: error: 'const struct vdpa_vq_state' has no member named 'avail_index'
    1430 |  mvq->used_idx = state->avail_index;
         |                       ^~
   drivers/vdpa/mlx5/net/mlx5_vnet.c:1431:24: error: 'const struct vdpa_vq_state' has no member named 'avail_index'
    1431 |  mvq->avail_idx = state->avail_index;
         |                        ^~
   drivers/vdpa/mlx5/net/mlx5_vnet.c: In function 'mlx5_vdpa_get_vq_state':
>> drivers/vdpa/mlx5/net/mlx5_vnet.c:1452:8: error: 'struct vdpa_vq_state' has no member named 'avail_index'
    1452 |   state->avail_index = mvq->used_idx;
         |        ^~
   drivers/vdpa/mlx5/net/mlx5_vnet.c:1461:7: error: 'struct vdpa_vq_state' has no member named 'avail_index'
    1461 |  state->avail_index = attr.used_index;
         |       ^~


vim +1430 drivers/vdpa/mlx5/net/mlx5_vnet.c

1a86b377aa2147 Eli Cohen  2020-08-04  1417  
1a86b377aa2147 Eli Cohen  2020-08-04  1418  static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx,
1a86b377aa2147 Eli Cohen  2020-08-04  1419  				  const struct vdpa_vq_state *state)
1a86b377aa2147 Eli Cohen  2020-08-04  1420  {
1a86b377aa2147 Eli Cohen  2020-08-04  1421  	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1a86b377aa2147 Eli Cohen  2020-08-04  1422  	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1a86b377aa2147 Eli Cohen  2020-08-04  1423  	struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1a86b377aa2147 Eli Cohen  2020-08-04  1424  
1a86b377aa2147 Eli Cohen  2020-08-04  1425  	if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) {
1a86b377aa2147 Eli Cohen  2020-08-04  1426  		mlx5_vdpa_warn(mvdev, "can't modify available index\n");
1a86b377aa2147 Eli Cohen  2020-08-04  1427  		return -EINVAL;
1a86b377aa2147 Eli Cohen  2020-08-04  1428  	}
1a86b377aa2147 Eli Cohen  2020-08-04  1429  
bc04d93ea30a0a Eli Cohen  2021-04-08 @1430  	mvq->used_idx = state->avail_index;
1a86b377aa2147 Eli Cohen  2020-08-04  1431  	mvq->avail_idx = state->avail_index;
1a86b377aa2147 Eli Cohen  2020-08-04  1432  	return 0;
1a86b377aa2147 Eli Cohen  2020-08-04  1433  }
1a86b377aa2147 Eli Cohen  2020-08-04  1434  
1a86b377aa2147 Eli Cohen  2020-08-04  1435  static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
1a86b377aa2147 Eli Cohen  2020-08-04  1436  {
1a86b377aa2147 Eli Cohen  2020-08-04  1437  	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1a86b377aa2147 Eli Cohen  2020-08-04  1438  	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1a86b377aa2147 Eli Cohen  2020-08-04  1439  	struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1a86b377aa2147 Eli Cohen  2020-08-04  1440  	struct mlx5_virtq_attr attr;
1a86b377aa2147 Eli Cohen  2020-08-04  1441  	int err;
1a86b377aa2147 Eli Cohen  2020-08-04  1442  
3176e974a750d6 Si-Wei Liu 2020-10-01  1443  	/* If the virtq object was destroyed, use the value saved at
3176e974a750d6 Si-Wei Liu 2020-10-01  1444  	 * the last minute of suspend_vq. This caters for userspace
3176e974a750d6 Si-Wei Liu 2020-10-01  1445  	 * that cares about emulating the index after vq is stopped.
3176e974a750d6 Si-Wei Liu 2020-10-01  1446  	 */
3176e974a750d6 Si-Wei Liu 2020-10-01  1447  	if (!mvq->initialized) {
bc04d93ea30a0a Eli Cohen  2021-04-08  1448  		/* Firmware returns a wrong value for the available index.
bc04d93ea30a0a Eli Cohen  2021-04-08  1449  		 * Since both values should be identical, we take the value of
bc04d93ea30a0a Eli Cohen  2021-04-08  1450  		 * used_idx which is reported correctly.
bc04d93ea30a0a Eli Cohen  2021-04-08  1451  		 */
bc04d93ea30a0a Eli Cohen  2021-04-08 @1452  		state->avail_index = mvq->used_idx;
3176e974a750d6 Si-Wei Liu 2020-10-01  1453  		return 0;
3176e974a750d6 Si-Wei Liu 2020-10-01  1454  	}
1a86b377aa2147 Eli Cohen  2020-08-04  1455  
1a86b377aa2147 Eli Cohen  2020-08-04  1456  	err = query_virtqueue(ndev, mvq, &attr);
1a86b377aa2147 Eli Cohen  2020-08-04  1457  	if (err) {
1a86b377aa2147 Eli Cohen  2020-08-04  1458  		mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n");
1a86b377aa2147 Eli Cohen  2020-08-04  1459  		return err;
1a86b377aa2147 Eli Cohen  2020-08-04  1460  	}
bc04d93ea30a0a Eli Cohen  2021-04-08  1461  	state->avail_index = attr.used_index;
1a86b377aa2147 Eli Cohen  2020-08-04  1462  	return 0;
1a86b377aa2147 Eli Cohen  2020-08-04  1463  }
1a86b377aa2147 Eli Cohen  2020-08-04  1464  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 67748 bytes --]

Re: [PATCH 1/4] vdpa: support packed virtqueue for set/get_vq_state()

[Jason Wang]

在 2021/6/1 下午6:47, Eli Cohen 写道:
> On Tue, Jun 01, 2021 at 04:45:00PM +0800, Jason Wang wrote:
>> This patch extends the vdpa_vq_state to support packed virtqueue
>> state which is basically the device/driver ring wrap counters and the
>> avail and used index. This will be used for the virito-vdpa support
>> for the packed virtqueue and the future vhost/vhost-vdpa support for
>> the packed virtqueue.
>>
>> Signed-off-by: Jason Wang <jasowang@redhat.com>
> You changed interface but did not modify mlx5. Does this compile on your
> system?


Yes but I'm using a minimal config without mlx5 enabled :(

V2 is posted.

Thanks


>
>> ---
>>   drivers/vdpa/ifcvf/ifcvf_main.c  |  4 ++--
>>   drivers/vdpa/vdpa_sim/vdpa_sim.c |  4 ++--
>>   drivers/vhost/vdpa.c             |  4 ++--
>>   include/linux/vdpa.h             | 25 +++++++++++++++++++++++--
>>   4 files changed, 29 insertions(+), 8 deletions(-)
>>
>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
>> index ab0ab5cf0f6e..5d3891b1ca28 100644
>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>> @@ -264,7 +264,7 @@ static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
>>   {
>>   	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>>   
>> -	state->avail_index = ifcvf_get_vq_state(vf, qid);
>> +	state->split.avail_index = ifcvf_get_vq_state(vf, qid);
>>   	return 0;
>>   }
>>   
>> @@ -273,7 +273,7 @@ static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
>>   {
>>   	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>>   
>> -	return ifcvf_set_vq_state(vf, qid, state->avail_index);
>> +	return ifcvf_set_vq_state(vf, qid, state->split.avail_index);
>>   }
>>   
>>   static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
>> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
>> index 98f793bc9376..14e024de5cbf 100644
>> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
>> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
>> @@ -374,7 +374,7 @@ static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
>>   	struct vringh *vrh = &vq->vring;
>>   
>>   	spin_lock(&vdpasim->lock);
>> -	vrh->last_avail_idx = state->avail_index;
>> +	vrh->last_avail_idx = state->split.avail_index;
>>   	spin_unlock(&vdpasim->lock);
>>   
>>   	return 0;
>> @@ -387,7 +387,7 @@ static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx,
>>   	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
>>   	struct vringh *vrh = &vq->vring;
>>   
>> -	state->avail_index = vrh->last_avail_idx;
>> +	state->split.avail_index = vrh->last_avail_idx;
>>   	return 0;
>>   }
>>   
>> diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
>> index fb41db3da611..210ab35a7ebf 100644
>> --- a/drivers/vhost/vdpa.c
>> +++ b/drivers/vhost/vdpa.c
>> @@ -383,7 +383,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
>>   		if (r)
>>   			return r;
>>   
>> -		vq->last_avail_idx = vq_state.avail_index;
>> +		vq->last_avail_idx = vq_state.split.avail_index;
>>   		break;
>>   	}
>>   
>> @@ -401,7 +401,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
>>   		break;
>>   
>>   	case VHOST_SET_VRING_BASE:
>> -		vq_state.avail_index = vq->last_avail_idx;
>> +		vq_state.split.avail_index = vq->last_avail_idx;
>>   		if (ops->set_vq_state(vdpa, idx, &vq_state))
>>   			r = -EINVAL;
>>   		break;
>> diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
>> index f311d227aa1b..3357ac98878d 100644
>> --- a/include/linux/vdpa.h
>> +++ b/include/linux/vdpa.h
>> @@ -28,13 +28,34 @@ struct vdpa_notification_area {
>>   };
>>   
>>   /**
>> - * struct vdpa_vq_state - vDPA vq_state definition
>> + * struct vdpa_vq_state_split - vDPA split virtqueue state
>>    * @avail_index: available index
>>    */
>> -struct vdpa_vq_state {
>> +struct vdpa_vq_state_split {
>>   	u16	avail_index;
>>   };
>>   
>> +/**
>> + * struct vdpa_vq_state_packed - vDPA packed virtqueue state
>> + * @last_avail_counter: last driver ring wrap counter observed by device
>> + * @last_avail_idx: device available index
>> + * @last_used_counter: device ring wrap counter
>> + * @last_used_idx: used index
>> + */
>> +struct vdpa_vq_state_packed {
>> +        u16	last_avail_counter:1;
>> +        u16	last_avail_idx:15;
>> +        u16	last_used_counter:1;
>> +        u16	last_used_idx:15;
>> +};
>> +
>> +struct vdpa_vq_state {
>> +     union {
>> +          struct vdpa_vq_state_split split;
>> +          struct vdpa_vq_state_packed packed;
>> +     };
>> +};
>> +
>>   struct vdpa_mgmt_dev;
>>   
>>   /**
>> -- 
>> 2.25.1
>>