From: Jason Wang <jasowang@redhat.com>
To: mst@redhat.com, jasowang@redhat.com
Cc: elic@nvidia.com, virtualization@lists.linux-foundation.org,
linux-kernel@vger.kernel.org
Subject: [PATCH 2/2] vdpa: mlx5: synchronize driver status with CVQ
Date: Mon, 21 Mar 2022 14:04:29 +0800 [thread overview]
Message-ID: <20220321060429.10457-2-jasowang@redhat.com> (raw)
In-Reply-To: <20220321060429.10457-1-jasowang@redhat.com>
Currently, CVQ doesn't have any synchronization with the driver
status. Then CVQ emulation code run in the middle of:
1) device reset
2) device status changed
3) map updating
The will lead several unexpected issue like trying to execute CVQ
command after the driver has been teared down.
Fixing this by using reslock to synchronize CVQ emulation code with
the driver status changing:
- protect the whole device reset, status changing and map updating
with reslock
- protect the CVQ handler with the reslock and check
VIRTIO_CONFIG_S_DRIVER_OK in the CVQ handler
This will guarantee that:
1) CVQ handler won't work if VIRTIO_CONFIG_S_DRIVER_OK is not set
2) CVQ handler will see a consistent state of the driver instead of
the partial one when it is running in the middle of the
teardown_driver() or setup_driver().
Cc: 5262912ef3cfc ("vdpa/mlx5: Add support for control VQ and MAC setting")
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
drivers/vdpa/mlx5/net/mlx5_vnet.c | 42 +++++++++++++++++++++++--------
1 file changed, 31 insertions(+), 11 deletions(-)
diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c
index d5a6fb3f9c41..524240f55c1c 100644
--- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
+++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
@@ -1618,11 +1618,17 @@ static void mlx5_cvq_kick_handler(struct work_struct *work)
mvdev = wqent->mvdev;
ndev = to_mlx5_vdpa_ndev(mvdev);
cvq = &mvdev->cvq;
+
+ mutex_lock(&ndev->reslock);
+
+ if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
+ goto done;
+
if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))
- return;
+ goto done;
if (!cvq->ready)
- return;
+ goto done;
while (true) {
err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head,
@@ -1663,6 +1669,9 @@ static void mlx5_cvq_kick_handler(struct work_struct *work)
break;
}
}
+
+done:
+ mutex_unlock(&ndev->reslock);
}
static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx)
@@ -2125,6 +2134,8 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
int err;
+ mutex_lock(&ndev->reslock);
+
suspend_vqs(ndev);
err = save_channels_info(ndev);
if (err)
@@ -2137,18 +2148,20 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb
goto err_mr;
if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
- return 0;
+ goto err_mr;
restore_channels_info(ndev);
err = setup_driver(mvdev);
if (err)
goto err_setup;
+ mutex_unlock(&ndev->reslock);
return 0;
err_setup:
mlx5_vdpa_destroy_mr(mvdev);
err_mr:
+ mutex_unlock(&ndev->reslock);
return err;
}
@@ -2157,7 +2170,8 @@ static int setup_driver(struct mlx5_vdpa_dev *mvdev)
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
int err;
- mutex_lock(&ndev->reslock);
+ WARN_ON(!mutex_is_locked(&ndev->reslock));
+
if (ndev->setup) {
mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n");
err = 0;
@@ -2187,7 +2201,6 @@ static int setup_driver(struct mlx5_vdpa_dev *mvdev)
goto err_fwd;
}
ndev->setup = true;
- mutex_unlock(&ndev->reslock);
return 0;
@@ -2198,23 +2211,22 @@ static int setup_driver(struct mlx5_vdpa_dev *mvdev)
err_rqt:
teardown_virtqueues(ndev);
out:
- mutex_unlock(&ndev->reslock);
return err;
}
static void teardown_driver(struct mlx5_vdpa_net *ndev)
{
- mutex_lock(&ndev->reslock);
+
+ WARN_ON(!mutex_is_locked(&ndev->reslock));
+
if (!ndev->setup)
- goto out;
+ return;
remove_fwd_to_tir(ndev);
destroy_tir(ndev);
destroy_rqt(ndev);
teardown_virtqueues(ndev);
ndev->setup = false;
-out:
- mutex_unlock(&ndev->reslock);
}
static void clear_vqs_ready(struct mlx5_vdpa_net *ndev)
@@ -2235,6 +2247,8 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
print_status(mvdev, status, true);
+ mutex_lock(&ndev->reslock);
+
if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) {
if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
err = setup_driver(mvdev);
@@ -2244,16 +2258,19 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
}
} else {
mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n");
- return;
+ goto err_clear;
}
}
ndev->mvdev.status = status;
+ mutex_unlock(&ndev->reslock);
return;
err_setup:
mlx5_vdpa_destroy_mr(&ndev->mvdev);
ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED;
+err_clear:
+ mutex_unlock(&ndev->reslock);
}
static int mlx5_vdpa_reset(struct vdpa_device *vdev)
@@ -2263,6 +2280,8 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
print_status(mvdev, 0, true);
mlx5_vdpa_info(mvdev, "performing device reset\n");
+
+ mutex_lock(&ndev->reslock);
teardown_driver(ndev);
clear_vqs_ready(ndev);
mlx5_vdpa_destroy_mr(&ndev->mvdev);
@@ -2275,6 +2294,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
if (mlx5_vdpa_create_mr(mvdev, NULL))
mlx5_vdpa_warn(mvdev, "create MR failed\n");
}
+ mutex_unlock(&ndev->reslock);
return 0;
}
--
2.18.1
next prev parent reply other threads:[~2022-03-21 6:04 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-03-21 6:04 [PATCH 1/2] vdpa: mlx5: prevent cvq work from hogging CPU Jason Wang
2022-03-21 6:04 ` Jason Wang [this message]
2022-03-21 6:56 ` [PATCH 2/2] vdpa: mlx5: synchronize driver status with CVQ Eli Cohen
2022-03-21 7:23 ` Michael S. Tsirkin
2022-03-21 7:36 ` Jason Wang
2022-03-21 6:31 ` [PATCH 1/2] vdpa: mlx5: prevent cvq work from hogging CPU Eli Cohen
2022-03-21 7:20 ` Michael S. Tsirkin
2022-03-21 7:35 ` Jason Wang
[not found] ` <20220321085317.3148-1-hdanton@sina.com>
2022-03-21 8:59 ` Jason Wang
2022-03-21 9:00 ` Jason Wang
[not found] ` <20220321123420.3207-1-hdanton@sina.com>
2022-03-22 1:59 ` Jason Wang
[not found] ` <20220324005345.3623-1-hdanton@sina.com>
2022-03-24 2:34 ` Jason Wang
[not found] ` <20220324060419.3682-1-hdanton@sina.com>
2022-03-24 6:17 ` Michael S. Tsirkin
2022-03-24 8:20 ` Jason Wang
[not found] ` <20220324120217.3746-1-hdanton@sina.com>
2022-03-24 12:24 ` Eli Cohen
2022-03-25 3:22 ` Jason Wang
2022-03-25 6:45 ` Michael S. Tsirkin
2022-03-25 7:53 ` Jason Wang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220321060429.10457-2-jasowang@redhat.com \
--to=jasowang@redhat.com \
--cc=elic@nvidia.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mst@redhat.com \
--cc=virtualization@lists.linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).