From: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
To: <kvm@vger.kernel.org>, <linux-kernel@vger.kernel.org>,
<linux-crypto@vger.kernel.org>
Cc: <linux-pci@vger.kernel.org>, <alex.williamson@redhat.com>,
<jgg@nvidia.com>, <cohuck@redhat.com>, <mgurtovoy@nvidia.com>,
<yishaih@nvidia.com>, <linuxarm@huawei.com>,
<liulongfang@huawei.com>, <prime.zeng@hisilicon.com>,
<jonathan.cameron@huawei.com>, <wangzhou1@hisilicon.com>
Subject: [PATCH v7 10/10] hisi_acc_vfio_pci: Use its own PCI reset_done error handler
Date: Wed, 2 Mar 2022 17:29:03 +0000 [thread overview]
Message-ID: <20220302172903.1995-11-shameerali.kolothum.thodi@huawei.com> (raw)
In-Reply-To: <20220302172903.1995-1-shameerali.kolothum.thodi@huawei.com>
Register private handler for pci_error_handlers.reset_done and update
state accordingly.
Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
---
.../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 61 +++++++++++++++++--
.../vfio/pci/hisilicon/hisi_acc_vfio_pci.h | 4 +-
2 files changed, 59 insertions(+), 6 deletions(-)
diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
index f733aa0b1a5b..906690e57020 100644
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -515,6 +515,27 @@ static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vde
}
}
+/*
+ * This function is called in all state_mutex unlock cases to
+ * handle a 'deferred_reset' if exists.
+ */
+static void
+hisi_acc_vf_state_mutex_unlock(struct hisi_acc_vf_core_device *hisi_acc_vdev)
+{
+again:
+ spin_lock(&hisi_acc_vdev->reset_lock);
+ if (hisi_acc_vdev->deferred_reset) {
+ hisi_acc_vdev->deferred_reset = false;
+ spin_unlock(&hisi_acc_vdev->reset_lock);
+ hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
+ hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
+ hisi_acc_vf_disable_fds(hisi_acc_vdev);
+ goto again;
+ }
+ mutex_unlock(&hisi_acc_vdev->state_mutex);
+ spin_unlock(&hisi_acc_vdev->reset_lock);
+}
+
static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev,
struct hisi_acc_vf_migration_file *migf)
{
@@ -750,7 +771,7 @@ static long hisi_acc_vf_save_unl_ioctl(struct file *filp,
mutex_lock(&hisi_acc_vdev->state_mutex);
if (hisi_acc_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY) {
- mutex_unlock(&hisi_acc_vdev->state_mutex);
+ hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
return -EINVAL;
}
@@ -772,7 +793,7 @@ static long hisi_acc_vf_save_unl_ioctl(struct file *filp,
ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
out:
mutex_unlock(&migf->lock);
- mutex_unlock(&hisi_acc_vdev->state_mutex);
+ hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
return ret;
}
@@ -967,7 +988,7 @@ hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev,
break;
}
}
- mutex_unlock(&hisi_acc_vdev->state_mutex);
+ hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
return res;
}
@@ -980,10 +1001,35 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,
mutex_lock(&hisi_acc_vdev->state_mutex);
*curr_state = hisi_acc_vdev->mig_state;
- mutex_unlock(&hisi_acc_vdev->state_mutex);
+ hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
return 0;
}
+static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
+{
+ struct hisi_acc_vf_core_device *hisi_acc_vdev = dev_get_drvdata(&pdev->dev);
+
+ if (hisi_acc_vdev->core_device.vdev.migration_flags !=
+ VFIO_MIGRATION_STOP_COPY)
+ return;
+
+ /*
+ * As the higher VFIO layers are holding locks across reset and using
+ * those same locks with the mm_lock we need to prevent ABBA deadlock
+ * with the state_mutex and mm_lock.
+ * In case the state_mutex was taken already we defer the cleanup work
+ * to the unlock flow of the other running context.
+ */
+ spin_lock(&hisi_acc_vdev->reset_lock);
+ hisi_acc_vdev->deferred_reset = true;
+ if (!mutex_trylock(&hisi_acc_vdev->state_mutex)) {
+ spin_unlock(&hisi_acc_vdev->reset_lock);
+ return;
+ }
+ spin_unlock(&hisi_acc_vdev->reset_lock);
+ hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
+}
+
static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
{
struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
@@ -1302,12 +1348,17 @@ static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);
+static const struct pci_error_handlers hisi_acc_vf_err_handlers = {
+ .reset_done = hisi_acc_vf_pci_aer_reset_done,
+ .error_detected = vfio_pci_core_aer_err_detected,
+};
+
static struct pci_driver hisi_acc_vfio_pci_driver = {
.name = KBUILD_MODNAME,
.id_table = hisi_acc_vfio_pci_table,
.probe = hisi_acc_vfio_pci_probe,
.remove = hisi_acc_vfio_pci_remove,
- .err_handler = &vfio_pci_core_err_handlers,
+ .err_handler = &hisi_acc_vf_err_handlers,
};
module_pci_driver(hisi_acc_vfio_pci_driver);
diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
index 51bc7e92a776..6c18f7c74f34 100644
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
@@ -96,6 +96,7 @@ struct hisi_acc_vf_migration_file {
struct hisi_acc_vf_core_device {
struct vfio_pci_core_device core_device;
u8 match_done:1;
+ u8 deferred_reset:1;
/* for migration state */
struct mutex state_mutex;
enum vfio_device_mig_state mig_state;
@@ -105,7 +106,8 @@ struct hisi_acc_vf_core_device {
struct hisi_qm vf_qm;
u32 vf_qm_state;
int vf_id;
-
+ /* for reset handler */
+ spinlock_t reset_lock;
struct hisi_acc_vf_migration_file resuming_migf;
struct hisi_acc_vf_migration_file saving_migf;
};
--
2.25.1
prev parent reply other threads:[~2022-03-02 17:31 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-03-02 17:28 [PATCH v7 00/10] vfio/hisilicon: add ACC live migration driver Shameer Kolothum
2022-03-02 17:28 ` [PATCH v7 01/10] crypto: hisilicon/qm: Move the QM header to include/linux Shameer Kolothum
2022-03-02 18:48 ` John Garry
2022-03-02 20:39 ` Shameerali Kolothum Thodi
2022-03-02 17:28 ` [PATCH v7 02/10] crypto: hisilicon/qm: Move few definitions to common header Shameer Kolothum
2022-03-02 18:55 ` John Garry
2022-03-02 20:41 ` Shameerali Kolothum Thodi
2022-03-02 17:28 ` [PATCH v7 03/10] hisi_acc_qm: Move VF PCI device IDs " Shameer Kolothum
2022-03-02 17:28 ` [PATCH v7 04/10] hisi_acc_vfio_pci: add new vfio_pci driver for HiSilicon ACC devices Shameer Kolothum
2022-03-02 19:02 ` John Garry
2022-03-02 20:32 ` Shameerali Kolothum Thodi
2022-03-02 17:28 ` [PATCH v7 05/10] hisi_acc_vfio_pci: Restrict access to VF dev BAR2 migration region Shameer Kolothum
2022-03-02 17:28 ` [PATCH v7 06/10] hisi_acc_vfio_pci: Add helper to retrieve the struct pci_driver Shameer Kolothum
2022-03-02 17:29 ` [PATCH v7 07/10] vfio: Extend the device migration protocol with PRE_COPY Shameer Kolothum
2022-03-02 20:31 ` Alex Williamson
2022-03-03 0:05 ` Jason Gunthorpe
2022-03-03 3:47 ` Alex Williamson
2022-03-03 13:01 ` Jason Gunthorpe
2022-03-03 15:20 ` Alex Williamson
2022-03-03 18:05 ` Shameerali Kolothum Thodi
2022-03-03 19:59 ` Alex Williamson
2022-03-03 23:49 ` Jason Gunthorpe
2022-03-04 19:56 ` Alex Williamson
2022-03-02 17:29 ` [PATCH v7 08/10] crypto: hisilicon/qm: Set the VF QM state register Shameer Kolothum
2022-03-02 17:29 ` [PATCH v7 09/10] hisi_acc_vfio_pci: Add support for VFIO live migration Shameer Kolothum
2022-03-03 0:21 ` Jason Gunthorpe
2022-03-03 12:57 ` Shameerali Kolothum Thodi
2022-03-03 13:04 ` Jason Gunthorpe
2022-03-03 13:17 ` Shameerali Kolothum Thodi
2022-03-03 13:19 ` Jason Gunthorpe
2022-03-03 13:27 ` Shameerali Kolothum Thodi
2022-03-02 17:29 ` Shameer Kolothum [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220302172903.1995-11-shameerali.kolothum.thodi@huawei.com \
--to=shameerali.kolothum.thodi@huawei.com \
--cc=alex.williamson@redhat.com \
--cc=cohuck@redhat.com \
--cc=jgg@nvidia.com \
--cc=jonathan.cameron@huawei.com \
--cc=kvm@vger.kernel.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-pci@vger.kernel.org \
--cc=linuxarm@huawei.com \
--cc=liulongfang@huawei.com \
--cc=mgurtovoy@nvidia.com \
--cc=prime.zeng@hisilicon.com \
--cc=wangzhou1@hisilicon.com \
--cc=yishaih@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).