From: Alex Williamson <alex.williamson@redhat.com>
To: Steve Sistare <steven.sistare@oracle.com>
Cc: "Daniel P. Berrange" <berrange@redhat.com>,
"Michael S. Tsirkin" <mst@redhat.com>,
"Jason Zeng" <jason.zeng@linux.intel.com>,
"Alex Bennée" <alex.bennee@linaro.org>,
"Juan Quintela" <quintela@redhat.com>,
qemu-devel@nongnu.org,
"Dr. David Alan Gilbert" <dgilbert@redhat.com>,
"Stefan Hajnoczi" <stefanha@redhat.com>,
"Marc-André Lureau" <marcandre.lureau@redhat.com>,
"Paolo Bonzini" <pbonzini@redhat.com>,
"Philippe Mathieu-Daudé" <philmd@redhat.com>,
"Markus Armbruster" <armbru@redhat.com>
Subject: Re: [PATCH V3 13/22] vfio-pci: cpr part 2
Date: Fri, 21 May 2021 16:24:48 -0600 [thread overview]
Message-ID: <20210521162448.65ec5f58.alex.williamson@redhat.com> (raw)
In-Reply-To: <1620390320-301716-14-git-send-email-steven.sistare@oracle.com>
On Fri, 7 May 2021 05:25:11 -0700
Steve Sistare <steven.sistare@oracle.com> wrote:
> Finish cpr for vfio-pci by preserving eventfd's and vector state.
>
> Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
> ---
> hw/vfio/pci.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
> 1 file changed, 108 insertions(+), 2 deletions(-)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index f7ac9f03..e983db4 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -2661,6 +2661,27 @@ static void vfio_put_device(VFIOPCIDevice *vdev)
> vfio_put_base_device(&vdev->vbasedev);
> }
>
> +static void setenv_event_fd(VFIOPCIDevice *vdev, int nr, const char *name,
> + EventNotifier *ev)
> +{
> + char envname[256];
> + int fd = event_notifier_get_fd(ev);
> + const char *vfname = vdev->vbasedev.name;
> +
> + if (fd >= 0) {
> + snprintf(envname, sizeof(envname), "%s_%s_%d", vfname, name, nr);
> + setenv_fd(envname, fd);
> + }
> +}
> +
> +static int getenv_event_fd(VFIOPCIDevice *vdev, int nr, const char *name)
> +{
> + char envname[256];
> + const char *vfname = vdev->vbasedev.name;
> + snprintf(envname, sizeof(envname), "%s_%s_%d", vfname, name, nr);
> + return getenv_fd(envname);
> +}
> +
> static void vfio_err_notifier_handler(void *opaque)
> {
> VFIOPCIDevice *vdev = opaque;
> @@ -2692,7 +2713,13 @@ static void vfio_err_notifier_handler(void *opaque)
> static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
> {
> Error *err = NULL;
> - int32_t fd;
> + int32_t fd = getenv_event_fd(vdev, 0, "err");
Arg order should match the actual env names, device name, interrupt
name, interrupt number.
> +
> + if (fd >= 0) {
> + event_notifier_init_fd(&vdev->err_notifier, fd);
> + qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
> + return;
> + }
>
> if (!vdev->pci_aer) {
> return;
> @@ -2753,7 +2780,14 @@ static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
> struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
> .index = VFIO_PCI_REQ_IRQ_INDEX };
> Error *err = NULL;
> - int32_t fd;
> + int32_t fd = getenv_event_fd(vdev, 0, "req");
> +
> + if (fd >= 0) {
> + event_notifier_init_fd(&vdev->req_notifier, fd);
> + qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
> + vdev->req_enabled = true;
> + return;
> + }
>
> if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
> return;
> @@ -3286,12 +3320,82 @@ static Property vfio_pci_dev_properties[] = {
> DEFINE_PROP_END_OF_LIST(),
> };
>
> +static int vfio_pci_pre_save(void *opaque)
> +{
> + VFIOPCIDevice *vdev = opaque;
> + int i;
> +
> + for (i = 0; i < vdev->nr_vectors; i++) {
> + VFIOMSIVector *vector = &vdev->msi_vectors[i];
> + if (vector->use) {
> + setenv_event_fd(vdev, i, "interrupt", &vector->interrupt);
> + if (vector->virq >= 0) {
> + setenv_event_fd(vdev, i, "kvm_interrupt",
> + &vector->kvm_interrupt);
> + }
> + }
> + }
> + setenv_event_fd(vdev, 0, "err", &vdev->err_notifier);
> + setenv_event_fd(vdev, 0, "req", &vdev->req_notifier);
> + return 0;
> +}
> +
> +static void vfio_claim_vectors(VFIOPCIDevice *vdev, int nr_vectors, bool msix)
> +{
> + int i, fd;
> + bool pending = false;
> + PCIDevice *pdev = &vdev->pdev;
> +
> + vdev->nr_vectors = nr_vectors;
> + vdev->msi_vectors = g_new0(VFIOMSIVector, nr_vectors);
> + vdev->interrupt = msix ? VFIO_INT_MSIX : VFIO_INT_MSI;
> +
> + for (i = 0; i < nr_vectors; i++) {
> + VFIOMSIVector *vector = &vdev->msi_vectors[i];
> +
> + fd = getenv_event_fd(vdev, i, "interrupt");
> + if (fd >= 0) {
> + vfio_vector_init(vdev, i, fd);
> + qemu_set_fd_handler(fd, vfio_msi_interrupt, NULL, vector);
> + }
> +
> + fd = getenv_event_fd(vdev, i, "kvm_interrupt");
> + if (fd >= 0) {
> + vfio_add_kvm_msi_virq(vdev, vector, i, msix, fd);
> + }
> +
> + if (msix && msix_is_pending(pdev, i) && msix_is_masked(pdev, i)) {
> + set_bit(i, vdev->msix->pending);
> + pending = true;
> + }
> + }
> +
> + if (msix) {
> + memory_region_set_enabled(&pdev->msix_pba_mmio, pending);
> + }
> +}
> +
> static int vfio_pci_post_load(void *opaque, int version_id)
> {
> VFIOPCIDevice *vdev = opaque;
> PCIDevice *pdev = &vdev->pdev;
> + int nr_vectors;
> bool enabled;
>
> + if (msix_enabled(pdev)) {
> + nr_vectors = vdev->msix->entries;
> + vfio_claim_vectors(vdev, nr_vectors, true);
> + msix_init_vector_notifiers(pdev, vfio_msix_vector_use,
> + vfio_msix_vector_release, NULL);
> +
> + } else if (msi_enabled(pdev)) {
> + nr_vectors = msi_nr_vectors_allocated(pdev);
> + vfio_claim_vectors(vdev, nr_vectors, false);
> +
> + } else if (vfio_pci_read_config(pdev, PCI_INTERRUPT_PIN, 1)) {
> + error_report("vfio_pci_post_load does not support INTX");
> + }
Why? Is post-load where we really want to find this out? Thanks,
Alex
> +
> pdev->reused = false;
> enabled = pci_get_word(pdev->config + PCI_COMMAND) & PCI_COMMAND_MASTER;
> memory_region_set_enabled(&pdev->bus_master_enable_region, enabled);
> @@ -3310,8 +3414,10 @@ static const VMStateDescription vfio_pci_vmstate = {
> .version_id = 0,
> .minimum_version_id = 0,
> .post_load = vfio_pci_post_load,
> + .pre_save = vfio_pci_pre_save,
> .needed = vfio_pci_needed,
> .fields = (VMStateField[]) {
> + VMSTATE_MSIX(pdev, VFIOPCIDevice),
> VMSTATE_END_OF_LIST()
> }
> };
next prev parent reply other threads:[~2021-05-21 22:25 UTC|newest]
Thread overview: 81+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-07 12:24 [PATCH V3 00/22] Live Update Steve Sistare
2021-05-07 12:24 ` [PATCH V3 01/22] as_flat_walk Steve Sistare
2021-05-07 12:25 ` [PATCH V3 02/22] qemu_ram_volatile Steve Sistare
2021-05-07 12:25 ` [PATCH V3 03/22] oslib: qemu_clr_cloexec Steve Sistare
2021-05-07 12:25 ` [PATCH V3 04/22] util: env var helpers Steve Sistare
2021-05-07 12:25 ` [PATCH V3 05/22] machine: memfd-alloc option Steve Sistare
2021-05-07 12:25 ` [PATCH V3 06/22] vl: add helper to request re-exec Steve Sistare
2021-05-07 14:31 ` Eric Blake
2021-05-13 20:19 ` Steven Sistare
2021-05-14 8:18 ` Daniel P. Berrangé
2021-05-12 16:27 ` Stefan Hajnoczi
2021-05-13 20:20 ` Steven Sistare
2021-05-07 12:25 ` [PATCH V3 07/22] cpr Steve Sistare
2021-05-12 16:19 ` Stefan Hajnoczi
2021-05-13 20:21 ` Steven Sistare
2021-05-14 11:28 ` Stefan Hajnoczi
2021-05-14 15:14 ` Steven Sistare
2021-05-18 13:42 ` Stefan Hajnoczi
2021-05-07 12:25 ` [PATCH V3 08/22] cpr: QMP interfaces Steve Sistare
2021-06-04 13:59 ` Eric Blake
2021-06-07 17:19 ` Steven Sistare
2021-05-07 12:25 ` [PATCH V3 09/22] cpr: HMP interfaces Steve Sistare
2021-05-07 12:25 ` [PATCH V3 10/22] pci: export functions for cpr Steve Sistare
2021-05-07 12:25 ` [PATCH V3 11/22] vfio-pci: refactor " Steve Sistare
2021-05-19 22:38 ` Alex Williamson
2021-05-21 13:33 ` Steven Sistare
2021-05-21 21:07 ` Alex Williamson
2021-05-21 21:18 ` Steven Sistare
2021-05-07 12:25 ` [PATCH V3 12/22] vfio-pci: cpr part 1 Steve Sistare
2021-05-21 22:24 ` Alex Williamson
2021-05-24 18:29 ` Steven Sistare
2021-06-11 18:15 ` Steven Sistare
2021-06-11 19:43 ` Steven Sistare
2021-05-07 12:25 ` [PATCH V3 13/22] vfio-pci: cpr part 2 Steve Sistare
2021-05-21 22:24 ` Alex Williamson [this message]
2021-05-24 18:31 ` Steven Sistare
2021-05-07 12:25 ` [PATCH V3 14/22] vhost: reset vhost devices upon cprsave Steve Sistare
2021-05-07 12:25 ` [PATCH V3 15/22] hostmem-memfd: cpr support Steve Sistare
2021-05-07 12:25 ` [PATCH V3 16/22] chardev: cpr framework Steve Sistare
2021-05-07 14:33 ` Eric Blake
2021-05-13 20:19 ` Steven Sistare
2021-05-07 12:25 ` [PATCH V3 17/22] chardev: cpr for simple devices Steve Sistare
2021-05-07 12:25 ` [PATCH V3 18/22] chardev: cpr for pty Steve Sistare
2021-05-07 12:25 ` [PATCH V3 19/22] chardev: cpr for sockets Steve Sistare
2021-05-07 12:25 ` [PATCH V3 20/22] cpr: only-cpr-capable option Steve Sistare
2021-05-07 12:25 ` [PATCH V3 21/22] cpr: maintainers Steve Sistare
2021-05-07 12:25 ` [PATCH V3 22/22] simplify savevm Steve Sistare
2021-05-07 13:00 ` [PATCH V3 00/22] Live Update no-reply
2021-05-13 20:42 ` Steven Sistare
2021-05-12 16:42 ` Stefan Hajnoczi
2021-05-13 20:21 ` Steven Sistare
2021-05-14 11:53 ` Stefan Hajnoczi
2021-05-14 15:15 ` Steven Sistare
2021-05-17 11:40 ` Stefan Hajnoczi
2021-05-17 19:10 ` Alex Williamson
2021-05-18 13:39 ` Stefan Hajnoczi
2021-05-18 15:48 ` Steven Sistare
2021-05-18 9:57 ` Dr. David Alan Gilbert
2021-05-18 16:00 ` Steven Sistare
2021-05-18 19:23 ` Dr. David Alan Gilbert
2021-05-18 20:01 ` Alex Williamson
2021-05-18 20:14 ` Steven Sistare
2021-05-20 13:00 ` [PATCH V3 00/22] Live Update [reboot] Dr. David Alan Gilbert
2021-05-21 14:55 ` Steven Sistare
2021-06-15 19:14 ` Dr. David Alan Gilbert
2021-06-24 15:05 ` Steven Sistare
2021-07-06 17:31 ` Steven Sistare
2021-05-20 13:13 ` [PATCH V3 00/22] Live Update [restart] Dr. David Alan Gilbert
2021-05-21 14:56 ` Steven Sistare
2021-05-24 10:39 ` Dr. David Alan Gilbert
2021-06-02 13:51 ` Steven Sistare
2021-06-03 19:36 ` Dr. David Alan Gilbert
2021-06-03 20:44 ` Daniel P. Berrangé
2021-06-07 16:40 ` [PATCH V3 00/22] Live Update [restart] : exec Steven Sistare
2021-06-14 14:31 ` Steven Sistare
2021-06-14 14:36 ` Daniel P. Berrangé
2021-06-15 19:05 ` Dr. David Alan Gilbert
2021-06-07 18:08 ` [PATCH V3 00/22] Live Update [restart] : code replication Steven Sistare
2021-06-14 14:33 ` Steven Sistare
2021-05-19 16:43 ` [PATCH V3 00/22] Live Update Steven Sistare
2021-06-02 15:19 ` Steven Sistare
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210521162448.65ec5f58.alex.williamson@redhat.com \
--to=alex.williamson@redhat.com \
--cc=alex.bennee@linaro.org \
--cc=armbru@redhat.com \
--cc=berrange@redhat.com \
--cc=dgilbert@redhat.com \
--cc=jason.zeng@linux.intel.com \
--cc=marcandre.lureau@redhat.com \
--cc=mst@redhat.com \
--cc=pbonzini@redhat.com \
--cc=philmd@redhat.com \
--cc=qemu-devel@nongnu.org \
--cc=quintela@redhat.com \
--cc=stefanha@redhat.com \
--cc=steven.sistare@oracle.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).