[RFC PATCH v5 10/26] vhost: Add Shadow VirtQueue kick forwarding capabilities

["Eugenio Pérez" <eperezma@redhat.com>]

At this mode no buffer forwarding will be performed in SVQ mode: Qemu
will just forward the guest's kicks to the device.

Also, host notifiers must be disabled at SVQ start, and they will not
start if SVQ has been enabled when device is stopped. This will be
addressed in next patches.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
---
 hw/virtio/vhost-shadow-virtqueue.h |  4 ++
 hw/virtio/vhost-shadow-virtqueue.c | 77 +++++++++++++++++++++++++++---
 2 files changed, 74 insertions(+), 7 deletions(-)

diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
index a19eede089..30ab9643b9 100644
--- a/hw/virtio/vhost-shadow-virtqueue.h
+++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -18,6 +18,10 @@ typedef struct VhostShadowVirtqueue VhostShadowVirtqueue;
 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
 const EventNotifier *vhost_svq_get_dev_kick_notifier(
                                               const VhostShadowVirtqueue *svq);
+void vhost_svq_start(struct vhost_dev *dev, unsigned idx,
+                     VhostShadowVirtqueue *svq, int svq_kick_fd);
+void vhost_svq_stop(struct vhost_dev *dev, unsigned idx,
+                    VhostShadowVirtqueue *svq);
 
 VhostShadowVirtqueue *vhost_svq_new(struct vhost_dev *dev, int idx);
 
diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
index 513d7f2782..fda60d11db 100644
--- a/hw/virtio/vhost-shadow-virtqueue.c
+++ b/hw/virtio/vhost-shadow-virtqueue.c
@@ -40,18 +40,36 @@ const EventNotifier *vhost_svq_get_dev_kick_notifier(
     return &svq->hdev_kick;
 }
 
+/* Forward guest notifications */
+static void vhost_handle_guest_kick(EventNotifier *n)
+{
+    VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
+                                             svq_kick);
+
+    if (unlikely(!event_notifier_test_and_clear(n))) {
+        return;
+    }
+
+    event_notifier_set(&svq->hdev_kick);
+}
+
 /**
- * Set a new file descriptor for the guest to kick SVQ and notify for avail
+ * Convenience function to set guest to SVQ kick fd
  *
- * @svq          The svq
- * @svq_kick_fd  The new svq kick fd
+ * @svq         The shadow VirtQueue
+ * @svq_kick_fd The guest to SVQ kick fd
+ * @check_old   Check old file descriptor for pending notifications
  */
-void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
+static void vhost_svq_set_svq_kick_fd_internal(VhostShadowVirtqueue *svq,
+                                               int svq_kick_fd,
+                                               bool check_old)
 {
     EventNotifier tmp;
 
-    event_notifier_set_handler(&svq->svq_kick, NULL);
-    event_notifier_init_fd(&tmp, event_notifier_get_fd(&svq->svq_kick));
+    if (check_old) {
+        event_notifier_set_handler(&svq->svq_kick, NULL);
+        event_notifier_init_fd(&tmp, event_notifier_get_fd(&svq->svq_kick));
+    }
 
     /*
      * event_notifier_set_handler already checks for guest's notifications if
@@ -59,12 +77,57 @@ void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
      * need to explicitely check for them.
      */
     event_notifier_init_fd(&svq->svq_kick, svq_kick_fd);
+    event_notifier_set_handler(&svq->svq_kick, vhost_handle_guest_kick);
 
-    if (event_notifier_test_and_clear(&tmp)) {
+    /*
+     * !check_old means that we are starting SVQ, taking the descriptor from
+     * vhost-vdpa device. This means that we can't trust old file descriptor
+     * pending notifications, since they could have been swallowed by kernel
+     * vhost or paused device. So let it enabled, and qemu event loop will call
+     * us to handle guest avail ring when SVQ is ready.
+     */
+    if (!check_old || event_notifier_test_and_clear(&tmp)) {
         event_notifier_set(&svq->hdev_kick);
     }
 }
 
+/**
+ * Set a new file descriptor for the guest to kick SVQ and notify for avail
+ *
+ * @svq          The svq
+ * @svq_kick_fd  The svq kick fd
+ *
+ * Note that SVQ will never close the old file descriptor.
+ */
+void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
+{
+    vhost_svq_set_svq_kick_fd_internal(svq, svq_kick_fd, true);
+}
+
+/*
+ * Start shadow virtqueue operation.
+ * @dev vhost device
+ * @hidx vhost virtqueue index
+ * @svq Shadow Virtqueue
+ */
+void vhost_svq_start(struct vhost_dev *dev, unsigned idx,
+                     VhostShadowVirtqueue *svq, int svq_kick_fd)
+{
+    vhost_svq_set_svq_kick_fd_internal(svq, svq_kick_fd, false);
+}
+
+/*
+ * Stop shadow virtqueue operation.
+ * @dev vhost device
+ * @idx vhost queue index
+ * @svq Shadow Virtqueue
+ */
+void vhost_svq_stop(struct vhost_dev *dev, unsigned idx,
+                    VhostShadowVirtqueue *svq)
+{
+    event_notifier_set_handler(&svq->svq_kick, NULL);
+}
+
 /*
  * Creates vhost shadow virtqueue, and instruct vhost device to use the shadow
  * methods and file descriptors.
-- 
2.27.0