From: Coiby Xu <coiby.xu@gmail.com>
To: qemu-devel@nongnu.org
Cc: kwolf@redhat.com, bharatlkmlkvm@gmail.com,
Coiby Xu <coiby.xu@gmail.com>,
stefanha@redhat.com
Subject: [PATCH v6 2/4] generic vhost user server
Date: Sun, 31 May 2020 01:14:39 +0800 [thread overview]
Message-ID: <20200530171441.660814-3-coiby.xu@gmail.com> (raw)
In-Reply-To: <20200530171441.660814-1-coiby.xu@gmail.com>
Sharing QEMU devices via vhost-user protocol.
Only one vhost-user client can connect to the server one time.
Signed-off-by: Coiby Xu <coiby.xu@gmail.com>
---
util/Makefile.objs | 1 +
util/vhost-user-server.c | 404 +++++++++++++++++++++++++++++++++++++++
util/vhost-user-server.h | 59 ++++++
3 files changed, 464 insertions(+)
create mode 100644 util/vhost-user-server.c
create mode 100644 util/vhost-user-server.h
diff --git a/util/Makefile.objs b/util/Makefile.objs
index fe339c2636..f54a6c80ec 100644
--- a/util/Makefile.objs
+++ b/util/Makefile.objs
@@ -40,6 +40,7 @@ util-obj-y += readline.o
util-obj-y += rcu.o
util-obj-$(CONFIG_MEMBARRIER) += sys_membarrier.o
util-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
+util-obj-$(CONFIG_LINUX) += vhost-user-server.o
util-obj-y += qemu-coroutine-sleep.o
util-obj-y += qemu-co-shared-resource.o
util-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
diff --git a/util/vhost-user-server.c b/util/vhost-user-server.c
new file mode 100644
index 0000000000..414d0d4e47
--- /dev/null
+++ b/util/vhost-user-server.c
@@ -0,0 +1,404 @@
+/*
+ * Sharing QEMU devices via vhost-user protocol
+ *
+ * Author: Coiby Xu <coiby.xu@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include <sys/eventfd.h>
+#include "qemu/main-loop.h"
+#include "vhost-user-server.h"
+
+static void vmsg_close_fds(VhostUserMsg *vmsg)
+{
+ int i;
+ for (i = 0; i < vmsg->fd_num; i++) {
+ close(vmsg->fds[i]);
+ }
+}
+
+static void vmsg_unblock_fds(VhostUserMsg *vmsg)
+{
+ int i;
+ for (i = 0; i < vmsg->fd_num; i++) {
+ qemu_set_nonblock(vmsg->fds[i]);
+ }
+}
+
+static void vu_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+ gpointer opaque);
+
+static void close_client(VuServer *server)
+{
+ vu_deinit(&server->vu_dev);
+ server->sioc = NULL;
+ object_unref(OBJECT(server->ioc));
+
+ server->sioc_slave = NULL;
+ object_unref(OBJECT(server->ioc_slave));
+ /*
+ * Set the callback function for network listener so another
+ * vhost-user client can connect to this server
+ */
+ qio_net_listener_set_client_func(server->listener,
+ vu_accept,
+ server,
+ NULL);
+}
+
+static void panic_cb(VuDev *vu_dev, const char *buf)
+{
+ VuServer *server = container_of(vu_dev, VuServer, vu_dev);
+
+ if (buf) {
+ error_report("vu_panic: %s", buf);
+ }
+
+ if (server->sioc) {
+ close_client(server);
+ }
+
+ if (server->device_panic_notifier) {
+ server->device_panic_notifier(server);
+ }
+}
+
+static QIOChannel *slave_io_channel(VuServer *server, int fd,
+ Error **local_err)
+{
+ if (server->sioc_slave) {
+ if (fd == server->sioc_slave->fd) {
+ return server->ioc_slave;
+ }
+ } else {
+ server->sioc_slave = qio_channel_socket_new_fd(fd, local_err);
+ if (!*local_err) {
+ server->ioc_slave = QIO_CHANNEL(server->sioc_slave);
+ return server->ioc_slave;
+ }
+ }
+
+ return NULL;
+}
+
+static bool coroutine_fn
+vu_message_read(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)
+{
+ struct iovec iov = {
+ .iov_base = (char *)vmsg,
+ .iov_len = VHOST_USER_HDR_SIZE,
+ };
+ int rc, read_bytes = 0;
+ Error *local_err = NULL;
+ /*
+ * Store fds/nfds returned from qio_channel_readv_full into
+ * temporary variables.
+ *
+ * VhostUserMsg is a packed structure, gcc will complain about passing
+ * pointer to a packed structure member if we pass &VhostUserMsg.fd_num
+ * and &VhostUserMsg.fds directly when calling qio_channel_readv_full,
+ * thus two temporary variables nfds and fds are used here.
+ */
+ size_t nfds = 0, nfds_t = 0;
+ int *fds = NULL, *fds_t = NULL;
+ VuServer *server = container_of(vu_dev, VuServer, vu_dev);
+ QIOChannel *ioc = NULL;
+
+ if (conn_fd == server->sioc->fd) {
+ ioc = server->ioc;
+ } else {
+ /* Slave communication will also use this function to read msg */
+ ioc = slave_io_channel(server, conn_fd, &local_err);
+ }
+
+ if (!ioc) {
+ error_report_err(local_err);
+ goto fail;
+ }
+
+ assert(qemu_in_coroutine());
+ do {
+ /*
+ * qio_channel_readv_full may have short reads, keeping calling it
+ * until getting VHOST_USER_HDR_SIZE or 0 bytes in total
+ */
+ rc = qio_channel_readv_full(ioc, &iov, 1, &fds_t, &nfds_t, &local_err);
+ if (rc < 0) {
+ if (rc == QIO_CHANNEL_ERR_BLOCK) {
+ qio_channel_yield(ioc, G_IO_IN);
+ continue;
+ } else {
+ error_report_err(local_err);
+ return false;
+ }
+ }
+ read_bytes += rc;
+ if (nfds_t > 0) {
+ fds = g_renew(int, fds_t, nfds + nfds_t);
+ memcpy(fds + nfds, fds_t, nfds_t);
+ nfds += nfds_t;
+ if (nfds > VHOST_MEMORY_MAX_NREGIONS) {
+ error_report("A maximum of %d fds are allowed, "
+ "however got %lu fds now",
+ VHOST_MEMORY_MAX_NREGIONS, nfds);
+ goto fail;
+ }
+ }
+ if (read_bytes == VHOST_USER_HDR_SIZE || rc == 0) {
+ break;
+ }
+ iov.iov_base = (char *)vmsg + read_bytes;
+ iov.iov_len = VHOST_USER_HDR_SIZE - read_bytes;
+ } while (true);
+
+ vmsg->fd_num = nfds;
+ if (nfds > 0) {
+ memcpy(vmsg->fds, fds, nfds * sizeof(int));
+ }
+ g_free(fds);
+ /* qio_channel_readv_full will make socket fds blocking, unblock them */
+ vmsg_unblock_fds(vmsg);
+ if (vmsg->size > sizeof(vmsg->payload)) {
+ error_report("Error: too big message request: %d, "
+ "size: vmsg->size: %u, "
+ "while sizeof(vmsg->payload) = %zu",
+ vmsg->request, vmsg->size, sizeof(vmsg->payload));
+ goto fail;
+ }
+
+ struct iovec iov_payload = {
+ .iov_base = (char *)&vmsg->payload,
+ .iov_len = vmsg->size,
+ };
+ if (vmsg->size) {
+ rc = qio_channel_readv_all_eof(ioc, &iov_payload, 1, &local_err);
+ if (rc == -1) {
+ error_report_err(local_err);
+ goto fail;
+ }
+ }
+
+ return true;
+
+fail:
+ vmsg_close_fds(vmsg);
+
+ return false;
+}
+
+
+static void vu_client_start(VuServer *server);
+static coroutine_fn void vu_client_trip(void *opaque)
+{
+ VuServer *server = opaque;
+
+ while (!server->aio_context_changed && server->sioc) {
+ vu_dispatch(&server->vu_dev);
+ }
+
+ if (server->aio_context_changed && server->sioc) {
+ server->aio_context_changed = false;
+ vu_client_start(server);
+ }
+}
+
+static void vu_client_start(VuServer *server)
+{
+ server->co_trip = qemu_coroutine_create(vu_client_trip, server);
+ aio_co_enter(server->ctx, server->co_trip);
+}
+
+/*
+ * a wrapper for vu_kick_cb
+ *
+ * since aio_dispatch can only pass one user data pointer to the
+ * callback function, pack VuDev and pvt into a struct. Then unpack it
+ * and pass them to vu_kick_cb
+ */
+static void kick_handler(void *opaque)
+{
+ KickInfo *kick_info = opaque;
+ kick_info->cb(kick_info->vu_dev, 0, (void *) kick_info->index);
+}
+
+
+static void
+set_watch(VuDev *vu_dev, int fd, int vu_evt,
+ vu_watch_cb cb, void *pvt)
+{
+
+ VuServer *server = container_of(vu_dev, VuServer, vu_dev);
+ g_assert(vu_dev);
+ g_assert(fd >= 0);
+ long index = (intptr_t) pvt;
+ g_assert(cb);
+ KickInfo *kick_info = &server->kick_info[index];
+ if (!kick_info->cb) {
+ kick_info->fd = fd;
+ kick_info->cb = cb;
+ qemu_set_nonblock(fd);
+ aio_set_fd_handler(server->ioc->ctx, fd, false, kick_handler,
+ NULL, NULL, kick_info);
+ kick_info->vu_dev = vu_dev;
+ }
+}
+
+
+static void remove_watch(VuDev *vu_dev, int fd)
+{
+ VuServer *server;
+ int i;
+ int index = -1;
+ g_assert(vu_dev);
+ g_assert(fd >= 0);
+
+ server = container_of(vu_dev, VuServer, vu_dev);
+ for (i = 0; i < vu_dev->max_queues; i++) {
+ if (server->kick_info[i].fd == fd) {
+ index = i;
+ break;
+ }
+ }
+
+ if (index == -1) {
+ return;
+ }
+ server->kick_info[i].cb = NULL;
+ aio_set_fd_handler(server->ioc->ctx, fd, false, NULL, NULL, NULL, NULL);
+}
+
+
+static void vu_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+ gpointer opaque)
+{
+ VuServer *server = opaque;
+
+ if (server->sioc) {
+ warn_report("Only one vhost-user client is allowed to "
+ "connect the server one time");
+ return;
+ }
+
+ if (!vu_init(&server->vu_dev, server->max_queues, sioc->fd, panic_cb,
+ vu_message_read, set_watch, remove_watch, server->vu_iface)) {
+ error_report("Failed to initialized libvhost-user");
+ return;
+ }
+
+ /*
+ * Unset the callback function for network listener to make another
+ * vhost-user client keeping waiting until this client disconnects
+ */
+ qio_net_listener_set_client_func(server->listener,
+ NULL,
+ NULL,
+ NULL);
+ server->sioc = sioc;
+ server->kick_info = g_new0(KickInfo, server->max_queues);
+ /*
+ * Increase the object reference, so cioc will not freed by
+ * qio_net_listener_channel_func which will call object_unref(OBJECT(sioc))
+ */
+ object_ref(OBJECT(server->sioc));
+ qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");
+ server->ioc = QIO_CHANNEL(sioc);
+ object_ref(OBJECT(server->ioc));
+ object_ref(OBJECT(sioc));
+ qio_channel_attach_aio_context(server->ioc, server->ctx);
+ qio_channel_set_blocking(QIO_CHANNEL(server->sioc), false, NULL);
+ vu_client_start(server);
+}
+
+
+void vhost_user_server_stop(VuServer *server)
+{
+ if (!server) {
+ return;
+ }
+
+ if (server->sioc) {
+ close_client(server);
+ }
+
+ if (server->listener) {
+ qio_net_listener_disconnect(server->listener);
+ object_unref(OBJECT(server->listener));
+ }
+}
+
+static void detach_context(VuServer *server)
+{
+ int i;
+ AioContext *ctx = server->ioc->ctx;
+ qio_channel_detach_aio_context(server->ioc);
+ for (i = 0; i < server->vu_dev.max_queues; i++) {
+ if (server->kick_info[i].cb) {
+ aio_set_fd_handler(ctx, server->kick_info[i].fd, false, NULL,
+ NULL, NULL, NULL);
+ }
+ }
+}
+
+static void attach_context(VuServer *server, AioContext *ctx)
+{
+ int i;
+ qio_channel_attach_aio_context(server->ioc, ctx);
+ server->aio_context_changed = true;
+ if (server->co_trip) {
+ aio_co_schedule(ctx, server->co_trip);
+ }
+ for (i = 0; i < server->vu_dev.max_queues; i++) {
+ if (server->kick_info[i].cb) {
+ aio_set_fd_handler(ctx, server->kick_info[i].fd, false,
+ kick_handler, NULL, NULL,
+ &server->kick_info[i]);
+ }
+ }
+}
+
+void vhost_user_server_set_aio_context(AioContext *ctx, VuServer *server)
+{
+ server->ctx = ctx ? ctx : qemu_get_aio_context();
+ if (!server->sioc) {
+ return;
+ }
+ if (ctx) {
+ attach_context(server, ctx);
+ } else {
+ detach_context(server);
+ }
+}
+
+
+bool vhost_user_server_start(uint16_t max_queues,
+ SocketAddress *socket_addr,
+ AioContext *ctx,
+ VuServer *server,
+ void *device_panic_notifier,
+ const VuDevIface *vu_iface,
+ Error **errp)
+{
+ server->listener = qio_net_listener_new();
+ if (qio_net_listener_open_sync(server->listener, socket_addr, 1,
+ errp) < 0) {
+ goto error;
+ }
+
+ qio_net_listener_set_name(server->listener, "vhost-user-backend-listener");
+
+ server->vu_iface = vu_iface;
+ server->max_queues = max_queues;
+ server->ctx = ctx;
+ server->device_panic_notifier = device_panic_notifier;
+ qio_net_listener_set_client_func(server->listener,
+ vu_accept,
+ server,
+ NULL);
+
+ return true;
+error:
+ g_free(server);
+ return false;
+}
diff --git a/util/vhost-user-server.h b/util/vhost-user-server.h
new file mode 100644
index 0000000000..4315556b66
--- /dev/null
+++ b/util/vhost-user-server.h
@@ -0,0 +1,59 @@
+/*
+ * Sharing QEMU devices via vhost-user protocol
+ *
+ * Author: Coiby Xu <coiby.xu@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+
+#ifndef VHOST_USER_SERVER_H
+#define VHOST_USER_SERVER_H
+
+#include "contrib/libvhost-user/libvhost-user.h"
+#include "io/channel-socket.h"
+#include "io/channel-file.h"
+#include "io/net-listener.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+#include "standard-headers/linux/virtio_blk.h"
+
+typedef struct KickInfo {
+ VuDev *vu_dev;
+ int fd; /*kick fd*/
+ long index; /*queue index*/
+ vu_watch_cb cb;
+} KickInfo;
+
+typedef struct VuServer {
+ QIONetListener *listener;
+ AioContext *ctx;
+ void (*device_panic_notifier)(struct VuServer *server) ;
+ int max_queues;
+ const VuDevIface *vu_iface;
+ VuDev vu_dev;
+ QIOChannel *ioc; /* The I/O channel with the client */
+ QIOChannelSocket *sioc; /* The underlying data channel with the client */
+ /* IOChannel for fd provided via VHOST_USER_SET_SLAVE_REQ_FD */
+ QIOChannel *ioc_slave;
+ QIOChannelSocket *sioc_slave;
+ Coroutine *co_trip; /* coroutine for processing VhostUserMsg */
+ KickInfo *kick_info; /* an array with the length of the queue number */
+ /* restart coroutine co_trip if AIOContext is changed */
+ bool aio_context_changed;
+} VuServer;
+
+
+bool vhost_user_server_start(uint16_t max_queues,
+ SocketAddress *unix_socket,
+ AioContext *ctx,
+ VuServer *server,
+ void *device_panic_notifier,
+ const VuDevIface *vu_iface,
+ Error **errp);
+
+void vhost_user_server_stop(VuServer *server);
+
+void vhost_user_server_set_aio_context(AioContext *ctx, VuServer *server);
+
+#endif /* VHOST_USER_SERVER_H */
--
2.26.2
next prev parent reply other threads:[~2020-05-30 17:19 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-30 17:14 [PATCH v6 0/4] vhost-user block device backend implementation Coiby Xu
2020-05-30 17:14 ` [PATCH v6 1/4] Allow vu_message_read to be replaced Coiby Xu
2020-05-30 17:14 ` Coiby Xu [this message]
2020-05-30 17:14 ` [PATCH v6 3/4] vhost-user block device backend server Coiby Xu
2020-05-30 17:14 ` [PATCH v6 4/4] new qTest case to test the vhost-user-blk-server Coiby Xu
2020-06-04 13:42 ` Thomas Huth
2020-06-04 13:49 ` Thomas Huth
2020-06-04 17:22 ` Coiby Xu
2020-05-30 18:29 ` [PATCH v6 0/4] vhost-user block device backend implementation no-reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200530171441.660814-3-coiby.xu@gmail.com \
--to=coiby.xu@gmail.com \
--cc=bharatlkmlkvm@gmail.com \
--cc=kwolf@redhat.com \
--cc=qemu-devel@nongnu.org \
--cc=stefanha@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.