From mboxrd@z Thu Jan 1 00:00:00 1970 From: zhiyong.yang@intel.com Subject: [PATCH v2 4/5] net/virtio-user: add support for server mode Date: Thu, 15 Mar 2018 17:45:13 +0800 Message-ID: <20180315094514.50417-5-zhiyong.yang@intel.com> References: <20180214145330.4679-2-zhiyong.yang@intel.com> <20180315094514.50417-1-zhiyong.yang@intel.com> Cc: maxime.coquelin@redhat.com, jianfeng.tan@intel.com, dong1.wang@intel.com, zhihong.wang@intel.com, thomas@monjalon.net, Zhiyong Yang To: dev@dpdk.org Return-path: Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by dpdk.org (Postfix) with ESMTP id 7FD5971B4 for ; Thu, 15 Mar 2018 10:45:25 +0100 (CET) In-Reply-To: <20180315094514.50417-1-zhiyong.yang@intel.com> List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" virtio-user adds support for server mode in this patch. Virtio-user with server mode creates socket file and then starts to wait for the first connection from vhost user with client mode in blocking mode. Server mode virtio-user supports many times' vhost reconnections with same configurations. Signed-off-by: Zhiyong Yang --- drivers/net/virtio/virtio_user/vhost_user.c | 77 ++++++++++++++++++++-- drivers/net/virtio/virtio_user/virtio_user_dev.c | 44 +++++++++---- drivers/net/virtio/virtio_user/virtio_user_dev.h | 8 +++ drivers/net/virtio/virtio_user_ethdev.c | 82 ++++++++++++++++++++++-- 4 files changed, 188 insertions(+), 23 deletions(-) diff --git a/drivers/net/virtio/virtio_user/vhost_user.c b/drivers/net/virtio/virtio_user/vhost_user.c index 91c6449bb..cd30f713a 100644 --- a/drivers/net/virtio/virtio_user/vhost_user.c +++ b/drivers/net/virtio/virtio_user/vhost_user.c @@ -378,6 +378,55 @@ vhost_user_sock(struct virtio_user_dev *dev, return 0; } +static void +virtio_user_set_block(int fd, bool enabled) +{ + int f; + + f = fcntl(fd, F_GETFL); + if (enabled) + fcntl(fd, F_SETFL, f & ~O_NONBLOCK); + else + fcntl(fd, F_SETFL, f | O_NONBLOCK); +} + +#define MAX_VIRTIO_USER_BACKLOG 128 +static int +virtio_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un) +{ + int ret; + int fd = dev->listenfd; + int connectfd; + + ret = bind(fd, (struct sockaddr *)un, sizeof(*un)); + if (ret < 0) { + PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again\n", + dev->path, strerror(errno)); + goto err; + } + ret = listen(fd, MAX_VIRTIO_USER_BACKLOG); + if (ret < 0) + goto err; + + virtio_user_set_block(fd, true); + PMD_DRV_LOG(NOTICE, "virtio user server mode is waiting for connection from vhost user."); + while (1) { + connectfd = accept(fd, NULL, NULL); + if (connectfd >= 0) { + dev->connected = true; + break; + } + } + + dev->vhostfd = connectfd; + virtio_user_set_block(connectfd, true); + + return 0; +err: + close(fd); + return -1; +} + /** * Set up environment to talk with a vhost user backend. * @@ -390,6 +439,7 @@ vhost_user_setup(struct virtio_user_dev *dev) { int fd; int flag; + int ret; struct sockaddr_un un; fd = socket(AF_UNIX, SOCK_STREAM, 0); @@ -405,13 +455,30 @@ vhost_user_setup(struct virtio_user_dev *dev) memset(&un, 0, sizeof(un)); un.sun_family = AF_UNIX; snprintf(un.sun_path, sizeof(un.sun_path), "%s", dev->path); - if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) { - PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno)); - close(fd); - return -1; + + if (dev->is_server) { + static pthread_t fdset_tid; + + dev->listenfd = fd; + if (fdset_tid == 0) { + ret = pthread_create(&fdset_tid, NULL, + rte_fdset_event_dispatch, + &dev->fdset); + if (ret < 0) + PMD_DRV_LOG(ERR, "failed to create fdset handling thread"); + } + return virtio_user_start_server(dev, &un); + + } else { + dev->vhostfd = fd; + if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) { + PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno)); + close(fd); + return -1; + } + dev->connected = true; } - dev->vhostfd = fd; return 0; } diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c index f90fee9e5..23312344f 100644 --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c @@ -142,6 +142,9 @@ int virtio_user_stop_device(struct virtio_user_dev *dev) { uint32_t i; + if (!dev->connected) + return -1; + for (i = 0; i < dev->max_queue_pairs; ++i) dev->ops->enable_qp(dev, i, 0); @@ -267,21 +270,27 @@ virtio_user_dev_setup(struct virtio_user_dev *dev) dev->vhostfds = NULL; dev->tapfds = NULL; - if (is_vhost_user_by_type(dev->path)) { - dev->ops = &ops_user; + if (dev->is_server) { + dev->ops = &ops_user;/* server mode only supports vhost user */ } else { - dev->ops = &ops_kernel; - - dev->vhostfds = malloc(dev->max_queue_pairs * sizeof(int)); - dev->tapfds = malloc(dev->max_queue_pairs * sizeof(int)); - if (!dev->vhostfds || !dev->tapfds) { - PMD_INIT_LOG(ERR, "Failed to malloc"); - return -1; - } - - for (q = 0; q < dev->max_queue_pairs; ++q) { - dev->vhostfds[q] = -1; - dev->tapfds[q] = -1; + if (is_vhost_user_by_type(dev->path)) { + dev->ops = &ops_user; + } else { + dev->ops = &ops_kernel; + + dev->vhostfds = malloc(dev->max_queue_pairs * + sizeof(int)); + dev->tapfds = malloc(dev->max_queue_pairs * + sizeof(int)); + if (!dev->vhostfds || !dev->tapfds) { + PMD_INIT_LOG(ERR, "Failed to malloc"); + return -1; + } + + for (q = 0; q < dev->max_queue_pairs; ++q) { + dev->vhostfds[q] = -1; + dev->tapfds[q] = -1; + } } } @@ -388,6 +397,10 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev) close(dev->vhostfd); + if (dev->is_server && dev->listenfd >= 0) + close(dev->listenfd); + + dev->connected = false; if (dev->vhostfds) { for (i = 0; i < dev->max_queue_pairs; ++i) close(dev->vhostfds[i]); @@ -396,6 +409,9 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev) } free(dev->ifname); + + if (dev->is_server) + unlink(dev->path); } static uint8_t diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.h b/drivers/net/virtio/virtio_user/virtio_user_dev.h index 64467b4f9..9b73b9531 100644 --- a/drivers/net/virtio/virtio_user/virtio_user_dev.h +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.h @@ -6,6 +6,8 @@ #define _VIRTIO_USER_DEV_H #include +#include +#include #include "../virtio_pci.h" #include "../virtio_ring.h" #include "vhost.h" @@ -13,6 +15,12 @@ struct virtio_user_dev { /* for vhost_user backend */ int vhostfd; + int listenfd; /* listening fd */ + bool connected; /* connection status */ + + /* support for server/clinet mode */ + bool is_server; + struct rte_fdset fdset; /* for vhost_kernel backend */ char *ifname; diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c index 263649006..d36c50c5b 100644 --- a/drivers/net/virtio/virtio_user_ethdev.c +++ b/drivers/net/virtio/virtio_user_ethdev.c @@ -65,8 +65,7 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, r = recv(dev->vhostfd, buf, 128, MSG_PEEK); if (r == 0 || (r < 0 && errno != EAGAIN)) { dev->status &= (~VIRTIO_NET_S_LINK_UP); - PMD_DRV_LOG(ERR, "virtio-user port %u is down", - hw->port_id); + /* Only client mode is available now. Once the * connection is broken, it can never be up * again. Besides, this function could be called @@ -74,9 +73,15 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, * callback cannot be unregistered here, set an * alarm to do it. */ - rte_eal_alarm_set(1, + if (dev->connected) { + dev->connected = false; + PMD_DRV_LOG(ERR, "virtio-user port %u is down", + hw->port_id); + rte_eal_alarm_set(1, virtio_user_delayed_handler, (void *)hw); + hw->started = 0; + } } else { dev->status |= VIRTIO_NET_S_LINK_UP; } @@ -278,12 +283,15 @@ static const char *valid_args[] = { VIRTIO_USER_ARG_QUEUE_SIZE, #define VIRTIO_USER_ARG_INTERFACE_NAME "iface" VIRTIO_USER_ARG_INTERFACE_NAME, +#define VIRTIO_USER_ARG_SERVER_MODE "server" + VIRTIO_USER_ARG_SERVER_MODE, NULL }; #define VIRTIO_USER_DEF_CQ_EN 0 #define VIRTIO_USER_DEF_Q_NUM 1 #define VIRTIO_USER_DEF_Q_SZ 256 +#define VIRTIO_USER_DEF_SERVER_MODE 0 static int get_string_arg(const char *key __rte_unused, @@ -365,6 +373,49 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev) rte_eth_dev_release_port(eth_dev); } +static void +virtio_user_server_reconnection(int fd, void *dat, int *remove __rte_unused) +{ + int ret; + int flag; + int connectfd; + struct virtio_user_dev *dev = dat; + struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id]; + struct virtio_hw *hw = eth_dev->data->dev_private; + + if (dev->connected) + return; + + connectfd = accept(fd, NULL, NULL); + if (connectfd < 0) + return; + + if (dev->vhostfd >= 0) + close(dev->vhostfd); + + dev->vhostfd = connectfd; + flag = fcntl(connectfd, F_GETFD); + fcntl(connectfd, F_SETFL, flag & ~O_NONBLOCK); + + ret = virtio_user_start_device(dev); + if (ret < 0) + return; + + if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { + eth_dev->intr_handle->fd = connectfd; + rte_intr_callback_register(eth_dev->intr_handle, + virtio_interrupt_handler, eth_dev); + + if (rte_intr_enable(eth_dev->intr_handle) < 0) { + PMD_DRV_LOG(ERR, "interrupt enable failed"); + return; + } + } + + hw->started = 1; + dev->connected = true; + PMD_INIT_LOG(NOTICE, "virtio user server reconnection succeeds!"); +} /* Dev initialization routine. Invoked once for each virtio vdev at * EAL init time, see rte_bus_probe(). * Returns 0 on success. @@ -378,11 +429,12 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) uint64_t queues = VIRTIO_USER_DEF_Q_NUM; uint64_t cq = VIRTIO_USER_DEF_CQ_EN; uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ; + uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE; char *path = NULL; char *ifname = NULL; char *mac_addr = NULL; int ret = -1; - + struct virtio_user_dev *vu_dev = NULL; kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args); if (!kvlist) { PMD_INIT_LOG(ERR, "error when parsing param"); @@ -445,6 +497,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) } } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE, + &get_integer_arg, &server_mode) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_SERVER_MODE); + goto end; + } + } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) { if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM, &get_integer_arg, &cq) < 0) { @@ -476,6 +537,11 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) } hw = eth_dev->data->dev_private; + vu_dev = virtio_user_get_dev(hw); + if (server_mode == 1) + vu_dev->is_server = true; + else + vu_dev->is_server = false; if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq, queue_size, mac_addr, &ifname) < 0) { PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); @@ -488,6 +554,14 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) goto end; } + if (vu_dev->is_server) { + ret = rte_fdset_add(&vu_dev->fdset, vu_dev->listenfd, + virtio_user_server_reconnection, NULL, + vu_dev); + if (ret < 0) + goto end; + } + /* previously called by rte_pci_probe() for physical dev */ if (eth_virtio_dev_init(eth_dev) < 0) { PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); -- 2.14.3