Re: [PATCH v3 3/4] net/virtio-user: add support for server mode

["Tan, Jianfeng" <jianfeng.tan@intel.com>]

Hi Zhiyong,

Triggered by the community discussion of thread creation in a library 
(http://dpdk.org/dev/patchwork/patch/36579/), we can think about if 
possible to avoid the pthread creation here.

Just a quick think, it could be feasible. The key idea is to properly 
set up the LSC interrupt for a server mode virtio-user.
- If the virtio-user device is just probed, we can setup LSC interrupt 
by registering the listen fd to interrupt thread. Whenever a vhost-user 
connection is coming, we will get a chance to read status, at that time, 
we could accept the connection, and report the status is up.
- And after the connection is setup, we setup LSC interrupt by 
registering the connection fd to interrupt thread; if the connection is 
broken, we also get a change to read status, in which we can switch to 
the above setting.

How do you think?

Thanks,
Jianfeng

On 3/21/2018 11:03 AM, zhiyong.yang@intel.com wrote:
> virtio-user adds support for server mode in this patch.
>
> Virtio-user with server mode creates socket file and then starts to wait
> for the first connection from vhost user with client mode in blocking mode.
>
> Server mode virtio-user supports many times' vhost reconnections with
> the same configurations.
>
> Signed-off-by: Zhiyong Yang <zhiyong.yang@intel.com>
> ---
>   drivers/net/virtio/virtio_user/vhost_user.c      | 96 ++++++++++++++++++++++--
>   drivers/net/virtio/virtio_user/virtio_user_dev.c | 47 ++++++++----
>   drivers/net/virtio/virtio_user/virtio_user_dev.h | 11 +++
>   drivers/net/virtio/virtio_user_ethdev.c          | 83 +++++++++++++++++++-
>   4 files changed, 215 insertions(+), 22 deletions(-)
>
> diff --git a/drivers/net/virtio/virtio_user/vhost_user.c b/drivers/net/virtio/virtio_user/vhost_user.c
> index 91c6449bb..90f4fed31 100644
> --- a/drivers/net/virtio/virtio_user/vhost_user.c
> +++ b/drivers/net/virtio/virtio_user/vhost_user.c
> @@ -378,6 +378,65 @@ vhost_user_sock(struct virtio_user_dev *dev,
>   	return 0;
>   }
>   
> +static void
> +virtio_user_set_block(int fd, bool enabled)
> +{
> +	int f;
> +
> +	f = fcntl(fd, F_GETFL);
> +	if (enabled)
> +		fcntl(fd, F_SETFL, f & ~O_NONBLOCK);
> +	else
> +		fcntl(fd, F_SETFL, f | O_NONBLOCK);
> +}
> +
> +#define MAX_VIRTIO_USER_BACKLOG 128
> +static int
> +virtio_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un)
> +{
> +	int ret;
> +	int fd = dev->listenfd;
> +	int connectfd;
> +
> +	ret = bind(fd, (struct sockaddr *)un, sizeof(*un));
> +	if (ret < 0) {
> +		PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again\n",
> +			    dev->path, strerror(errno));
> +		goto err;
> +	}
> +	ret = listen(fd, MAX_VIRTIO_USER_BACKLOG);
> +	if (ret < 0)
> +		goto err;
> +
> +	virtio_user_set_block(fd, true);
> +	PMD_DRV_LOG(NOTICE, "virtio user server mode is waiting for connection from vhost user.");
> +	while (1) {
> +		connectfd = accept(fd, NULL, NULL);
> +		if (connectfd >= 0) {
> +			dev->connected = true;
> +			break;
> +		}
> +	}
> +
> +	dev->vhostfd = connectfd;
> +	virtio_user_set_block(connectfd, true);
> +
> +	return 0;
> +err:
> +	close(dev->epoll_fd);
> +	close(dev->listenfd);
> +	return -1;
> +}
> +
> +static __attribute__((noreturn)) void *
> +event_dispatch(void *arg)
> +{
> +	struct virtio_user_dev *dev = arg;
> +
> +	while (1)
> +		rte_epoll_wait(dev->epoll_fd, &dev->rte_epoll_ev, 128, -1);
> +}
> +
>   /**
>    * Set up environment to talk with a vhost user backend.
>    *
> @@ -390,6 +449,7 @@ vhost_user_setup(struct virtio_user_dev *dev)
>   {
>   	int fd;
>   	int flag;
> +	int ret;
>   	struct sockaddr_un un;
>   
>   	fd = socket(AF_UNIX, SOCK_STREAM, 0);
> @@ -405,13 +465,39 @@ vhost_user_setup(struct virtio_user_dev *dev)
>   	memset(&un, 0, sizeof(un));
>   	un.sun_family = AF_UNIX;
>   	snprintf(un.sun_path, sizeof(un.sun_path), "%s", dev->path);
> -	if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
> -		PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
> -		close(fd);
> -		return -1;
> +
> +	if (dev->is_server) {
> +		static pthread_t fdset_tid;
> +
> +		dev->listenfd = fd;
> +		dev->epoll_fd = rte_intr_tls_epfd();
> +		if (dev->epoll_fd < 0) {
> +			PMD_DRV_LOG(ERR, "Can't create epoll file descriptor");
> +			return -1;
> +		}
> +		virtio_user_set_block(dev->epoll_fd, true);
> +		if (fdset_tid == 0) {
> +			ret = pthread_create(&fdset_tid, NULL, event_dispatch,
> +					     dev);
> +			if (ret < 0) {
> +				PMD_DRV_LOG(ERR, "failed to create fdset handling thread");
> +				close(fd);
> +				close(dev->epoll_fd);
> +				return -1;
> +			}
> +		}
> +		return virtio_user_start_server(dev, &un);
> +
> +	} else {
> +		dev->vhostfd = fd;
> +		if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
> +			PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
> +			close(fd);
> +			return -1;
> +		}
> +		dev->connected = true;
>   	}
>   
> -	dev->vhostfd = fd;
>   	return 0;
>   }
>   
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> index f90fee9e5..1430b7cbd 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> @@ -142,6 +142,9 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
>   {
>   	uint32_t i;
>   
> +	if (!dev->connected)
> +		return -1;
> +
>   	for (i = 0; i < dev->max_queue_pairs; ++i)
>   		dev->ops->enable_qp(dev, i, 0);
>   
> @@ -267,21 +270,27 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
>   	dev->vhostfds = NULL;
>   	dev->tapfds = NULL;
>   
> -	if (is_vhost_user_by_type(dev->path)) {
> -		dev->ops = &ops_user;
> +	if (dev->is_server) {
> +		dev->ops = &ops_user;/* server mode only supports vhost user */
>   	} else {
> -		dev->ops = &ops_kernel;
> -
> -		dev->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
> -		dev->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
> -		if (!dev->vhostfds || !dev->tapfds) {
> -			PMD_INIT_LOG(ERR, "Failed to malloc");
> -			return -1;
> -		}
> -
> -		for (q = 0; q < dev->max_queue_pairs; ++q) {
> -			dev->vhostfds[q] = -1;
> -			dev->tapfds[q] = -1;
> +		if (is_vhost_user_by_type(dev->path)) {
> +			dev->ops = &ops_user;
> +		} else {
> +			dev->ops = &ops_kernel;
> +
> +			dev->vhostfds = malloc(dev->max_queue_pairs *
> +					       sizeof(int));
> +			dev->tapfds = malloc(dev->max_queue_pairs *
> +					     sizeof(int));
> +			if (!dev->vhostfds || !dev->tapfds) {
> +				PMD_INIT_LOG(ERR, "Failed to malloc");
> +				return -1;
> +			}
> +
> +			for (q = 0; q < dev->max_queue_pairs; ++q) {
> +				dev->vhostfds[q] = -1;
> +				dev->tapfds[q] = -1;
> +			}
>   		}
>   	}
>   
> @@ -388,6 +397,13 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
>   
>   	close(dev->vhostfd);
>   
> +	if (dev->is_server && dev->listenfd >= 0)
> +		close(dev->listenfd);
> +
> +	if (dev->is_server && dev->epoll_fd >= 0)
> +		close(dev->epoll_fd);
> +
> +	dev->connected = false;
>   	if (dev->vhostfds) {
>   		for (i = 0; i < dev->max_queue_pairs; ++i)
>   			close(dev->vhostfds[i]);
> @@ -396,6 +412,9 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
>   	}
>   
>   	free(dev->ifname);
> +
> +	if (dev->is_server)
> +		unlink(dev->path);
>   }
>   
>   static uint8_t
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.h b/drivers/net/virtio/virtio_user/virtio_user_dev.h
> index 64467b4f9..a8e16d172 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.h
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.h
> @@ -6,6 +6,9 @@
>   #define _VIRTIO_USER_DEV_H
>   
>   #include <limits.h>
> +#include <stdbool.h>
> +#include <sys/epoll.h>
> +#include <rte_interrupts.h>
>   #include "../virtio_pci.h"
>   #include "../virtio_ring.h"
>   #include "vhost.h"
> @@ -13,6 +16,14 @@
>   struct virtio_user_dev {
>   	/* for vhost_user backend */
>   	int		vhostfd;
> +	int		listenfd;   /* listening fd  */
> +	bool		connected;  /* connection status */
> +
> +	int		epoll_fd;
> +	struct rte_epoll_event rte_epoll_ev;
> +
> +	/* support for server/clinet mode */
> +	bool		is_server;
>   
>   	/* for vhost_kernel backend */
>   	char		*ifname;
> diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c
> index 263649006..e60542de5 100644
> --- a/drivers/net/virtio/virtio_user_ethdev.c
> +++ b/drivers/net/virtio/virtio_user_ethdev.c
> @@ -65,8 +65,7 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
>   			r = recv(dev->vhostfd, buf, 128, MSG_PEEK);
>   			if (r == 0 || (r < 0 && errno != EAGAIN)) {
>   				dev->status &= (~VIRTIO_NET_S_LINK_UP);
> -				PMD_DRV_LOG(ERR, "virtio-user port %u is down",
> -					    hw->port_id);
> +
>   				/* Only client mode is available now. Once the
>   				 * connection is broken, it can never be up
>   				 * again. Besides, this function could be called
> @@ -74,9 +73,15 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
>   				 * callback cannot be unregistered here, set an
>   				 * alarm to do it.
>   				 */
> -				rte_eal_alarm_set(1,
> +				if (dev->connected) {
> +					dev->connected = false;
> +					PMD_DRV_LOG(ERR, "virtio-user port %u is down",
> +						    hw->port_id);
> +					rte_eal_alarm_set(1,
>   						  virtio_user_delayed_handler,
>   						  (void *)hw);
> +					hw->started = 0;
> +				}
>   			} else {
>   				dev->status |= VIRTIO_NET_S_LINK_UP;
>   			}
> @@ -278,12 +283,15 @@ static const char *valid_args[] = {
>   	VIRTIO_USER_ARG_QUEUE_SIZE,
>   #define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
>   	VIRTIO_USER_ARG_INTERFACE_NAME,
> +#define VIRTIO_USER_ARG_SERVER_MODE "server"
> +	VIRTIO_USER_ARG_SERVER_MODE,
>   	NULL
>   };
>   
>   #define VIRTIO_USER_DEF_CQ_EN	0
>   #define VIRTIO_USER_DEF_Q_NUM	1
>   #define VIRTIO_USER_DEF_Q_SZ	256
> +#define VIRTIO_USER_DEF_SERVER_MODE	0
>   
>   static int
>   get_string_arg(const char *key __rte_unused,
> @@ -365,6 +373,49 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
>   	rte_eth_dev_release_port(eth_dev);
>   }
>   
> +static void
> +virtio_user_server_reconnection(int fd, void *dat)
> +{
> +	int ret;
> +	int flag;
> +	int connectfd;
> +	struct virtio_user_dev *dev = dat;
> +	struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
> +	struct virtio_hw *hw = eth_dev->data->dev_private;
> +
> +	if (dev->connected)
> +		return;
> +
> +	connectfd = accept(fd, NULL, NULL);
> +	if (connectfd < 0)
> +		return;
> +
> +	if (dev->vhostfd >= 0)
> +		close(dev->vhostfd);
> +
> +	dev->vhostfd = connectfd;
> +	flag = fcntl(connectfd, F_GETFD);
> +	fcntl(connectfd, F_SETFL, flag & ~O_NONBLOCK);
> +
> +	ret = virtio_user_start_device(dev);
> +	if (ret < 0)
> +		return;
> +
> +	if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
> +		eth_dev->intr_handle->fd = connectfd;
> +		rte_intr_callback_register(eth_dev->intr_handle,
> +					   virtio_interrupt_handler, eth_dev);
> +
> +		if (rte_intr_enable(eth_dev->intr_handle) < 0) {
> +			PMD_DRV_LOG(ERR, "interrupt enable failed");
> +			return;
> +		}
> +	}
> +
> +	hw->started = 1;
> +	dev->connected = true;
> +	PMD_INIT_LOG(NOTICE, "virtio user server reconnection succeeds!");
> +}
>   /* Dev initialization routine. Invoked once for each virtio vdev at
>    * EAL init time, see rte_bus_probe().
>    * Returns 0 on success.
> @@ -378,10 +429,12 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
>   	uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
>   	uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
>   	uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
> +	uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
>   	char *path = NULL;
>   	char *ifname = NULL;
>   	char *mac_addr = NULL;
>   	int ret = -1;
> +	struct virtio_user_dev *vu_dev = NULL;
>   
>   	kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args);
>   	if (!kvlist) {
> @@ -445,6 +498,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
>   		}
>   	}
>   
> +	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
> +		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
> +				       &get_integer_arg, &server_mode) < 0) {
> +			PMD_INIT_LOG(ERR, "error to parse %s",
> +				     VIRTIO_USER_ARG_SERVER_MODE);
> +			goto end;
> +		}
> +	}
> +
>   	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
>   		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
>   				       &get_integer_arg, &cq) < 0) {
> @@ -476,6 +538,11 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
>   		}
>   
>   		hw = eth_dev->data->dev_private;
> +		vu_dev = virtio_user_get_dev(hw);
> +		if (server_mode == 1)
> +			vu_dev->is_server = true;
> +		else
> +			vu_dev->is_server = false;
>   		if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq,
>   				 queue_size, mac_addr, &ifname) < 0) {
>   			PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
> @@ -488,6 +555,16 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
>   			goto end;
>   	}
>   
> +	if (vu_dev->is_server) {
> +		vu_dev->rte_epoll_ev.epdata.event = EPOLLIN | EPOLLET;
> +		vu_dev->rte_epoll_ev.epdata.cb_fun = virtio_user_server_reconnection;
> +		vu_dev->rte_epoll_ev.epdata.cb_arg = vu_dev;
> +		ret = rte_epoll_ctl(vu_dev->epoll_fd, EPOLL_CTL_ADD,
> +				    vu_dev->listenfd, &vu_dev->rte_epoll_ev);
> +		if (ret < 0)
> +			goto end;
> +	}
> +
>   	/* previously called by rte_pci_probe() for physical dev */
>   	if (eth_virtio_dev_init(eth_dev) < 0) {
>   		PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");