[PATCH] vhost: add support for interrupt mode

[PATCH] vhost: add support for interrupt mode

[Junjie Chen]

In some cases we want vhost dequeue work in interrupt mode to
release cpus to others when no data to transmit. So we install
interrupt handler of vhost device and interrupt vectors for each
rx queue when creating new backend according to vhost intrerupt
configuration. Thus, applications could register a epoll event fd
to associate rx queues with interrupt vectors.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
 drivers/net/vhost/rte_eth_vhost.c | 116 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 116 insertions(+)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 4e541e3..f474235 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -552,12 +552,115 @@ update_queuing_status(struct rte_eth_dev *dev)
 }
 
 static int
+eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup", qid);
+		return -1;
+	}
+
+	rte_vhost_get_vhost_vring(vq->vid, qid << 1, &vring);
+	vring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT);
+	rte_wmb();
+
+	return 0;
+}
+
+static int
+eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup", qid);
+		return -1;
+	}
+
+	rte_vhost_get_vhost_vring(vq->vid, qid << 1, &vring);
+	vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
+	rte_wmb();
+
+	return 0;
+}
+
+static void
+eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
+{
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (intr_handle) {
+		if (intr_handle->intr_vec)
+			free(intr_handle->intr_vec);
+		free(intr_handle);
+	}
+
+	dev->intr_handle = NULL;
+}
+
+static int
+eth_vhost_install_intr(struct rte_eth_dev *dev)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int count = 0;
+	int nb_rxq = dev->data->nb_rx_queues;
+	int i;
+	int ret;
+
+	/* uninstall firstly if we are reconnecting */
+	if (dev->intr_handle)
+		eth_vhost_uninstall_intr(dev);
+
+	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
+	if (!dev->intr_handle) {
+		RTE_LOG(ERR, PMD, "fail to allocate intr_handle");
+		return -ENOMEM;
+	}
+	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
+
+	dev->intr_handle->intr_vec =
+		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
+
+	if (!dev->intr_handle->intr_vec) {
+		RTE_LOG(ERR, PMD,
+			"failed to allocate memory for interrupt vector");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < nb_rxq; i++) {
+		vq = dev->data->rx_queues[i];
+		if (!vq)
+			continue;
+		ret = rte_vhost_get_vhost_vring(vq->vid, i << 1, &vring);
+		if (ret < 0)
+			continue;
+		RTE_LOG(INFO, PMD, "install intr vec for rxq%d", i);
+		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+		dev->intr_handle->efds[i] = vring.callfd;
+		count++;
+	}
+
+	dev->intr_handle->nb_efd = count;
+	dev->intr_handle->max_intr = count + 1;
+	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
+
+	return 0;
+}
+
+static int
 new_device(int vid)
 {
 	struct rte_eth_dev *eth_dev;
 	struct internal_list *list;
 	struct pmd_internal *internal;
 	struct vhost_queue *vq;
+	struct rte_eth_conf *dev_conf;
 	unsigned i;
 	char ifname[PATH_MAX];
 #ifdef RTE_LIBRTE_VHOST_NUMA
@@ -609,6 +712,15 @@ new_device(int vid)
 
 	RTE_LOG(INFO, PMD, "New connection established\n");
 
+	dev_conf = &eth_dev->data->dev_conf;
+
+	if (dev_conf->intr_conf.rxq) {
+		if (eth_vhost_install_intr(eth_dev) < 0) {
+			RTE_LOG(INFO, PMD, "Failed to prepare intr handler.");
+			return -1;
+		}
+	}
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 
 	return 0;
@@ -663,6 +775,8 @@ destroy_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Connection closed\n");
 
+	eth_vhost_uninstall_intr(eth_dev);
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 }
 
@@ -1007,6 +1121,8 @@ static const struct eth_dev_ops ops = {
 	.xstats_reset = vhost_dev_xstats_reset,
 	.xstats_get = vhost_dev_xstats_get,
 	.xstats_get_names = vhost_dev_xstats_get_names,
+	.rx_queue_intr_enable = eth_rxq_intr_enable,
+	.rx_queue_intr_disable = eth_rxq_intr_disable,
 };
 
 static struct rte_vdev_driver pmd_vhost_drv;
-- 
2.0.1

[PATCH v2] vhost: add support for interrupt mode

[Junjie Chen]

In some cases we want vhost dequeue work in interrupt mode to
release cpus to others when no data to transmit. So we install
interrupt handler of vhost device and interrupt vectors for each
rx queue when creating new backend according to vhost intrerupt
configuration. Thus, applications could register a epoll event fd
to associate rx queues with interrupt vectors.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
Changes in v2:
- update rx queue index
- fill efd_counter_size for intr handler
- update log

 drivers/net/vhost/rte_eth_vhost.c | 130 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 3aae01c..b2d925e 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -552,12 +552,129 @@ update_queuing_status(struct rte_eth_dev *dev)
 }
 
 static int
+eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
+	vring.used->flags &= (~VRING_USED_F_NO_NOTIFY);
+	rte_wmb();
+
+	return ret;
+}
+
+static int
+eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
+	vring.used->flags |= VRING_USED_F_NO_NOTIFY;
+	rte_wmb();
+
+	return 0;
+}
+
+static void
+eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
+{
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (intr_handle) {
+		if (intr_handle->intr_vec)
+			free(intr_handle->intr_vec);
+		free(intr_handle);
+	}
+
+	dev->intr_handle = NULL;
+}
+
+static int
+eth_vhost_install_intr(struct rte_eth_dev *dev)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int count = 0;
+	int nb_rxq = dev->data->nb_rx_queues;
+	int i;
+	int ret;
+
+	/* uninstall firstly if we are reconnecting */
+	if (dev->intr_handle)
+		eth_vhost_uninstall_intr(dev);
+
+	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
+	if (!dev->intr_handle) {
+		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
+		return -ENOMEM;
+	}
+	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
+
+	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
+
+	dev->intr_handle->intr_vec =
+		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
+
+	if (!dev->intr_handle->intr_vec) {
+		RTE_LOG(ERR, PMD,
+			"Failed to allocate memory for interrupt vector\n");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < nb_rxq; i++) {
+		vq = dev->data->rx_queues[i];
+		if (!vq)
+			continue;
+		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
+		if (ret < 0)
+			continue;
+		RTE_LOG(INFO, PMD, "Install intr vec for rxq%d\n", i);
+		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+		dev->intr_handle->efds[i] = vring.kickfd;
+		count++;
+	}
+
+	dev->intr_handle->nb_efd = count;
+	dev->intr_handle->max_intr = count + 1;
+	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
+
+	return 0;
+}
+
+static int
 new_device(int vid)
 {
 	struct rte_eth_dev *eth_dev;
 	struct internal_list *list;
 	struct pmd_internal *internal;
 	struct vhost_queue *vq;
+	struct rte_eth_conf *dev_conf;
 	unsigned i;
 	char ifname[PATH_MAX];
 #ifdef RTE_LIBRTE_VHOST_NUMA
@@ -609,6 +726,15 @@ new_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Vhost device %d created\n", vid);
 
+	dev_conf = &eth_dev->data->dev_conf;
+
+	if (dev_conf->intr_conf.rxq) {
+		if (eth_vhost_install_intr(eth_dev) < 0) {
+			RTE_LOG(INFO, PMD, "Failed to prepare intr handler.");
+			return -1;
+		}
+	}
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 
 	return 0;
@@ -663,6 +789,8 @@ destroy_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
 
+	eth_vhost_uninstall_intr(eth_dev);
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 }
 
@@ -1007,6 +1135,8 @@ static const struct eth_dev_ops ops = {
 	.xstats_reset = vhost_dev_xstats_reset,
 	.xstats_get = vhost_dev_xstats_get,
 	.xstats_get_names = vhost_dev_xstats_get_names,
+	.rx_queue_intr_enable = eth_rxq_intr_enable,
+	.rx_queue_intr_disable = eth_rxq_intr_disable,
 };
 
 static struct rte_vdev_driver pmd_vhost_drv;
-- 
2.0.1

Re: [PATCH v2] vhost: add support for interrupt mode

[Tan, Jianfeng]

On 3/29/2018 10:37 PM, Junjie Chen wrote:
> In some cases we want vhost dequeue work in interrupt mode to
> release cpus to others when no data to transmit. So we install
> interrupt handler of vhost device and interrupt vectors for each
> rx queue when creating new backend according to vhost intrerupt
> configuration. Thus, applications could register a epoll event fd
> to associate rx queues with interrupt vectors.
>
> Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
> ---
> Changes in v2:
> - update rx queue index
> - fill efd_counter_size for intr handler
> - update log
>
>   drivers/net/vhost/rte_eth_vhost.c | 130 ++++++++++++++++++++++++++++++++++++++
>   1 file changed, 130 insertions(+)
>
> diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
> index 3aae01c..b2d925e 100644
> --- a/drivers/net/vhost/rte_eth_vhost.c
> +++ b/drivers/net/vhost/rte_eth_vhost.c
> @@ -552,12 +552,129 @@ update_queuing_status(struct rte_eth_dev *dev)
>   }
>   
>   static int
> +eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
> +{
> +	struct rte_vhost_vring vring;
> +	struct vhost_queue *vq;
> +	int ret = 0;
> +
> +	vq = dev->data->rx_queues[qid];
> +	if (!vq) {
> +		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
> +		return -1;
> +	}
> +
> +	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
> +	if (ret < 0) {
> +		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
> +		return ret;
> +	}
> +	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
> +	vring.used->flags &= (~VRING_USED_F_NO_NOTIFY);

Instead, you might want to use the API, rte_vhost_enable_guest_notification.

> +	rte_wmb();
> +
> +	return ret;
> +}
> +
> +static int
> +eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
> +{
> +	struct rte_vhost_vring vring;
> +	struct vhost_queue *vq;
> +	int ret = 0;
> +
> +	vq = dev->data->rx_queues[qid];
> +	if (!vq) {
> +		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
> +		return -1;
> +	}
> +
> +	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
> +	if (ret < 0) {
> +		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
> +		return ret;
> +	}
> +	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
> +	vring.used->flags |= VRING_USED_F_NO_NOTIFY;
> +	rte_wmb();

Ditto.

> +
> +	return 0;
> +}
> +
> +static void
> +eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
> +{
> +	struct rte_intr_handle *intr_handle = dev->intr_handle;
> +
> +	if (intr_handle) {
> +		if (intr_handle->intr_vec)
> +			free(intr_handle->intr_vec);
> +		free(intr_handle);
> +	}
> +
> +	dev->intr_handle = NULL;
> +}
> +
> +static int
> +eth_vhost_install_intr(struct rte_eth_dev *dev)
> +{
> +	struct rte_vhost_vring vring;
> +	struct vhost_queue *vq;
> +	int count = 0;
> +	int nb_rxq = dev->data->nb_rx_queues;
> +	int i;
> +	int ret;
> +
> +	/* uninstall firstly if we are reconnecting */
> +	if (dev->intr_handle)
> +		eth_vhost_uninstall_intr(dev);
> +
> +	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
> +	if (!dev->intr_handle) {
> +		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
> +		return -ENOMEM;
> +	}
> +	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
> +
> +	dev->intr_handle->efd_counter_size = sizeof(uint64_t);

I did not read-clean this for virtio-user. But we'd better keep 
consistent, to avoid confusion. If we need read-clean this, we could fix 
virtio-user.

> +
> +	dev->intr_handle->intr_vec =
> +		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
> +
> +	if (!dev->intr_handle->intr_vec) {
> +		RTE_LOG(ERR, PMD,
> +			"Failed to allocate memory for interrupt vector\n");

For completeness, we need to uninstall before return.

> +		return -ENOMEM;
> +	}
> +
> +	for (i = 0; i < nb_rxq; i++) {
> +		vq = dev->data->rx_queues[i];
> +		if (!vq)
> +			continue;

Put such check at the entry of this function?

> +		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
> +		if (ret < 0)

Does that mean we shall take care of this failure?

> +			continue;
> +		RTE_LOG(INFO, PMD, "Install intr vec for rxq%d\n", i);
> +		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
> +		dev->intr_handle->efds[i] = vring.kickfd;

Before use of the kickfd, we need to check the validation?

> +		count++;
> +	}
> +
> +	dev->intr_handle->nb_efd = count;
> +	dev->intr_handle->max_intr = count + 1;
> +	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
> +
> +	return 0;
> +}
> +
> +static int
>   new_device(int vid)
>   {
>   	struct rte_eth_dev *eth_dev;
>   	struct internal_list *list;
>   	struct pmd_internal *internal;
>   	struct vhost_queue *vq;
> +	struct rte_eth_conf *dev_conf;
>   	unsigned i;
>   	char ifname[PATH_MAX];
>   #ifdef RTE_LIBRTE_VHOST_NUMA
> @@ -609,6 +726,15 @@ new_device(int vid)
>   
>   	RTE_LOG(INFO, PMD, "Vhost device %d created\n", vid);
>   
> +	dev_conf = &eth_dev->data->dev_conf;
> +
> +	if (dev_conf->intr_conf.rxq) {
> +		if (eth_vhost_install_intr(eth_dev) < 0) {
> +			RTE_LOG(INFO, PMD, "Failed to prepare intr handler.");
> +			return -1;
> +		}
> +	}
> +
>   	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
>   
>   	return 0;
> @@ -663,6 +789,8 @@ destroy_device(int vid)
>   
>   	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
>   
> +	eth_vhost_uninstall_intr(eth_dev);
> +
>   	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
>   }
>   
> @@ -1007,6 +1135,8 @@ static const struct eth_dev_ops ops = {
>   	.xstats_reset = vhost_dev_xstats_reset,
>   	.xstats_get = vhost_dev_xstats_get,
>   	.xstats_get_names = vhost_dev_xstats_get_names,
> +	.rx_queue_intr_enable = eth_rxq_intr_enable,
> +	.rx_queue_intr_disable = eth_rxq_intr_disable,
>   };
>   
>   static struct rte_vdev_driver pmd_vhost_drv;

Re: [PATCH v2] vhost: add support for interrupt mode

[Chen, Junjie J]

> 
> 
> 
> On 3/29/2018 10:37 PM, Junjie Chen wrote:
> > In some cases we want vhost dequeue work in interrupt mode to release
> > cpus to others when no data to transmit. So we install interrupt
> > handler of vhost device and interrupt vectors for each rx queue when
> > creating new backend according to vhost intrerupt configuration. Thus,
> > applications could register a epoll event fd to associate rx queues
> > with interrupt vectors.
> >
> > Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
> > ---
> > Changes in v2:
> > - update rx queue index
> > - fill efd_counter_size for intr handler
> > - update log
> >
> >   drivers/net/vhost/rte_eth_vhost.c | 130
> ++++++++++++++++++++++++++++++++++++++
> >   1 file changed, 130 insertions(+)
> >
> > diff --git a/drivers/net/vhost/rte_eth_vhost.c
> > b/drivers/net/vhost/rte_eth_vhost.c
> > index 3aae01c..b2d925e 100644
> > --- a/drivers/net/vhost/rte_eth_vhost.c
> > +++ b/drivers/net/vhost/rte_eth_vhost.c
> > @@ -552,12 +552,129 @@ update_queuing_status(struct rte_eth_dev
> *dev)
> >   }
> >
> >   static int
> > +eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid) {
> > +	struct rte_vhost_vring vring;
> > +	struct vhost_queue *vq;
> > +	int ret = 0;
> > +
> > +	vq = dev->data->rx_queues[qid];
> > +	if (!vq) {
> > +		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
> > +		return -1;
> > +	}
> > +
> > +	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
> > +	if (ret < 0) {
> > +		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
> > +		return ret;
> > +	}
> > +	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
> > +	vring.used->flags &= (~VRING_USED_F_NO_NOTIFY);
> 
> Instead, you might want to use the API, rte_vhost_enable_guest_notification.
Will do.
> 
> > +	rte_wmb();
> > +
> > +	return ret;
> > +}
> > +
> > +static int
> > +eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid) {
> > +	struct rte_vhost_vring vring;
> > +	struct vhost_queue *vq;
> > +	int ret = 0;
> > +
> > +	vq = dev->data->rx_queues[qid];
> > +	if (!vq) {
> > +		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
> > +		return -1;
> > +	}
> > +
> > +	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
> > +	if (ret < 0) {
> > +		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
> > +		return ret;
> > +	}
> > +	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
> > +	vring.used->flags |= VRING_USED_F_NO_NOTIFY;
> > +	rte_wmb();
> 
> Ditto.
Will do.
> 
> > +
> > +	return 0;
> > +}
> > +
> > +static void
> > +eth_vhost_uninstall_intr(struct rte_eth_dev *dev) {
> > +	struct rte_intr_handle *intr_handle = dev->intr_handle;
> > +
> > +	if (intr_handle) {
> > +		if (intr_handle->intr_vec)
> > +			free(intr_handle->intr_vec);
> > +		free(intr_handle);
> > +	}
> > +
> > +	dev->intr_handle = NULL;
> > +}
> > +
> > +static int
> > +eth_vhost_install_intr(struct rte_eth_dev *dev) {
> > +	struct rte_vhost_vring vring;
> > +	struct vhost_queue *vq;
> > +	int count = 0;
> > +	int nb_rxq = dev->data->nb_rx_queues;
> > +	int i;
> > +	int ret;
> > +
> > +	/* uninstall firstly if we are reconnecting */
> > +	if (dev->intr_handle)
> > +		eth_vhost_uninstall_intr(dev);
> > +
> > +	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
> > +	if (!dev->intr_handle) {
> > +		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
> > +		return -ENOMEM;
> > +	}
> > +	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
> > +
> > +	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
> 
> I did not read-clean this for virtio-user. But we'd better keep consistent, to
> avoid confusion. If we need read-clean this, we could fix virtio-user.
So, do you want to change this in this patch? Or in other separated one?
> 
> > +
> > +	dev->intr_handle->intr_vec =
> > +		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
> > +
> > +	if (!dev->intr_handle->intr_vec) {
> > +		RTE_LOG(ERR, PMD,
> > +			"Failed to allocate memory for interrupt vector\n");
> 
> For completeness, we need to uninstall before return.
Will do.
> 
> > +		return -ENOMEM;
> > +	}
> > +
> > +	for (i = 0; i < nb_rxq; i++) {
> > +		vq = dev->data->rx_queues[i];
> > +		if (!vq)
> > +			continue;
> 
> Put such check at the entry of this function?
> 
> > +		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
> > +		if (ret < 0)
> 
> Does that mean we shall take care of this failure?

For above two checks, I had a fix to check overall queue setup in patch: https://dpdk.org/dev/patchwork/patch/36766/ recently.
So I would like to put install_intr function into queue_setup helper function in patch 36766, and for internal vq, vring, and kickfd check of install_intr, I'd like to skip setup if check fail. How do you think? 

> 
> > +			continue;
> > +		RTE_LOG(INFO, PMD, "Install intr vec for rxq%d\n", i);
> > +		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
> > +		dev->intr_handle->efds[i] = vring.kickfd;
> 
> Before use of the kickfd, we need to check the validation?
Will do.
> 
> > +		count++;
> > +	}
> > +
> > +	dev->intr_handle->nb_efd = count;
> > +	dev->intr_handle->max_intr = count + 1;
> > +	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
> > +
> > +	return 0;
> > +}
> > +
> > +static int
> >   new_device(int vid)
> >   {
> >   	struct rte_eth_dev *eth_dev;
> >   	struct internal_list *list;
> >   	struct pmd_internal *internal;
> >   	struct vhost_queue *vq;
> > +	struct rte_eth_conf *dev_conf;
> >   	unsigned i;
> >   	char ifname[PATH_MAX];
> >   #ifdef RTE_LIBRTE_VHOST_NUMA
> > @@ -609,6 +726,15 @@ new_device(int vid)
> >
> >   	RTE_LOG(INFO, PMD, "Vhost device %d created\n", vid);
> >
> > +	dev_conf = &eth_dev->data->dev_conf;
> > +
> > +	if (dev_conf->intr_conf.rxq) {
> > +		if (eth_vhost_install_intr(eth_dev) < 0) {
> > +			RTE_LOG(INFO, PMD, "Failed to prepare intr handler.");
> > +			return -1;
> > +		}
> > +	}
> > +
> >   	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC,
> > NULL);
> >
> >   	return 0;
> > @@ -663,6 +789,8 @@ destroy_device(int vid)
> >
> >   	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
> >
> > +	eth_vhost_uninstall_intr(eth_dev);
> > +
> >   	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC,
> NULL);
> >   }
> >
> > @@ -1007,6 +1135,8 @@ static const struct eth_dev_ops ops = {
> >   	.xstats_reset = vhost_dev_xstats_reset,
> >   	.xstats_get = vhost_dev_xstats_get,
> >   	.xstats_get_names = vhost_dev_xstats_get_names,
> > +	.rx_queue_intr_enable = eth_rxq_intr_enable,
> > +	.rx_queue_intr_disable = eth_rxq_intr_disable,
> >   };
> >
> >   static struct rte_vdev_driver pmd_vhost_drv;

[PATCH v3] vhost: add support for interrupt mode

[Junjie Chen]

In some cases we want vhost dequeue work in interrupt mode to
release cpus to others when no data to transmit. So we install
interrupt handler of vhost device and interrupt vectors for each
rx queue when creating new backend according to vhost intrerupt
configuration. Thus, applications could register a epoll event fd
to associate rx queues with interrupt vectors.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
Changes in v3:
- handle failure in the middle of intr setup.
- use vhost API to enable interrupt.
- rebase to check rxq existence.
- update vhost API to support guest notification.
Changes in v2:
- update rx queue index.
- fill efd_counter_size for intr handler.
- update log.

 drivers/net/vhost/rte_eth_vhost.c | 201 ++++++++++++++++++++++++++++++--------
 lib/librte_vhost/vhost.c          |  13 ++-
 2 files changed, 167 insertions(+), 47 deletions(-)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 11b6076..c14caae 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -1,35 +1,7 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright (c) 2016 IGEL Co., Ltd.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of IGEL Co.,Ltd. nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2016 Intel Corporation
  */
+
 #include <unistd.h>
 #include <pthread.h>
 #include <stdbool.h>
@@ -554,28 +526,174 @@ update_queuing_status(struct rte_eth_dev *dev)
 	}
 }
 
+static int
+eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1);
+	rte_wmb();
+
+	return ret;
+}
+
+static int
+eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
+	rte_wmb();
+
+	return 0;
+}
+
 static void
+eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
+{
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (intr_handle) {
+		if (intr_handle->intr_vec)
+			free(intr_handle->intr_vec);
+		free(intr_handle);
+	}
+
+	dev->intr_handle = NULL;
+}
+
+static int
+eth_vhost_install_intr(struct rte_eth_dev *dev)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int count = 0;
+	int nb_rxq = dev->data->nb_rx_queues;
+	int i;
+	int ret;
+
+	/* uninstall firstly if we are reconnecting */
+	if (dev->intr_handle)
+		eth_vhost_uninstall_intr(dev);
+
+	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
+	if (!dev->intr_handle) {
+		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
+		return -ENOMEM;
+	}
+	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
+
+	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
+
+	dev->intr_handle->intr_vec =
+		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
+
+	if (!dev->intr_handle->intr_vec) {
+		RTE_LOG(ERR, PMD,
+			"Failed to allocate memory for interrupt vector\n");
+		free(dev->intr_handle);
+		return -ENOMEM;
+	}
+
+	RTE_LOG(INFO, PMD, "Prepare intr vec\n");
+	for (i = 0; i < nb_rxq; i++) {
+		vq = dev->data->rx_queues[i];
+		if (!vq) {
+			RTE_LOG(INFO, PMD, "rxq-%d not setup yet, skip!\n", i);
+			continue;
+		}
+
+		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
+		if (ret < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to get rxq-%d's vring, skip!\n", i);
+			continue;
+		}
+
+		if (vring.kickfd < 0) {
+			RTE_LOG(INFO, PMD,
+				"rxq-%d's kickfd is invalid, skip!\n", i);
+			continue;
+		}
+		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+		dev->intr_handle->efds[i] = vring.kickfd;
+		count++;
+		RTE_LOG(INFO, PMD, "Installed intr vec for rxq-%d\n", i);
+	}
+
+	dev->intr_handle->nb_efd = count;
+	dev->intr_handle->max_intr = count + 1;
+	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
+
+	return 0;
+}
+
+static int
 queue_setup(struct rte_eth_dev *eth_dev, struct pmd_internal *internal)
 {
 	struct vhost_queue *vq;
+	struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
+	int vid = internal->vid;
 	int i;
 
 	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 		vq = eth_dev->data->rx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
+
 	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 		vq = eth_dev->data->tx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
+
+	for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
+		rte_vhost_enable_guest_notification(vid, i, 0);
+
+	if (dev_conf->intr_conf.rxq) {
+		if (eth_vhost_install_intr(eth_dev) < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to install interrupt handler.");
+				return -1;
+		}
+	}
+
+	return 0;
 }
 
 static int
@@ -584,7 +702,6 @@ new_device(int vid)
 	struct rte_eth_dev *eth_dev;
 	struct internal_list *list;
 	struct pmd_internal *internal;
-	unsigned i;
 	char ifname[PATH_MAX];
 #ifdef RTE_LIBRTE_VHOST_NUMA
 	int newnode;
@@ -608,16 +725,13 @@ new_device(int vid)
 
 	internal->vid = vid;
 	if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) {
-		queue_setup(eth_dev, internal);
-		rte_atomic32_set(&internal->dev_attached, 1);
+		if (!queue_setup(eth_dev, internal))
+			rte_atomic32_set(&internal->dev_attached, 1);
 	} else {
 		RTE_LOG(INFO, PMD, "RX/TX queues have not setup yet\n");
 		rte_atomic32_set(&internal->dev_attached, 0);
 	}
 
-	for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
-		rte_vhost_enable_guest_notification(vid, i, 0);
-
 	rte_vhost_get_mtu(vid, &eth_dev->data->mtu);
 
 	eth_dev->data->dev_link.link_status = ETH_LINK_UP;
@@ -681,6 +795,8 @@ destroy_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
 
+	eth_vhost_uninstall_intr(eth_dev);
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 }
 
@@ -793,7 +909,10 @@ eth_dev_start(struct rte_eth_dev *eth_dev)
 	struct pmd_internal *internal = eth_dev->data->dev_private;
 
 	if (unlikely(rte_atomic32_read(&internal->dev_attached) == 0)) {
-		queue_setup(eth_dev, internal);
+		if (!queue_setup(eth_dev, internal)) {
+			RTE_LOG(ERR, PMD, "Failed to set up rx queue\n");
+			return -1;
+		}
 		rte_atomic32_set(&internal->dev_attached, 1);
 	}
 
@@ -1030,6 +1149,8 @@ static const struct eth_dev_ops ops = {
 	.xstats_reset = vhost_dev_xstats_reset,
 	.xstats_get = vhost_dev_xstats_get,
 	.xstats_get_names = vhost_dev_xstats_get_names,
+	.rx_queue_intr_enable = eth_rxq_intr_enable,
+	.rx_queue_intr_disable = eth_rxq_intr_disable,
 };
 
 static struct rte_vdev_driver pmd_vhost_drv;
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index f6f12a0..68e9340 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -545,16 +545,15 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
 {
 	struct virtio_net *dev = get_device(vid);
 
-	if (dev == NULL)
+	if (!dev)
 		return -1;
 
-	if (enable) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"guest notification isn't supported.\n");
-		return -1;
-	}
+	if (enable)
+		dev->virtqueue[queue_id]->used->flags &=
+		    ~VRING_USED_F_NO_NOTIFY;
+	else
+		dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 
-	dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 	return 0;
 }
 
-- 
2.0.1

[PATCH v4] vhost: add support for interrupt mode

[Junjie Chen]

In some cases we want vhost dequeue work in interrupt mode to
release cpus to others when no data to transmit. So we install
interrupt handler of vhost device and interrupt vectors for each
rx queue when creating new backend according to vhost intrerupt
configuration. Thus, applications could register a epoll event fd
to associate rx queues with interrupt vectors.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
Changes in v4:
- revert back license change
Changes in v3:
- handle failure in the middle of intr setup.
- use vhost API to enable interrupt.
- rebase to check rxq existence.
- update vhost API to support guest notification.
Changes in v2:
- update rx queue index.
- fill efd_counter_size for intr handler.
- update log.
 drivers/net/vhost/rte_eth_vhost.c | 167 ++++++++++++++++++++++++++++++++++++--
 lib/librte_vhost/vhost.c          |  13 ++-
 2 files changed, 164 insertions(+), 16 deletions(-)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 11b6076..9a23c12 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -554,28 +554,174 @@ update_queuing_status(struct rte_eth_dev *dev)
 	}
 }
 
+static int
+eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1);
+	rte_wmb();
+
+	return ret;
+}
+
+static int
+eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
+	rte_wmb();
+
+	return 0;
+}
+
 static void
+eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
+{
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (intr_handle) {
+		if (intr_handle->intr_vec)
+			free(intr_handle->intr_vec);
+		free(intr_handle);
+	}
+
+	dev->intr_handle = NULL;
+}
+
+static int
+eth_vhost_install_intr(struct rte_eth_dev *dev)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int count = 0;
+	int nb_rxq = dev->data->nb_rx_queues;
+	int i;
+	int ret;
+
+	/* uninstall firstly if we are reconnecting */
+	if (dev->intr_handle)
+		eth_vhost_uninstall_intr(dev);
+
+	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
+	if (!dev->intr_handle) {
+		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
+		return -ENOMEM;
+	}
+	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
+
+	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
+
+	dev->intr_handle->intr_vec =
+		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
+
+	if (!dev->intr_handle->intr_vec) {
+		RTE_LOG(ERR, PMD,
+			"Failed to allocate memory for interrupt vector\n");
+		free(dev->intr_handle);
+		return -ENOMEM;
+	}
+
+	RTE_LOG(INFO, PMD, "Prepare intr vec\n");
+	for (i = 0; i < nb_rxq; i++) {
+		vq = dev->data->rx_queues[i];
+		if (!vq) {
+			RTE_LOG(INFO, PMD, "rxq-%d not setup yet, skip!\n", i);
+			continue;
+		}
+
+		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
+		if (ret < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to get rxq-%d's vring, skip!\n", i);
+			continue;
+		}
+
+		if (vring.kickfd < 0) {
+			RTE_LOG(INFO, PMD,
+				"rxq-%d's kickfd is invalid, skip!\n", i);
+			continue;
+		}
+		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+		dev->intr_handle->efds[i] = vring.kickfd;
+		count++;
+		RTE_LOG(INFO, PMD, "Installed intr vec for rxq-%d\n", i);
+	}
+
+	dev->intr_handle->nb_efd = count;
+	dev->intr_handle->max_intr = count + 1;
+	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
+
+	return 0;
+}
+
+static int
 queue_setup(struct rte_eth_dev *eth_dev, struct pmd_internal *internal)
 {
 	struct vhost_queue *vq;
+	struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
+	int vid = internal->vid;
 	int i;
 
 	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 		vq = eth_dev->data->rx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
+
 	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 		vq = eth_dev->data->tx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
+
+	for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
+		rte_vhost_enable_guest_notification(vid, i, 0);
+
+	if (dev_conf->intr_conf.rxq) {
+		if (eth_vhost_install_intr(eth_dev) < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to install interrupt handler.");
+				return -1;
+		}
+	}
+
+	return 0;
 }
 
 static int
@@ -584,7 +730,6 @@ new_device(int vid)
 	struct rte_eth_dev *eth_dev;
 	struct internal_list *list;
 	struct pmd_internal *internal;
-	unsigned i;
 	char ifname[PATH_MAX];
 #ifdef RTE_LIBRTE_VHOST_NUMA
 	int newnode;
@@ -608,16 +753,13 @@ new_device(int vid)
 
 	internal->vid = vid;
 	if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) {
-		queue_setup(eth_dev, internal);
-		rte_atomic32_set(&internal->dev_attached, 1);
+		if (!queue_setup(eth_dev, internal))
+			rte_atomic32_set(&internal->dev_attached, 1);
 	} else {
 		RTE_LOG(INFO, PMD, "RX/TX queues have not setup yet\n");
 		rte_atomic32_set(&internal->dev_attached, 0);
 	}
 
-	for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
-		rte_vhost_enable_guest_notification(vid, i, 0);
-
 	rte_vhost_get_mtu(vid, &eth_dev->data->mtu);
 
 	eth_dev->data->dev_link.link_status = ETH_LINK_UP;
@@ -681,6 +823,8 @@ destroy_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
 
+	eth_vhost_uninstall_intr(eth_dev);
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 }
 
@@ -793,7 +937,10 @@ eth_dev_start(struct rte_eth_dev *eth_dev)
 	struct pmd_internal *internal = eth_dev->data->dev_private;
 
 	if (unlikely(rte_atomic32_read(&internal->dev_attached) == 0)) {
-		queue_setup(eth_dev, internal);
+		if (!queue_setup(eth_dev, internal)) {
+			RTE_LOG(ERR, PMD, "Failed to set up rx queue\n");
+			return -1;
+		}
 		rte_atomic32_set(&internal->dev_attached, 1);
 	}
 
@@ -1030,6 +1177,8 @@ static const struct eth_dev_ops ops = {
 	.xstats_reset = vhost_dev_xstats_reset,
 	.xstats_get = vhost_dev_xstats_get,
 	.xstats_get_names = vhost_dev_xstats_get_names,
+	.rx_queue_intr_enable = eth_rxq_intr_enable,
+	.rx_queue_intr_disable = eth_rxq_intr_disable,
 };
 
 static struct rte_vdev_driver pmd_vhost_drv;
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index f6f12a0..68e9340 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -545,16 +545,15 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
 {
 	struct virtio_net *dev = get_device(vid);
 
-	if (dev == NULL)
+	if (!dev)
 		return -1;
 
-	if (enable) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"guest notification isn't supported.\n");
-		return -1;
-	}
+	if (enable)
+		dev->virtqueue[queue_id]->used->flags &=
+		    ~VRING_USED_F_NO_NOTIFY;
+	else
+		dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 
-	dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 	return 0;
 }
 
-- 
2.0.1

[PATCH v5] vhost: add support for interrupt mode

[Junjie Chen]

In some cases we want vhost dequeue work in interrupt mode to
release cpus to others when no data to transmit. So we install
interrupt handler of vhost device and interrupt vectors for each
rx queue when creating new backend according to vhost intrerupt
configuration. Thus, applications could register a epoll event fd
to associate rx queues with interrupt vectors.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
Changes in v5:
- update license to DPDK new license format
- rebase code to master 
Changes in v4:
- revert back license change
Changes in v3:
- handle failure in the middle of intr setup.
- use vhost API to enable interrupt.
- rebase to check rxq existence.
- update vhost API to support guest notification.
Changes in v2:
- update rx queue index.
- fill efd_counter_size for intr handler.
- update log.
 drivers/net/vhost/rte_eth_vhost.c | 267 +++++++++++++++++++++++++++++---------
 lib/librte_vhost/vhost.c          |  13 +-
 2 files changed, 212 insertions(+), 68 deletions(-)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 11b6076..536e089 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -1,35 +1,8 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright (c) 2016 IGEL Co., Ltd.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of IGEL Co.,Ltd. nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016 IGEL Co., Ltd.
+ * Copyright(c) 2016-2018 Intel Corporation
  */
+
 #include <unistd.h>
 #include <pthread.h>
 #include <stdbool.h>
@@ -528,10 +501,13 @@ update_queuing_status(struct rte_eth_dev *dev)
 	unsigned int i;
 	int allow_queuing = 1;
 
-	if (rte_atomic32_read(&internal->dev_attached) == 0)
+	if (!dev->data->rx_queues || !dev->data->tx_queues) {
+		RTE_LOG(ERR, PMD, "RX/TX queues not setup yet\n");
 		return;
+	}
 
-	if (rte_atomic32_read(&internal->started) == 0)
+	if (rte_atomic32_read(&internal->started) == 0 ||
+		rte_atomic32_read(&internal->dev_attached) == 0)
 		allow_queuing = 0;
 
 	/* Wait until rx/tx_pkt_burst stops accessing vhost device */
@@ -554,25 +530,157 @@ update_queuing_status(struct rte_eth_dev *dev)
 	}
 }
 
+static int
+eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1);
+	rte_wmb();
+
+	return ret;
+}
+
+static int
+eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
+	rte_wmb();
+
+	return 0;
+}
+
+static void
+eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
+{
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (intr_handle) {
+		if (intr_handle->intr_vec)
+			free(intr_handle->intr_vec);
+		free(intr_handle);
+	}
+
+	dev->intr_handle = NULL;
+}
+
+static int
+eth_vhost_install_intr(struct rte_eth_dev *dev)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int count = 0;
+	int nb_rxq = dev->data->nb_rx_queues;
+	int i;
+	int ret;
+
+	/* uninstall firstly if we are reconnecting */
+	if (dev->intr_handle)
+		eth_vhost_uninstall_intr(dev);
+
+	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
+	if (!dev->intr_handle) {
+		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
+		return -ENOMEM;
+	}
+	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
+
+	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
+
+	dev->intr_handle->intr_vec =
+		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
+
+	if (!dev->intr_handle->intr_vec) {
+		RTE_LOG(ERR, PMD,
+			"Failed to allocate memory for interrupt vector\n");
+		free(dev->intr_handle);
+		return -ENOMEM;
+	}
+
+	RTE_LOG(INFO, PMD, "Prepare intr vec\n");
+	for (i = 0; i < nb_rxq; i++) {
+		vq = dev->data->rx_queues[i];
+		if (!vq) {
+			RTE_LOG(INFO, PMD, "rxq-%d not setup yet, skip!\n", i);
+			continue;
+		}
+
+		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
+		if (ret < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to get rxq-%d's vring, skip!\n", i);
+			continue;
+		}
+
+		if (vring.kickfd < 0) {
+			RTE_LOG(INFO, PMD,
+				"rxq-%d's kickfd is invalid, skip!\n", i);
+			continue;
+		}
+		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+		dev->intr_handle->efds[i] = vring.kickfd;
+		count++;
+		RTE_LOG(INFO, PMD, "Installed intr vec for rxq-%d\n", i);
+	}
+
+	dev->intr_handle->nb_efd = count;
+	dev->intr_handle->max_intr = count + 1;
+	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
+
+	return 0;
+}
+
 static void
 queue_setup(struct rte_eth_dev *eth_dev, struct pmd_internal *internal)
 {
 	struct vhost_queue *vq;
+	int vid = internal->vid;
 	int i;
 
 	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 		vq = eth_dev->data->rx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
+
 	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 		vq = eth_dev->data->tx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
@@ -584,8 +692,9 @@ new_device(int vid)
 	struct rte_eth_dev *eth_dev;
 	struct internal_list *list;
 	struct pmd_internal *internal;
-	unsigned i;
+	struct rte_eth_conf *dev_conf;
 	char ifname[PATH_MAX];
+	int i;
 #ifdef RTE_LIBRTE_VHOST_NUMA
 	int newnode;
 #endif
@@ -599,6 +708,7 @@ new_device(int vid)
 
 	eth_dev = list->eth_dev;
 	internal = eth_dev->data->dev_private;
+	dev_conf = &eth_dev->data->dev_conf;
 
 #ifdef RTE_LIBRTE_VHOST_NUMA
 	newnode = rte_vhost_get_numa_node(vid);
@@ -609,14 +719,18 @@ new_device(int vid)
 	internal->vid = vid;
 	if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) {
 		queue_setup(eth_dev, internal);
-		rte_atomic32_set(&internal->dev_attached, 1);
-	} else {
-		RTE_LOG(INFO, PMD, "RX/TX queues have not setup yet\n");
-		rte_atomic32_set(&internal->dev_attached, 0);
-	}
 
-	for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
-		rte_vhost_enable_guest_notification(vid, i, 0);
+		for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
+			rte_vhost_enable_guest_notification(vid, i, 0);
+
+		if (dev_conf->intr_conf.rxq) {
+			if (eth_vhost_install_intr(eth_dev) < 0) {
+				RTE_LOG(INFO, PMD,
+					"Failed to install interrupt handler.");
+					return -1;
+			}
+		}
+	}
 
 	rte_vhost_get_mtu(vid, &eth_dev->data->mtu);
 
@@ -626,6 +740,8 @@ new_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Vhost device %d created\n", vid);
 
+	rte_atomic32_set(&internal->dev_attached, 1);
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 
 	return 0;
@@ -657,17 +773,19 @@ destroy_device(int vid)
 
 	eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
 
-	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
-		vq = eth_dev->data->rx_queues[i];
-		if (vq == NULL)
-			continue;
-		vq->vid = -1;
-	}
-	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
-		vq = eth_dev->data->tx_queues[i];
-		if (vq == NULL)
-			continue;
-		vq->vid = -1;
+	if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) {
+		for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
+			vq = eth_dev->data->rx_queues[i];
+			if (!vq)
+				continue;
+			vq->vid = -1;
+		}
+		for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
+			vq = eth_dev->data->tx_queues[i];
+			if (!vq)
+				continue;
+			vq->vid = -1;
+		}
 	}
 
 	state = vring_states[eth_dev->data->port_id];
@@ -681,6 +799,8 @@ destroy_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
 
+	eth_vhost_uninstall_intr(eth_dev);
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 }
 
@@ -791,10 +911,30 @@ static int
 eth_dev_start(struct rte_eth_dev *eth_dev)
 {
 	struct pmd_internal *internal = eth_dev->data->dev_private;
+	struct rte_eth_conf *dev_conf;
+	int vid = internal->vid;
+	int i;
 
-	if (unlikely(rte_atomic32_read(&internal->dev_attached) == 0)) {
-		queue_setup(eth_dev, internal);
-		rte_atomic32_set(&internal->dev_attached, 1);
+	dev_conf = &eth_dev->data->dev_conf;
+
+	if (!eth_dev->data->rx_queues || !eth_dev->data->tx_queues) {
+		RTE_LOG(ERR, PMD, "RX/TX queues not setup yet\n");
+		return -1;
+	}
+
+	queue_setup(eth_dev, internal);
+
+	if (likely(rte_atomic32_read(&internal->dev_attached) == 1)) {
+		for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
+			rte_vhost_enable_guest_notification(vid, i, 0);
+
+		if (dev_conf->intr_conf.rxq) {
+			if (eth_vhost_install_intr(eth_dev) < 0) {
+				RTE_LOG(INFO, PMD,
+					"Failed to install interrupt handler.");
+					return -1;
+			}
+		}
 	}
 
 	rte_atomic32_set(&internal->started, 1);
@@ -836,10 +976,13 @@ eth_dev_close(struct rte_eth_dev *dev)
 	pthread_mutex_unlock(&internal_list_lock);
 	rte_free(list);
 
-	for (i = 0; i < dev->data->nb_rx_queues; i++)
-		rte_free(dev->data->rx_queues[i]);
-	for (i = 0; i < dev->data->nb_tx_queues; i++)
-		rte_free(dev->data->tx_queues[i]);
+	if (dev->data->rx_queues)
+		for (i = 0; i < dev->data->nb_rx_queues; i++)
+			rte_free(dev->data->rx_queues[i]);
+
+	if (dev->data->tx_queues)
+		for (i = 0; i < dev->data->nb_tx_queues; i++)
+			rte_free(dev->data->tx_queues[i]);
 
 	rte_free(dev->data->mac_addrs);
 	free(internal->dev_name);
@@ -1030,6 +1173,8 @@ static const struct eth_dev_ops ops = {
 	.xstats_reset = vhost_dev_xstats_reset,
 	.xstats_get = vhost_dev_xstats_get,
 	.xstats_get_names = vhost_dev_xstats_get_names,
+	.rx_queue_intr_enable = eth_rxq_intr_enable,
+	.rx_queue_intr_disable = eth_rxq_intr_disable,
 };
 
 static struct rte_vdev_driver pmd_vhost_drv;
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index f6f12a0..68e9340 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -545,16 +545,15 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
 {
 	struct virtio_net *dev = get_device(vid);
 
-	if (dev == NULL)
+	if (!dev)
 		return -1;
 
-	if (enable) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"guest notification isn't supported.\n");
-		return -1;
-	}
+	if (enable)
+		dev->virtqueue[queue_id]->used->flags &=
+		    ~VRING_USED_F_NO_NOTIFY;
+	else
+		dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 
-	dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 	return 0;
 }
 
-- 
2.0.1

Re: [PATCH v6 1/2] vhost: add support for interrupt mode

[Tan, Jianfeng]

On 4/13/2018 12:28 AM, Junjie Chen wrote:
> In some cases we want vhost dequeue work in interrupt mode to
> release cpus to others when no data to transmit. So we install
> interrupt handler of vhost device and interrupt vectors for each
> rx queue when creating new backend according to vhost intrerupt
> configuration. Thus, applications could register a epoll event fd
> to associate rx queues with interrupt vectors.
>
> Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>

Reviewed-by: Jianfeng Tan <jianfeng.tan@intel.com>

> ---
> Changes in v6:
> - rebase code to master
> Changes in v5:
> - update license to DPDK new license format
> - rebase code to master
> Changes in v4:
> - revert back license change
> Changes in v3:
> - handle failure in the middle of intr setup.
> - use vhost API to enable interrupt.
> - rebase to check rxq existence.
> - update vhost API to support guest notification.
> Changes in v2:
> - update rx queue index.
> - fill efd_counter_size for intr handler.
> - update log.
>   drivers/net/vhost/rte_eth_vhost.c | 158 +++++++++++++++++++++++++++++++++++++-
>   lib/librte_vhost/vhost.c          |  14 ++--
>   2 files changed, 163 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
> index e392d71..8b4b716 100644
> --- a/drivers/net/vhost/rte_eth_vhost.c
> +++ b/drivers/net/vhost/rte_eth_vhost.c
> @@ -520,6 +520,136 @@ find_internal_resource(char *ifname)
>   	return list;
>   }
>   
> +static int
> +eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
> +{
> +	struct rte_vhost_vring vring;
> +	struct vhost_queue *vq;
> +	int ret = 0;
> +
> +	vq = dev->data->rx_queues[qid];
> +	if (!vq) {
> +		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
> +		return -1;
> +	}
> +
> +	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
> +	if (ret < 0) {
> +		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
> +		return ret;
> +	}
> +	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
> +	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1);
> +	rte_wmb();
> +
> +	return ret;
> +}
> +
> +static int
> +eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
> +{
> +	struct rte_vhost_vring vring;
> +	struct vhost_queue *vq;
> +	int ret = 0;
> +
> +	vq = dev->data->rx_queues[qid];
> +	if (!vq) {
> +		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
> +		return -1;
> +	}
> +
> +	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
> +	if (ret < 0) {
> +		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
> +		return ret;
> +	}
> +	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
> +	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
> +	rte_wmb();
> +
> +	return 0;
> +}
> +
> +static void
> +eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
> +{
> +	struct rte_intr_handle *intr_handle = dev->intr_handle;
> +
> +	if (intr_handle) {
> +		if (intr_handle->intr_vec)
> +			free(intr_handle->intr_vec);
> +		free(intr_handle);
> +	}
> +
> +	dev->intr_handle = NULL;
> +}
> +
> +static int
> +eth_vhost_install_intr(struct rte_eth_dev *dev)
> +{
> +	struct rte_vhost_vring vring;
> +	struct vhost_queue *vq;
> +	int count = 0;
> +	int nb_rxq = dev->data->nb_rx_queues;
> +	int i;
> +	int ret;
> +
> +	/* uninstall firstly if we are reconnecting */
> +	if (dev->intr_handle)
> +		eth_vhost_uninstall_intr(dev);
> +
> +	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
> +	if (!dev->intr_handle) {
> +		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
> +		return -ENOMEM;
> +	}
> +	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
> +
> +	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
> +
> +	dev->intr_handle->intr_vec =
> +		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
> +
> +	if (!dev->intr_handle->intr_vec) {
> +		RTE_LOG(ERR, PMD,
> +			"Failed to allocate memory for interrupt vector\n");
> +		free(dev->intr_handle);
> +		return -ENOMEM;
> +	}
> +
> +	RTE_LOG(INFO, PMD, "Prepare intr vec\n");
> +	for (i = 0; i < nb_rxq; i++) {
> +		vq = dev->data->rx_queues[i];
> +		if (!vq) {
> +			RTE_LOG(INFO, PMD, "rxq-%d not setup yet, skip!\n", i);
> +			continue;
> +		}
> +
> +		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
> +		if (ret < 0) {
> +			RTE_LOG(INFO, PMD,
> +				"Failed to get rxq-%d's vring, skip!\n", i);
> +			continue;
> +		}
> +
> +		if (vring.kickfd < 0) {
> +			RTE_LOG(INFO, PMD,
> +				"rxq-%d's kickfd is invalid, skip!\n", i);
> +			continue;
> +		}
> +		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
> +		dev->intr_handle->efds[i] = vring.kickfd;
> +		count++;
> +		RTE_LOG(INFO, PMD, "Installed intr vec for rxq-%d\n", i);
> +	}
> +
> +	dev->intr_handle->nb_efd = count;
> +	dev->intr_handle->max_intr = count + 1;
> +	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
> +
> +	return 0;
> +}
> +
>   static void
>   update_queuing_status(struct rte_eth_dev *dev)
>   {
> @@ -585,6 +715,7 @@ new_device(int vid)
>   	struct rte_eth_dev *eth_dev;
>   	struct internal_list *list;
>   	struct pmd_internal *internal;
> +	struct rte_eth_conf *dev_conf;
>   	unsigned i;
>   	char ifname[PATH_MAX];
>   #ifdef RTE_LIBRTE_VHOST_NUMA
> @@ -600,6 +731,7 @@ new_device(int vid)
>   
>   	eth_dev = list->eth_dev;
>   	internal = eth_dev->data->dev_private;
> +	dev_conf = &eth_dev->data->dev_conf;
>   
>   #ifdef RTE_LIBRTE_VHOST_NUMA
>   	newnode = rte_vhost_get_numa_node(vid);
> @@ -608,8 +740,17 @@ new_device(int vid)
>   #endif
>   
>   	internal->vid = vid;
> -	if (rte_atomic32_read(&internal->started) == 1)
> +	if (rte_atomic32_read(&internal->started) == 1) {
>   		queue_setup(eth_dev, internal);
> +
> +		if (dev_conf->intr_conf.rxq) {
> +			if (eth_vhost_install_intr(eth_dev) < 0) {
> +				RTE_LOG(INFO, PMD,
> +					"Failed to install interrupt handler.");
> +					return -1;
> +			}
> +		}
> +	}
>   	else
>   		RTE_LOG(INFO, PMD, "RX/TX queues not exist yet\n");
>   
> @@ -680,6 +821,7 @@ destroy_device(int vid)
>   	rte_spinlock_unlock(&state->lock);
>   
>   	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
> +	eth_vhost_uninstall_intr(eth_dev);
>   
>   	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
>   }
> @@ -791,8 +933,20 @@ static int
>   eth_dev_start(struct rte_eth_dev *eth_dev)
>   {
>   	struct pmd_internal *internal = eth_dev->data->dev_private;
> +	struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
>   
>   	queue_setup(eth_dev, internal);
> +	
> +	if (rte_atomic32_read(&internal->dev_attached) == 1) {
> +		if (dev_conf->intr_conf.rxq) {
> +			if (eth_vhost_install_intr(eth_dev) < 0) {
> +				RTE_LOG(INFO, PMD,
> +					"Failed to install interrupt handler.");
> +					return -1;
> +			}
> +		}
> +	}
> +
>   	rte_atomic32_set(&internal->started, 1);
>   	update_queuing_status(eth_dev);
>   
> @@ -1029,6 +1183,8 @@ static const struct eth_dev_ops ops = {
>   	.xstats_reset = vhost_dev_xstats_reset,
>   	.xstats_get = vhost_dev_xstats_get,
>   	.xstats_get_names = vhost_dev_xstats_get_names,
> +	.rx_queue_intr_enable = eth_rxq_intr_enable,
> +	.rx_queue_intr_disable = eth_rxq_intr_disable,
>   };
>   
>   static struct rte_vdev_driver pmd_vhost_drv;
> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
> index f6f12a0..10f795f 100644
> --- a/lib/librte_vhost/vhost.c
> +++ b/lib/librte_vhost/vhost.c
> @@ -545,16 +545,14 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
>   {
>   	struct virtio_net *dev = get_device(vid);
>   
> -	if (dev == NULL)
> -		return -1;
> -
> -	if (enable) {
> -		RTE_LOG(ERR, VHOST_CONFIG,
> -			"guest notification isn't supported.\n");
> +	if (!dev)
>   		return -1;
> -	}
>   
> -	dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
> +	if (enable)
> +		dev->virtqueue[queue_id]->used->flags &=
> +			~VRING_USED_F_NO_NOTIFY;
> +	else	
> +		dev->virtqueue[queue_id]->used->flags |= VRING_USED_F_NO_NOTIFY;
>   	return 0;
>   }
>   

Re: [PATCH v6 2/2] net/vhost: update license to SPDX format

[Maxime Coquelin]

On 04/12/2018 06:43 PM, Junjie Chen wrote:
> Update license to SPDX, also add Intel license.
> 
> Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
> ---
>   drivers/net/vhost/rte_eth_vhost.c | 34 +++-------------------------------
>   drivers/net/vhost/rte_eth_vhost.h | 35 +++--------------------------------
>   2 files changed, 6 insertions(+), 63 deletions(-)
> 
> diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
> index 8b4b716..c6b8637 100644
> --- a/drivers/net/vhost/rte_eth_vhost.c
> +++ b/drivers/net/vhost/rte_eth_vhost.c
> @@ -1,34 +1,6 @@
> -/*-
> - *   BSD LICENSE
> - *
> - *   Copyright (c) 2016 IGEL Co., Ltd.
> - *   All rights reserved.
> - *
> - *   Redistribution and use in source and binary forms, with or without
> - *   modification, are permitted provided that the following conditions
> - *   are met:
> - *
> - *     * Redistributions of source code must retain the above copyright
> - *       notice, this list of conditions and the following disclaimer.
> - *     * Redistributions in binary form must reproduce the above copyright
> - *       notice, this list of conditions and the following disclaimer in
> - *       the documentation and/or other materials provided with the
> - *       distribution.
> - *     * Neither the name of IGEL Co.,Ltd. nor the names of its
> - *       contributors may be used to endorse or promote products derived
> - *       from this software without specific prior written permission.
> - *
> - *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> - *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> - *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> - *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> - *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> - *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> - *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> - *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> - *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> - *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> - *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2016 IGEL Co., Ltd.
> + * Copyright(c) 2016-2018 Intel Corporation
>    */
>   #include <unistd.h>
>   #include <pthread.h>
> diff --git a/drivers/net/vhost/rte_eth_vhost.h b/drivers/net/vhost/rte_eth_vhost.h
> index 948f3c8..0e68b9f 100644
> --- a/drivers/net/vhost/rte_eth_vhost.h
> +++ b/drivers/net/vhost/rte_eth_vhost.h
> @@ -1,36 +1,7 @@
> -/*-
> - *   BSD LICENSE
> - *
> - *   Copyright(c) 2016 IGEL Co., Ltd.
> - *   All rights reserved.
> - *
> - *   Redistribution and use in source and binary forms, with or without
> - *   modification, are permitted provided that the following conditions
> - *   are met:
> - *
> - *     * Redistributions of source code must retain the above copyright
> - *       notice, this list of conditions and the following disclaimer.
> - *     * Redistributions in binary form must reproduce the above copyright
> - *       notice, this list of conditions and the following disclaimer in
> - *       the documentation and/or other materials provided with the
> - *       distribution.
> - *     * Neither the name of IGEL Co., Ltd. nor the names of its
> - *       contributors may be used to endorse or promote products derived
> - *       from this software without specific prior written permission.
> - *
> - *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> - *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> - *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> - *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> - *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> - *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> - *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> - *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> - *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> - *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> - *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2016 IGEL Co., Ltd.
> + * Copyright(c) 2016-2018 Intel Corporation
>    */
> -
>   #ifndef _RTE_ETH_VHOST_H_
>   #define _RTE_ETH_VHOST_H_
>   
> 

Acked-by: Maxime Coquelin <maxime.coquelin@redhat.com>

I'll apply it for -rc2.

Thanks,
Maxime

[PATCH v6 1/2] vhost: add support for interrupt mode

[Junjie Chen]

In some cases we want vhost dequeue work in interrupt mode to
release cpus to others when no data to transmit. So we install
interrupt handler of vhost device and interrupt vectors for each
rx queue when creating new backend according to vhost intrerupt
configuration. Thus, applications could register a epoll event fd
to associate rx queues with interrupt vectors.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
Changes in v6:
- rebase code to master
Changes in v5:
- update license to DPDK new license format
- rebase code to master 
Changes in v4:
- revert back license change
Changes in v3:
- handle failure in the middle of intr setup.
- use vhost API to enable interrupt.
- rebase to check rxq existence.
- update vhost API to support guest notification.
Changes in v2:
- update rx queue index.
- fill efd_counter_size for intr handler.
- update log.
 drivers/net/vhost/rte_eth_vhost.c | 158 +++++++++++++++++++++++++++++++++++++-
 lib/librte_vhost/vhost.c          |  14 ++--
 2 files changed, 163 insertions(+), 9 deletions(-)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index e392d71..8b4b716 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -520,6 +520,136 @@ find_internal_resource(char *ifname)
 	return list;
 }
 
+static int
+eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1);
+	rte_wmb();
+
+	return ret;
+}
+
+static int
+eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
+	rte_wmb();
+
+	return 0;
+}
+
+static void
+eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
+{
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (intr_handle) {
+		if (intr_handle->intr_vec)
+			free(intr_handle->intr_vec);
+		free(intr_handle);
+	}
+
+	dev->intr_handle = NULL;
+}
+
+static int
+eth_vhost_install_intr(struct rte_eth_dev *dev)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int count = 0;
+	int nb_rxq = dev->data->nb_rx_queues;
+	int i;
+	int ret;
+
+	/* uninstall firstly if we are reconnecting */
+	if (dev->intr_handle)
+		eth_vhost_uninstall_intr(dev);
+
+	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
+	if (!dev->intr_handle) {
+		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
+		return -ENOMEM;
+	}
+	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
+
+	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
+
+	dev->intr_handle->intr_vec =
+		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
+
+	if (!dev->intr_handle->intr_vec) {
+		RTE_LOG(ERR, PMD,
+			"Failed to allocate memory for interrupt vector\n");
+		free(dev->intr_handle);
+		return -ENOMEM;
+	}
+
+	RTE_LOG(INFO, PMD, "Prepare intr vec\n");
+	for (i = 0; i < nb_rxq; i++) {
+		vq = dev->data->rx_queues[i];
+		if (!vq) {
+			RTE_LOG(INFO, PMD, "rxq-%d not setup yet, skip!\n", i);
+			continue;
+		}
+
+		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
+		if (ret < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to get rxq-%d's vring, skip!\n", i);
+			continue;
+		}
+
+		if (vring.kickfd < 0) {
+			RTE_LOG(INFO, PMD,
+				"rxq-%d's kickfd is invalid, skip!\n", i);
+			continue;
+		}
+		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+		dev->intr_handle->efds[i] = vring.kickfd;
+		count++;
+		RTE_LOG(INFO, PMD, "Installed intr vec for rxq-%d\n", i);
+	}
+
+	dev->intr_handle->nb_efd = count;
+	dev->intr_handle->max_intr = count + 1;
+	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
+
+	return 0;
+}
+
 static void
 update_queuing_status(struct rte_eth_dev *dev)
 {
@@ -585,6 +715,7 @@ new_device(int vid)
 	struct rte_eth_dev *eth_dev;
 	struct internal_list *list;
 	struct pmd_internal *internal;
+	struct rte_eth_conf *dev_conf;
 	unsigned i;
 	char ifname[PATH_MAX];
 #ifdef RTE_LIBRTE_VHOST_NUMA
@@ -600,6 +731,7 @@ new_device(int vid)
 
 	eth_dev = list->eth_dev;
 	internal = eth_dev->data->dev_private;
+	dev_conf = &eth_dev->data->dev_conf;
 
 #ifdef RTE_LIBRTE_VHOST_NUMA
 	newnode = rte_vhost_get_numa_node(vid);
@@ -608,8 +740,17 @@ new_device(int vid)
 #endif
 
 	internal->vid = vid;
-	if (rte_atomic32_read(&internal->started) == 1)
+	if (rte_atomic32_read(&internal->started) == 1) {
 		queue_setup(eth_dev, internal);
+
+		if (dev_conf->intr_conf.rxq) {
+			if (eth_vhost_install_intr(eth_dev) < 0) {
+				RTE_LOG(INFO, PMD,
+					"Failed to install interrupt handler.");
+					return -1;
+			}
+		}
+	}
 	else
 		RTE_LOG(INFO, PMD, "RX/TX queues not exist yet\n");
 
@@ -680,6 +821,7 @@ destroy_device(int vid)
 	rte_spinlock_unlock(&state->lock);
 
 	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
+	eth_vhost_uninstall_intr(eth_dev);
 
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 }
@@ -791,8 +933,20 @@ static int
 eth_dev_start(struct rte_eth_dev *eth_dev)
 {
 	struct pmd_internal *internal = eth_dev->data->dev_private;
+	struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
 
 	queue_setup(eth_dev, internal);
+	
+	if (rte_atomic32_read(&internal->dev_attached) == 1) {
+		if (dev_conf->intr_conf.rxq) {
+			if (eth_vhost_install_intr(eth_dev) < 0) {
+				RTE_LOG(INFO, PMD,
+					"Failed to install interrupt handler.");
+					return -1;
+			}
+		}
+	}
+
 	rte_atomic32_set(&internal->started, 1);
 	update_queuing_status(eth_dev);
 
@@ -1029,6 +1183,8 @@ static const struct eth_dev_ops ops = {
 	.xstats_reset = vhost_dev_xstats_reset,
 	.xstats_get = vhost_dev_xstats_get,
 	.xstats_get_names = vhost_dev_xstats_get_names,
+	.rx_queue_intr_enable = eth_rxq_intr_enable,
+	.rx_queue_intr_disable = eth_rxq_intr_disable,
 };
 
 static struct rte_vdev_driver pmd_vhost_drv;
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index f6f12a0..10f795f 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -545,16 +545,14 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
 {
 	struct virtio_net *dev = get_device(vid);
 
-	if (dev == NULL)
-		return -1;
-
-	if (enable) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"guest notification isn't supported.\n");
+	if (!dev)
 		return -1;
-	}
 
-	dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
+	if (enable)
+		dev->virtqueue[queue_id]->used->flags &=
+			~VRING_USED_F_NO_NOTIFY;
+	else	
+		dev->virtqueue[queue_id]->used->flags |= VRING_USED_F_NO_NOTIFY;
 	return 0;
 }
 
-- 
2.0.1

[PATCH v6 2/2] net/vhost: update license to SPDX format

[Junjie Chen]

Update license to SPDX, also add Intel license.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
 drivers/net/vhost/rte_eth_vhost.c | 34 +++-------------------------------
 1 file changed, 3 insertions(+), 31 deletions(-)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 8b4b716..c6b8637 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -1,34 +1,6 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright (c) 2016 IGEL Co., Ltd.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of IGEL Co.,Ltd. nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016 IGEL Co., Ltd.
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 #include <unistd.h>
 #include <pthread.h>
-- 
2.0.1

[PATCH v6 2/2] net/vhost: update license to SPDX format

[Junjie Chen]

Update license to SPDX, also add Intel license.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
 drivers/net/vhost/rte_eth_vhost.c | 34 +++-------------------------------
 drivers/net/vhost/rte_eth_vhost.h | 35 +++--------------------------------
 2 files changed, 6 insertions(+), 63 deletions(-)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 8b4b716..c6b8637 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -1,34 +1,6 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright (c) 2016 IGEL Co., Ltd.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of IGEL Co.,Ltd. nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016 IGEL Co., Ltd.
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 #include <unistd.h>
 #include <pthread.h>
diff --git a/drivers/net/vhost/rte_eth_vhost.h b/drivers/net/vhost/rte_eth_vhost.h
index 948f3c8..0e68b9f 100644
--- a/drivers/net/vhost/rte_eth_vhost.h
+++ b/drivers/net/vhost/rte_eth_vhost.h
@@ -1,36 +1,7 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2016 IGEL Co., Ltd.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of IGEL Co., Ltd. nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016 IGEL Co., Ltd.
+ * Copyright(c) 2016-2018 Intel Corporation
  */
-
 #ifndef _RTE_ETH_VHOST_H_
 #define _RTE_ETH_VHOST_H_
 
-- 
2.0.1

Re: [PATCH v6 1/2] vhost: add support for interrupt mode

[Maxime Coquelin]

On 04/12/2018 06:28 PM, Junjie Chen wrote:
> In some cases we want vhost dequeue work in interrupt mode to
> release cpus to others when no data to transmit. So we install
> interrupt handler of vhost device and interrupt vectors for each
> rx queue when creating new backend according to vhost intrerupt
> configuration. Thus, applications could register a epoll event fd
> to associate rx queues with interrupt vectors.
> 
> Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
> ---
> Changes in v6:
> - rebase code to master
> Changes in v5:
> - update license to DPDK new license format
> - rebase code to master
> Changes in v4:
> - revert back license change
> Changes in v3:
> - handle failure in the middle of intr setup.
> - use vhost API to enable interrupt.
> - rebase to check rxq existence.
> - update vhost API to support guest notification.
> Changes in v2:
> - update rx queue index.
> - fill efd_counter_size for intr handler.
> - update log.
>   drivers/net/vhost/rte_eth_vhost.c | 158 +++++++++++++++++++++++++++++++++++++-
>   lib/librte_vhost/vhost.c          |  14 ++--
>   2 files changed, 163 insertions(+), 9 deletions(-)
> 

Applied to dpdk-next-virtio/master.

Please next time run checkpatch before posting, I had to fix below
errors when applying.

Thanks,
Maxime

### [dpdk-dev,v6,1/2] vhost: add support for interrupt mode

CHECK:BRACES: braces {} should be used on all arms of this statement
#208: FILE: drivers/net/vhost/rte_eth_vhost.c:743:
+	if (rte_atomic32_read(&internal->started) == 1) {
[...]
  	else
[...]

ERROR:TRAILING_WHITESPACE: trailing whitespace
#237: FILE: drivers/net/vhost/rte_eth_vhost.c:939:
+^I$

ERROR:TRAILING_WHITESPACE: trailing whitespace
#282: FILE: lib/librte_vhost/vhost.c:554:
+^Ielse^I$

total: 2 errors, 0 warnings, 225 lines checked

0/1 valid patch

Re: [PATCH v6 1/2] vhost: add support for interrupt mode

[Chen, Junjie J]

Hi 
> Please next time run checkpatch before posting, I had to fix below errors when

Thanks, I ran for v1-v5 and too confident for the last minor change.
> applying.
> 
> Thanks,
> Maxime
> 
> ### [dpdk-dev,v6,1/2] vhost: add support for interrupt mode
> 
> CHECK:BRACES: braces {} should be used on all arms of this statement
> #208: FILE: drivers/net/vhost/rte_eth_vhost.c:743:
> +	if (rte_atomic32_read(&internal->started) == 1) {
> [...]
>   	else
> [...]
> 
> ERROR:TRAILING_WHITESPACE: trailing whitespace
> #237: FILE: drivers/net/vhost/rte_eth_vhost.c:939:
> +^I$
> 
> ERROR:TRAILING_WHITESPACE: trailing whitespace
> #282: FILE: lib/librte_vhost/vhost.c:554:
> +^Ielse^I$
> 
> total: 2 errors, 0 warnings, 225 lines checked
> 
> 0/1 valid patch

Re: [PATCH v6 2/2] net/vhost: update license to SPDX format

[Takanari Hayama]

On 2018/04/13 1:43, Junjie Chen wrote:
> Update license to SPDX, also add Intel license.
> 
> Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
> ---
>  drivers/net/vhost/rte_eth_vhost.c | 34 +++-------------------------------
>  drivers/net/vhost/rte_eth_vhost.h | 35 +++--------------------------------
>  2 files changed, 6 insertions(+), 63 deletions(-)

Acked-by: Takanari Hayama <taki@igel.co.jp>

--
Takanari Hayama, Ph.D. <taki@igel.co.jp>
IGEL Co.,Ltd.
http://www.igel.co.jp/

Re: [PATCH v6 2/2] net/vhost: update license to SPDX format

[Maxime Coquelin]

On 04/19/2018 03:07 AM, Takanari Hayama wrote:
> 
> On 2018/04/13 1:43, Junjie Chen wrote:
>> Update license to SPDX, also add Intel license.
>>
>> Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
>> ---
>>   drivers/net/vhost/rte_eth_vhost.c | 34 +++-------------------------------
>>   drivers/net/vhost/rte_eth_vhost.h | 35 +++--------------------------------
>>   2 files changed, 6 insertions(+), 63 deletions(-)
> 
> Acked-by: Takanari Hayama <taki@igel.co.jp>

Thanks Takanari,

Applied to dpdk-next-virtio/master.

Maxime
> 
> --
> Takanari Hayama, Ph.D. <taki@igel.co.jp>
> IGEL Co.,Ltd.
> http://www.igel.co.jp/
>