[dpdk-dev] [PATCH v3 3/5] net/iavf_client: enable interrupt on control queue

[Jingjing Wu <jingjing.wu@intel.com>]

New devarg "intr": if intr=1, use interrupt mode on control queue

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 drivers/net/iavf/iavf.h               |  19 ++++
 drivers/net/iavf/iavf_client_ethdev.c | 131 ++++++++++++++++++++++----
 drivers/net/iavf/iavf_ethdev.c        |  18 +---
 3 files changed, 134 insertions(+), 34 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index c34f971721..5516ecf021 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -198,6 +198,7 @@ struct iavf_adapter {
 #ifdef RTE_LIBRTE_IAVF_CLIENT
 	/* used for avf_client driver */
 	struct vfio_device *user_dev;
+	int intr_mode; /* interrupt mode if true */
 #endif
 	bool rx_bulk_alloc_allowed;
 	/* For vector PMD */
@@ -234,6 +235,22 @@ iavf_init_adminq_parameter(struct iavf_hw *hw)
 	hw->aq.asq_buf_size = IAVF_AQ_BUF_SZ;
 }
 
+/* Enable default admin queue interrupt setting */
+static inline void
+iavf_enable_irq0(struct iavf_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1,
+		       IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
+
+	IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01,
+		       IAVF_VFINT_DYN_CTL01_INTENA_MASK |
+		       IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		       IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	IAVF_WRITE_FLUSH(hw);
+}
+
 static inline void
 iavf_disable_irq0(struct iavf_hw *hw)
 {
@@ -342,4 +359,6 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
 int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
+void iavf_dev_interrupt_handler(void *param);
+
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_client_ethdev.c b/drivers/net/iavf/iavf_client_ethdev.c
index 989f9d6062..770a7a8200 100644
--- a/drivers/net/iavf/iavf_client_ethdev.c
+++ b/drivers/net/iavf/iavf_client_ethdev.c
@@ -6,6 +6,8 @@
 #include <stdint.h>
 #include <string.h>
 
+#include <sys/eventfd.h>
+
 #include <rte_common.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
@@ -18,6 +20,9 @@
 #include "iavf.h"
 #include "iavf_rxtx.h"
 
+#define AVF_CLIENT_ARG_PATH           "path"
+#define AVF_CLIENT_ARG_INTR           "intr"
+
 static int iavf_client_dev_close(struct rte_eth_dev *dev);
 static int iavf_client_dev_reset(struct rte_eth_dev *dev);
 
@@ -25,11 +30,27 @@ static int iavf_client_dev_reset(struct rte_eth_dev *dev);
 static struct eth_dev_ops iavf_client_eth_dev_ops;
 
 static const char *valid_args[] = {
-#define AVF_CLIENT_ARG_PATH           "path"
 	AVF_CLIENT_ARG_PATH,
+	AVF_CLIENT_ARG_INTR,
 	NULL
 };
 
+static void
+iavf_client_event_handler(void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	eventfd_t buf;
+
+	eventfd_read(dev->intr_handle->fd, &buf);
+
+	iavf_disable_irq0(hw);
+
+	iavf_handle_virtchnl_msg(dev);
+
+	iavf_enable_irq0(hw);
+}
+
 /* set up vfio_device for iavf_client*/
 static int
 iavf_client_vfio_user_setup(struct rte_eth_dev *dev, const char *path)
@@ -51,6 +72,11 @@ iavf_client_vfio_user_setup(struct rte_eth_dev *dev, const char *path)
 
 	hw->back = IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 
+	if (!vfio_dev->nb_irqs && adapter->intr_mode) {
+		PMD_INIT_LOG(ERR, "No irq support on device");
+		return -1;
+	}
+
 	if (!dev->intr_handle) {
 		dev->intr_handle = rte_zmalloc_socket("iavf_client_intr",
 				sizeof(*dev->intr_handle),
@@ -62,7 +88,7 @@ iavf_client_vfio_user_setup(struct rte_eth_dev *dev, const char *path)
 
 	}
 
-	dev->intr_handle->fd = -1;
+	dev->intr_handle->fd = vfio_dev->irqfds[0];
 	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
 	dev->intr_handle->max_intr = 1;
 
@@ -145,26 +171,52 @@ iavf_client_eth_init(struct rte_eth_dev *eth_dev)
 	rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr,
 			&eth_dev->data->mac_addrs[0]);
 
-	rte_eal_alarm_set(IAVF_CLIENT_ALARM_INTERVAL,
-			  iavf_client_dev_alarm_handler, eth_dev);
+	if (adapter->intr_mode) {
+		/* register callback func to eal lib */
+		rte_intr_callback_register(eth_dev->intr_handle,
+					   iavf_client_event_handler,
+					   (void *)eth_dev);
+		iavf_enable_irq0(hw);
+	} else {
+		rte_eal_alarm_set(IAVF_CLIENT_ALARM_INTERVAL,
+				  iavf_client_dev_alarm_handler, eth_dev);
+	}
 	return 0;
 }
 
 static int
 iavf_client_dev_reset(struct rte_eth_dev *dev)
 {
+	struct iavf_adapter *adapter =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
 	int ret;
 
-	rte_eal_alarm_cancel(iavf_client_dev_alarm_handler, dev);
+	if (adapter->intr_mode) {
+		iavf_disable_irq0(hw);
+		/* unregister callback func from eal lib */
+		rte_intr_callback_unregister(intr_handle,
+					     iavf_client_event_handler, dev);
+	} else {
+		rte_eal_alarm_cancel(iavf_client_dev_alarm_handler, dev);
+	}
 
 	iavf_shutdown_adminq(hw);
 	ret = iavf_init_vf(dev);
 
 	/* send reset msg to PF */
 	iavf_vf_reset(hw);
-	rte_eal_alarm_set(IAVF_CLIENT_ALARM_INTERVAL,
-			  iavf_client_dev_alarm_handler, dev);
+	if (adapter->intr_mode) {
+		/* register callback func to eal lib */
+		rte_intr_callback_register(dev->intr_handle,
+					   iavf_client_event_handler,
+					   (void *)dev);
+		iavf_enable_irq0(hw);
+	} else {
+		rte_eal_alarm_set(IAVF_CLIENT_ALARM_INTERVAL,
+				  iavf_client_dev_alarm_handler, dev);
+	}
 
 	return ret;
 }
@@ -176,6 +228,15 @@ iavf_client_dev_close(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
+	if (adapter->intr_mode) {
+		iavf_disable_irq0(hw);
+		/* unregister callback func from eal lib */
+		rte_intr_callback_unregister(dev->intr_handle,
+					     iavf_client_event_handler, dev);
+	} else {
+		rte_eal_alarm_cancel(iavf_client_dev_alarm_handler, dev);
+	}
+
 	if (!adapter->stopped) {
 		iavf_stop_queues(dev);
 
@@ -188,10 +249,10 @@ iavf_client_dev_close(struct rte_eth_dev *dev)
 		iavf_add_del_all_mac_addr(adapter, false);
 		adapter->stopped = 1;
 	}
+
 	iavf_shutdown_adminq(hw);
-	iavf_disable_irq0(hw);
-	rte_eal_alarm_cancel(iavf_client_dev_alarm_handler, dev);
 	client_vfio_user_release((struct vfio_device *)hw->hw_addr);
+
 	return 0;
 }
 
@@ -210,6 +271,23 @@ iavf_client_get_string_arg(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+iavf_client_intr_check(__rte_unused const char *key,
+			const char *value, void *opaque)
+{
+	int *intr = (int *)opaque;
+	int ret = 0;
+
+	if (!strcmp(value, "1"))
+		*intr  = 1;
+	else if (!strcmp(value, "0"))
+		*intr = 0;
+	else
+		ret = -1;
+
+	return ret;
+}
+
 static int
 iavf_client_pmd_probe(struct rte_vdev_device *vdev)
 {
@@ -217,6 +295,7 @@ iavf_client_pmd_probe(struct rte_vdev_device *vdev)
 	struct rte_eth_dev *eth_dev;
 	struct iavf_adapter *adapter;
 	char *path = NULL;
+	int intr_mode = 0;
 	int ret;
 
 	kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_args);
@@ -227,35 +306,52 @@ iavf_client_pmd_probe(struct rte_vdev_device *vdev)
 
 	if (rte_kvargs_count(kvlist, AVF_CLIENT_ARG_PATH) == 1) {
 		if (rte_kvargs_process(kvlist, AVF_CLIENT_ARG_PATH,
-				       &iavf_client_get_string_arg, &path) < 0) {
+					&iavf_client_get_string_arg,
+					&path) < 0) {
 			PMD_INIT_LOG(ERR, "error to parse %s",
 				     AVF_CLIENT_ARG_PATH);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto free_kvlist;
 		}
 	} else {
 		PMD_INIT_LOG(ERR, "arg %s is mandatory for iavf_client",
 			     AVF_CLIENT_ARG_PATH);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto free_kvlist;
+	}
+
+	if (rte_kvargs_count(kvlist, AVF_CLIENT_ARG_INTR) == 1) {
+		if (rte_kvargs_process(kvlist, AVF_CLIENT_ARG_INTR,
+					iavf_client_intr_check,
+					&intr_mode) < 0) {
+			PMD_INIT_LOG(ERR, "arg %s must be 1 or 0",
+				     AVF_CLIENT_ARG_INTR);
+			ret = -EINVAL;
+			goto free_kvlist;
+		}
 	}
 
 	eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*adapter));
 
 	ret = iavf_client_vfio_user_setup(eth_dev, path);
-	if (ret) {
+	if (ret)
 		goto err;
-	}
+
+	adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
+	adapter->intr_mode = intr_mode;
 
 	ret = iavf_client_eth_init(eth_dev);
-	if (ret) {
+	if (ret)
 		goto err;
-	}
 
 	rte_eth_dev_probing_finish(eth_dev);
+
 	rte_kvargs_free(kvlist);
 
 	return 0;
 err:
 	rte_eth_dev_release_port(eth_dev);
+free_kvlist:
 	rte_kvargs_free(kvlist);
 	return ret;
 }
@@ -287,4 +383,5 @@ static struct rte_vdev_driver iavf_client_driver = {
 RTE_PMD_REGISTER_VDEV(net_iavf_client, iavf_client_driver);
 RTE_PMD_REGISTER_ALIAS(net_iavf_client, iavf_client);
 RTE_PMD_REGISTER_PARAM_STRING(net_iavf_client,
-	"path=<path>");
+	"path=<path>"
+	"intr=[0|1]");
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 6b5e47adf2..bc07ecbed7 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1869,23 +1869,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 	return -1;
 }
 
-/* Enable default admin queue interrupt setting */
-static inline void
-iavf_enable_irq0(struct iavf_hw *hw)
-{
-	/* Enable admin queue interrupt trigger */
-	IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1,
-		       IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
-
-	IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01,
-		       IAVF_VFINT_DYN_CTL01_INTENA_MASK |
-		       IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK |
-		       IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
-
-	IAVF_WRITE_FLUSH(hw);
-}
-
-static void
+void
 iavf_dev_interrupt_handler(void *param)
 {
 	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
-- 
2.21.1