[PATCH 0/2] i40evf: support interrupt based pf reset request

[PATCH 0/2] i40evf: support interrupt based pf reset request

[Jingjing Wu]

If DPDK is used on VF while the host is using Linux Kernel driver
as PF driver on FVL NIC, some setting on PF will trigger VF reset.
DPDK VF need to know the event.
This patch set makes the interrupt based request of PF reset from PF
supported by enabling the adminq event process in VF driver.
Users can register a callback for this interrupt event to get
informed, when a PF reset request detected like:
    rte_eth_dev_callback_register(portid,
                RTE_ETH_EVENT_INTR_RESET,
                reset_event_callback,
                arg);

Jingjing Wu (2):
  i40evf: allocate virtchnl cmd buffer for each vf
  i40evf: support interrupt based pf reset request

 drivers/net/i40e/i40e_ethdev.h    |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c | 425 +++++++++++++++++++++++++++-----------
 lib/librte_ether/rte_ethdev.h     |   1 +
 3 files changed, 304 insertions(+), 124 deletions(-)

-- 
2.4.0

[PATCH 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Jingjing Wu]

Currently, i40evf PMD uses a global static buffer to send virtchnl
command to host driver. It is shared by multi VFs.
This patch changed to allocate virtchnl cmd buffer for each VF.

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 drivers/net/i40e/i40e_ethdev.h    |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c | 183 +++++++++++++++-----------------------
 2 files changed, 75 insertions(+), 110 deletions(-)

diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index 1f9792b..93122ad 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -494,7 +494,9 @@ struct i40e_vf {
 	bool link_up;
 	bool vf_reset;
 	volatile uint32_t pend_cmd; /* pending command not finished yet */
+	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	u16 pend_msg; /* flags indicates events from pf not handled yet */
+	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
 
 	/* VSI info */
 	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 14d2a50..ebcd881 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -103,9 +103,6 @@ enum i40evf_aq_result {
 	I40EVF_MSG_CMD,      /* Read async command result */
 };
 
-/* A share buffer to store the command result from PF driver */
-static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
-
 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 static int i40evf_dev_start(struct rte_eth_dev *dev);
 static void i40evf_dev_stop(struct rte_eth_dev *dev);
@@ -237,31 +234,39 @@ i40evf_set_mac_type(struct i40e_hw *hw)
 }
 
 /*
- * Parse admin queue message.
- *
- * return value:
- *  < 0: meet error
- *  0: read sys msg
- *  > 0: read cmd result
+ * Read data in admin queue to get msg from pf driver
  */
 static enum i40evf_aq_result
-i40evf_parse_pfmsg(struct i40e_vf *vf,
-		   struct i40e_arq_event_info *event,
-		   struct i40evf_arq_msg_info *data)
+i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 {
-	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
-			rte_le_to_cpu_32(event->desc.cookie_high);
-	enum i40e_status_code retval = (enum i40e_status_code)\
-			rte_le_to_cpu_32(event->desc.cookie_low);
-	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info event;
+	enum i40e_virtchnl_ops opcode;
+	enum i40e_status_code retval;
+	int ret;
+	enum i40evf_aq_result result = I40EVF_MSG_NON;
 
+	event.buf_len = data->buf_len;
+	event.msg_buf = data->msg;
+	ret = i40e_clean_arq_element(hw, &event, NULL);
+	/* Can't read any msg from adminQ */
+	if (ret) {
+		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
+			result = I40EVF_MSG_NON;
+		else
+			result = I40EVF_MSG_ERR;
+		return result;
+	}
+
+	opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
+	retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 	/* pf sys event */
 	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 		struct i40e_virtchnl_pf_event *vpe =
-			(struct i40e_virtchnl_pf_event *)event->msg_buf;
+			(struct i40e_virtchnl_pf_event *)event.msg_buf;
 
-		/* Initialize ret to sys event */
-		ret = I40EVF_MSG_SYS;
+		result = I40EVF_MSG_SYS;
 		switch (vpe->event) {
 		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 			vf->link_up =
@@ -286,74 +291,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
 		}
 	} else {
 		/* async reply msg on command issued by vf previously */
-		ret = I40EVF_MSG_CMD;
+		result = I40EVF_MSG_CMD;
 		/* Actual data length read from PF */
-		data->msg_len = event->msg_len;
+		data->msg_len = event.msg_len;
 	}
-	/* fill the ops and result to notify VF */
+
 	data->result = retval;
 	data->ops = opcode;
 
-	return ret;
-}
-
-/*
- * Read data in admin queue to get msg from pf driver
- */
-static enum i40evf_aq_result
-i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
-{
-	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	struct i40e_arq_event_info event;
-	int ret;
-	enum i40evf_aq_result result = I40EVF_MSG_NON;
-
-	event.buf_len = data->buf_len;
-	event.msg_buf = data->msg;
-	ret = i40e_clean_arq_element(hw, &event, NULL);
-	/* Can't read any msg from adminQ */
-	if (ret) {
-		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
-			result = I40EVF_MSG_NON;
-		else
-			result = I40EVF_MSG_ERR;
-		return result;
-	}
-
-	/* Parse the event */
-	result = i40evf_parse_pfmsg(vf, &event, data);
-
 	return result;
 }
 
-/*
- * Polling read until command result return from pf driver or meet error.
- */
-static int
-i40evf_wait_cmd_done(struct rte_eth_dev *dev,
-		     struct i40evf_arq_msg_info *data)
-{
-	int i = 0;
-	enum i40evf_aq_result ret;
-
-#define MAX_TRY_TIMES 20
-#define ASQ_DELAY_MS  100
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, data);
-		if (ret == I40EVF_MSG_CMD)
-			return 0;
-		else if (ret == I40EVF_MSG_ERR)
-			return -1;
-
-		/* If don't read msg or read sys event, continue */
-	} while(i++ < MAX_TRY_TIMES);
-
-	return -1;
-}
-
 /**
  * clear current command. Only call in case execute
  * _atomic_set_cmd successfully.
@@ -380,13 +328,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 	return !ret;
 }
 
+#define MAX_TRY_TIMES 20
+#define ASQ_DELAY_MS  100
+
 static int
 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	int err = -1;
 	struct i40evf_arq_msg_info info;
+	enum i40evf_aq_result ret;
+	int err = -1;
+	int i = 0;
 
 	if (_atomic_set_cmd(vf, args->ops))
 		return -1;
@@ -404,19 +357,22 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	err = i40evf_wait_cmd_done(dev, &info);
-	/* read message and it's expected one */
-	if (!err && args->ops == info.ops)
-		_clear_cmd(vf);
-	else if (err) {
-		PMD_DRV_LOG(ERR, "Failed to read message from AdminQ");
-		_clear_cmd(vf);
-	}
-	else if (args->ops != info.ops)
-		PMD_DRV_LOG(ERR, "command mismatch, expect %u, get %u",
-			    args->ops, info.ops);
+	do {
+		/* Delay some time first */
+		rte_delay_ms(ASQ_DELAY_MS);
+		ret = i40evf_read_pfmsg(dev, &info);
+		if (ret == I40EVF_MSG_CMD) {
+			err = 0;
+			break;
+		} else if (ret == I40EVF_MSG_ERR) {
+			err = -1;
+			break;
+		}
+		/* If don't read msg or read sys event, continue */
+	} while (i++ < MAX_TRY_TIMES);
+	_clear_cmd(vf);
 
-	return (err | info.result);
+	return (err | vf->cmd_retval);
 }
 
 /*
@@ -436,7 +392,7 @@ i40evf_check_api_version(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_VERSION;
 	args.in_args = (uint8_t *)&version;
 	args.in_args_size = sizeof(version);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -474,7 +430,7 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev)
 	uint32_t caps, len;
 
 	args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	if (PF_IS_V11(vf)) {
 		caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
@@ -527,7 +483,7 @@ i40evf_config_promisc(struct rte_eth_dev *dev,
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 	args.in_args = (uint8_t *)&promisc;
 	args.in_args_size = sizeof(promisc);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -554,7 +510,7 @@ i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 	args.in_args = (uint8_t *)&offload;
 	args.in_args_size = sizeof(offload);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -585,7 +541,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 	args.in_args = (uint8_t *)&tpid_info;
 	args.in_args_size = sizeof(tpid_info);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -664,7 +620,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 	args.in_args = (uint8_t *)vc_vqci;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -717,7 +673,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 		(enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 	args.in_args = (uint8_t *)vc_vqcei;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -779,7 +735,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 	args.in_args = (u8 *)cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -810,7 +766,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 		args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 	args.in_args = (u8 *)&queue_select;
 	args.in_args_size = sizeof(queue_select);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -902,7 +858,7 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -939,7 +895,7 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -962,7 +918,7 @@ i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 	args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 	args.in_args = (u8 *)&q_stats;
 	args.in_args_size = sizeof(q_stats);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -1056,7 +1012,7 @@ i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1083,7 +1039,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1095,6 +1051,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 static int
 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 {
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	int err;
 	struct vf_cmd_info args;
 	struct rte_eth_link *new_link;
@@ -1102,7 +1059,7 @@ i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
 	args.in_args = NULL;
 	args.in_args_size = 0;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err) {
@@ -1160,7 +1117,7 @@ i40evf_reset_vf(struct i40e_hw *hw)
 		reset = rd32(hw, I40E_VFGEN_RSTAT) &
 			I40E_VFGEN_RSTAT_VFR_STATE_MASK;
 		reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT;
-		if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset)
+		if (I40E_VFR_VFACTIVE == reset)
 			break;
 		else
 			rte_delay_ms(50);
@@ -1196,7 +1153,6 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-
 	/* Reset VF and wait until it's complete */
 	if (i40evf_reset_vf(hw)) {
 		PMD_INIT_LOG(ERR, "reset NIC failed");
@@ -1214,6 +1170,11 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "init_adminq failed");
 		return -1;
 	}
+	vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
+	if (!vf->aq_resp) {
+		PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
+			goto err_aq;
+	}
 	if (i40evf_check_api_version(dev) != 0) {
 		PMD_INIT_LOG(ERR, "check_api version failed");
 		goto err_aq;
@@ -1279,6 +1240,8 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 		i40evf_dev_close(dev);
 	rte_free(vf->vf_res);
 	vf->vf_res = NULL;
+	rte_free(vf->aq_resp);
+	vf->aq_resp = NULL;
 
 	return 0;
 }
-- 
2.4.0

[PATCH 2/2] i40evf: support interrupt based pf reset request

[Jingjing Wu]

Interrupt based request of PF reset from PF is supported by
enabling the adminq event process in VF driver.
Users can register a callback for this interrupt event to get
informed, when a PF reset request detected like:
  rte_eth_dev_callback_register(portid,
		RTE_ETH_EVENT_INTR_RESET,
		reset_event_callback,
		arg);

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 drivers/net/i40e/i40e_ethdev_vf.c | 274 +++++++++++++++++++++++++++++++++-----
 lib/librte_ether/rte_ethdev.h     |   1 +
 2 files changed, 245 insertions(+), 30 deletions(-)

diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index ebcd881..cfeee77 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -74,8 +74,6 @@
 #define I40EVF_BUSY_WAIT_DELAY 10
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
-/*ITR index for NOITR*/
-#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
 
 struct i40evf_arq_msg_info {
 	enum i40e_virtchnl_ops ops;
@@ -151,6 +149,9 @@ static int
 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 static int
 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   uint16_t msglen);
 
 /* Default hash key buffer for RSS */
 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
@@ -357,20 +358,42 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, &info);
-		if (ret == I40EVF_MSG_CMD) {
-			err = 0;
-			break;
-		} else if (ret == I40EVF_MSG_ERR) {
-			err = -1;
-			break;
-		}
-		/* If don't read msg or read sys event, continue */
-	} while (i++ < MAX_TRY_TIMES);
-	_clear_cmd(vf);
+	switch (args->ops) {
+	case I40E_VIRTCHNL_OP_RESET_VF:
+		/*no need to process in this function */
+		break;
+	case I40E_VIRTCHNL_OP_VERSION:
+	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
+		/* for init adminq commands, need to poll the response */
+		do {
+			/* Delay some time first */
+			rte_delay_ms(ASQ_DELAY_MS);
+			ret = i40evf_read_pfmsg(dev, &info);
+			if (ret == I40EVF_MSG_CMD) {
+				err = 0;
+				break;
+			} else if (ret == I40EVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		_clear_cmd(vf);
+		break;
+
+	default:
+		/* for other adminq in running time, waiting the cmd done flag */
+		do {
+			/* Delay some time first */
+			rte_delay_ms(ASQ_DELAY_MS);
+			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
+				err = 0;
+				break;
+			}
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		break;
+	}
 
 	return (err | vf->cmd_retval);
 }
@@ -719,7 +742,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 
 	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = 1;
-	map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
+	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 	/* Alway use default dynamic MSIX interrupt */
 	map_info->vecmap[0].vector_id = vector_id;
@@ -1093,6 +1116,38 @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/* Disable IRQ0 */
+static inline void
+i40evf_disable_irq0(struct i40e_hw *hw)
+{
+	/* Disable all interrupt types */
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	I40EVF_WRITE_FLUSH(hw);
+}
+
+/* Enable IRQ0 */
+static inline void
+i40evf_enable_irq0(struct i40e_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	uint32_t val;
+
+	i40evf_disable_irq0(hw);
+	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
+	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
+		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
+
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		I40E_VFINT_DYN_CTL01_INTENA_MASK |
+		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	I40EVF_WRITE_FLUSH(hw);
+}
+
 static int
 i40evf_reset_vf(struct i40e_hw *hw)
 {
@@ -1137,6 +1192,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	int i, err, bufsz;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	uint16_t interval =
+		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1218,6 +1275,15 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	ether_addr_copy((struct ether_addr *)vf->vsi_res->default_mac_addr,
 					(struct ether_addr *)hw->mac.addr);
 
+	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
+	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       (I40E_ITR_INDEX_DEFAULT <<
+				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
+			       (interval <<
+				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
+	}
+
 	return 0;
 
 err_alloc:
@@ -1246,11 +1312,142 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   __rte_unused uint16_t msglen)
+{
+	struct i40e_virtchnl_pf_event *pf_msg =
+			(struct i40e_virtchnl_pf_event *)msg;
+
+	switch (pf_msg->event) {
+	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
+		break;
+	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
+		break;
+	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
+		break;
+	default:
+		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
+		break;
+	}
+}
+
+static void
+i40evf_handle_aq_msg(struct rte_eth_dev *dev)
+{
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info info;
+	struct i40e_virtchnl_msg *v_msg;
+	uint16_t pending, opcode;
+	int ret;
+
+	info.buf_len = I40E_AQ_BUF_SZ;
+	if (!vf->aq_resp) {
+		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
+		return;
+	}
+	info.msg_buf = vf->aq_resp;
+	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
+
+	pending = 1;
+	while (pending) {
+		ret = i40e_clean_arq_element(hw, &info, &pending);
+
+		if (ret != I40E_SUCCESS) {
+			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
+				    "ret: %d", ret);
+			break;
+		}
+		opcode = rte_le_to_cpu_16(info.desc.opcode);
+
+		switch (opcode) {
+		case i40e_aqc_opc_send_msg_to_vf:
+			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
+				/* process event*/
+				i40evf_handle_pf_event(dev, info.msg_buf,
+							info.msg_len);
+			else {
+				/* read message and it's expected one */
+				if (v_msg->v_opcode == vf->pend_cmd) {
+					vf->cmd_retval = v_msg->v_retval;
+					/* prevent compiler reordering */
+					rte_compiler_barrier();
+					_clear_cmd(vf);
+				} else
+					PMD_DRV_LOG(ERR, "command mismatch,"
+						"expect %u, get %u",
+						vf->pend_cmd, v_msg->v_opcode);
+				 PMD_DRV_LOG(DEBUG, "adminq response is received,"
+					     " opcode = %d\n", v_msg->v_opcode);
+			}
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
+				    opcode);
+			break;
+		}
+	}
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt. Only adminq interrupt is processed in VF.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
+			   void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	i40evf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
+		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
+		goto done;
+	}
+
+	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
+		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
+		i40evf_handle_aq_msg(dev);
+	}
+
+	/* Link Status Change interrupt */
+	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
+		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
+				   " do nothing\n");
+
+done:
+	i40evf_enable_irq0(hw);
+	rte_intr_enable(&(dev->pci_dev->intr_handle));
+}
+
+
 static int
 i40evf_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
 			eth_dev->data->dev_private);
+	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1285,6 +1482,16 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
+	/* register callback func to eal lib */
+	rte_intr_callback_register(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)eth_dev);
+
+	/* configure and enable device interrupt */
+	i40evf_enable_irq0(hw);
+	/* intr is enabled in i40evf_enable_queues_intr when dev_start */
+
+	/* enable uio intr after callback register */
+	rte_intr_enable(&(pci_dev->intr_handle));
 	/* copy mac addr */
 	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
 					ETHER_ADDR_LEN, 0);
@@ -1662,7 +1869,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 		I40E_WRITE_REG(hw,
 			       I40E_VFINT_DYN_CTL01,
 			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
-			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1673,11 +1881,10 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
 			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
-	else
-		/* To support Linux PF host */
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
-				I40E_VFINT_DYN_CTL01_INTENA_MASK |
-				I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't set I40E_VFINT_DYN_CTL01,
+	 * because it is already done in i40evf_enable_irq0.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1690,7 +1897,8 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 
 	if (!rte_intr_allow_others(intr_handle)) {
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1700,8 +1908,10 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
 						    - 1),
 			       0);
-	else
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't zero I40E_VFINT_DYN_CTL01,
+	 * because interrupt 0 is also used for adminq processing.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1825,10 +2035,6 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* vf don't allow intr except for rxq intr */
-	if (dev->data->dev_conf.intr_conf.rxq != 0)
-		rte_intr_enable(intr_handle);
-
 	i40evf_enable_queues_intr(dev);
 	return 0;
 
@@ -2020,12 +2226,20 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = dev->pci_dev;
 
 	i40evf_dev_stop(dev);
 	hw->adapter_stopped = 1;
 	i40e_dev_free_queues(dev);
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(&(pci_dev->intr_handle));
+
+	/* unregister callback func from eal lib */
+	rte_intr_callback_unregister(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)dev);
+	i40evf_disable_irq0(hw);
 }
 
 static int
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index bada8ad..1be1783 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -2666,6 +2666,7 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
 enum rte_eth_event_type {
 	RTE_ETH_EVENT_UNKNOWN,  /**< unknown event type */
 	RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
+	RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.4.0

[PATCH v2 0/2] i40evf: support interrupt based pf reset request

[Jingjing Wu]

v2 changes:
  remove the change on vf reset status checking
  add pf event report support in release note

If DPDK is used on VF while the host is using Linux Kernel
driver as PF driver on FVL NIC, some setting on PF will trigger
VF reset. DPDK VF need to know the event.
This patch set makes the interrupt based request of PF reset
from PF supported by enabling the adminq event process in
VF driver.
Users can register a callback for this interrupt event to get
informed, when a PF reset request detected like:
    rte_eth_dev_callback_register(portid,
                RTE_ETH_EVENT_INTR_RESET,
                reset_event_callback,
                arg);


Jingjing Wu (2):
  i40evf: allocate virtchnl cmd buffer for each vf
  i40evf: support interrupt based pf reset request

 doc/guides/rel_notes/release_2_3.rst |   1 +
 drivers/net/i40e/i40e_ethdev.h       |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c    | 423 +++++++++++++++++++++++++----------
 lib/librte_ether/rte_ethdev.h        |   1 +
 4 files changed, 304 insertions(+), 123 deletions(-)

-- 
2.4.0

[PATCH v2 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Jingjing Wu]

Currently, i40evf PMD uses a global static buffer to send virtchnl
command to host driver. It is shared by multi VFs.
This patch changed to allocate virtchnl cmd buffer for each VF.

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 drivers/net/i40e/i40e_ethdev.h    |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c | 181 +++++++++++++++-----------------------
 2 files changed, 74 insertions(+), 109 deletions(-)

diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index 1f9792b..93122ad 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -494,7 +494,9 @@ struct i40e_vf {
 	bool link_up;
 	bool vf_reset;
 	volatile uint32_t pend_cmd; /* pending command not finished yet */
+	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	u16 pend_msg; /* flags indicates events from pf not handled yet */
+	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
 
 	/* VSI info */
 	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 14d2a50..64e6957 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -103,9 +103,6 @@ enum i40evf_aq_result {
 	I40EVF_MSG_CMD,      /* Read async command result */
 };
 
-/* A share buffer to store the command result from PF driver */
-static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
-
 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 static int i40evf_dev_start(struct rte_eth_dev *dev);
 static void i40evf_dev_stop(struct rte_eth_dev *dev);
@@ -237,31 +234,39 @@ i40evf_set_mac_type(struct i40e_hw *hw)
 }
 
 /*
- * Parse admin queue message.
- *
- * return value:
- *  < 0: meet error
- *  0: read sys msg
- *  > 0: read cmd result
+ * Read data in admin queue to get msg from pf driver
  */
 static enum i40evf_aq_result
-i40evf_parse_pfmsg(struct i40e_vf *vf,
-		   struct i40e_arq_event_info *event,
-		   struct i40evf_arq_msg_info *data)
+i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 {
-	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
-			rte_le_to_cpu_32(event->desc.cookie_high);
-	enum i40e_status_code retval = (enum i40e_status_code)\
-			rte_le_to_cpu_32(event->desc.cookie_low);
-	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info event;
+	enum i40e_virtchnl_ops opcode;
+	enum i40e_status_code retval;
+	int ret;
+	enum i40evf_aq_result result = I40EVF_MSG_NON;
 
+	event.buf_len = data->buf_len;
+	event.msg_buf = data->msg;
+	ret = i40e_clean_arq_element(hw, &event, NULL);
+	/* Can't read any msg from adminQ */
+	if (ret) {
+		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
+			result = I40EVF_MSG_NON;
+		else
+			result = I40EVF_MSG_ERR;
+		return result;
+	}
+
+	opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
+	retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 	/* pf sys event */
 	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 		struct i40e_virtchnl_pf_event *vpe =
-			(struct i40e_virtchnl_pf_event *)event->msg_buf;
+			(struct i40e_virtchnl_pf_event *)event.msg_buf;
 
-		/* Initialize ret to sys event */
-		ret = I40EVF_MSG_SYS;
+		result = I40EVF_MSG_SYS;
 		switch (vpe->event) {
 		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 			vf->link_up =
@@ -286,74 +291,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
 		}
 	} else {
 		/* async reply msg on command issued by vf previously */
-		ret = I40EVF_MSG_CMD;
+		result = I40EVF_MSG_CMD;
 		/* Actual data length read from PF */
-		data->msg_len = event->msg_len;
+		data->msg_len = event.msg_len;
 	}
-	/* fill the ops and result to notify VF */
+
 	data->result = retval;
 	data->ops = opcode;
 
-	return ret;
-}
-
-/*
- * Read data in admin queue to get msg from pf driver
- */
-static enum i40evf_aq_result
-i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
-{
-	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	struct i40e_arq_event_info event;
-	int ret;
-	enum i40evf_aq_result result = I40EVF_MSG_NON;
-
-	event.buf_len = data->buf_len;
-	event.msg_buf = data->msg;
-	ret = i40e_clean_arq_element(hw, &event, NULL);
-	/* Can't read any msg from adminQ */
-	if (ret) {
-		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
-			result = I40EVF_MSG_NON;
-		else
-			result = I40EVF_MSG_ERR;
-		return result;
-	}
-
-	/* Parse the event */
-	result = i40evf_parse_pfmsg(vf, &event, data);
-
 	return result;
 }
 
-/*
- * Polling read until command result return from pf driver or meet error.
- */
-static int
-i40evf_wait_cmd_done(struct rte_eth_dev *dev,
-		     struct i40evf_arq_msg_info *data)
-{
-	int i = 0;
-	enum i40evf_aq_result ret;
-
-#define MAX_TRY_TIMES 20
-#define ASQ_DELAY_MS  100
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, data);
-		if (ret == I40EVF_MSG_CMD)
-			return 0;
-		else if (ret == I40EVF_MSG_ERR)
-			return -1;
-
-		/* If don't read msg or read sys event, continue */
-	} while(i++ < MAX_TRY_TIMES);
-
-	return -1;
-}
-
 /**
  * clear current command. Only call in case execute
  * _atomic_set_cmd successfully.
@@ -380,13 +328,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 	return !ret;
 }
 
+#define MAX_TRY_TIMES 20
+#define ASQ_DELAY_MS  100
+
 static int
 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	int err = -1;
 	struct i40evf_arq_msg_info info;
+	enum i40evf_aq_result ret;
+	int err = -1;
+	int i = 0;
 
 	if (_atomic_set_cmd(vf, args->ops))
 		return -1;
@@ -404,19 +357,22 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	err = i40evf_wait_cmd_done(dev, &info);
-	/* read message and it's expected one */
-	if (!err && args->ops == info.ops)
-		_clear_cmd(vf);
-	else if (err) {
-		PMD_DRV_LOG(ERR, "Failed to read message from AdminQ");
-		_clear_cmd(vf);
-	}
-	else if (args->ops != info.ops)
-		PMD_DRV_LOG(ERR, "command mismatch, expect %u, get %u",
-			    args->ops, info.ops);
+	do {
+		/* Delay some time first */
+		rte_delay_ms(ASQ_DELAY_MS);
+		ret = i40evf_read_pfmsg(dev, &info);
+		if (ret == I40EVF_MSG_CMD) {
+			err = 0;
+			break;
+		} else if (ret == I40EVF_MSG_ERR) {
+			err = -1;
+			break;
+		}
+		/* If don't read msg or read sys event, continue */
+	} while (i++ < MAX_TRY_TIMES);
+	_clear_cmd(vf);
 
-	return (err | info.result);
+	return (err | vf->cmd_retval);
 }
 
 /*
@@ -436,7 +392,7 @@ i40evf_check_api_version(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_VERSION;
 	args.in_args = (uint8_t *)&version;
 	args.in_args_size = sizeof(version);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -474,7 +430,7 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev)
 	uint32_t caps, len;
 
 	args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	if (PF_IS_V11(vf)) {
 		caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
@@ -527,7 +483,7 @@ i40evf_config_promisc(struct rte_eth_dev *dev,
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 	args.in_args = (uint8_t *)&promisc;
 	args.in_args_size = sizeof(promisc);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -554,7 +510,7 @@ i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 	args.in_args = (uint8_t *)&offload;
 	args.in_args_size = sizeof(offload);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -585,7 +541,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 	args.in_args = (uint8_t *)&tpid_info;
 	args.in_args_size = sizeof(tpid_info);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -664,7 +620,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 	args.in_args = (uint8_t *)vc_vqci;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -717,7 +673,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 		(enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 	args.in_args = (uint8_t *)vc_vqcei;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -779,7 +735,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 	args.in_args = (u8 *)cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -810,7 +766,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 		args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 	args.in_args = (u8 *)&queue_select;
 	args.in_args_size = sizeof(queue_select);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -902,7 +858,7 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -939,7 +895,7 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -962,7 +918,7 @@ i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 	args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 	args.in_args = (u8 *)&q_stats;
 	args.in_args_size = sizeof(q_stats);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -1056,7 +1012,7 @@ i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1083,7 +1039,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1095,6 +1051,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 static int
 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 {
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	int err;
 	struct vf_cmd_info args;
 	struct rte_eth_link *new_link;
@@ -1102,7 +1059,7 @@ i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
 	args.in_args = NULL;
 	args.in_args_size = 0;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err) {
@@ -1196,7 +1153,6 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-
 	/* Reset VF and wait until it's complete */
 	if (i40evf_reset_vf(hw)) {
 		PMD_INIT_LOG(ERR, "reset NIC failed");
@@ -1214,6 +1170,11 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "init_adminq failed");
 		return -1;
 	}
+	vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
+	if (!vf->aq_resp) {
+		PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
+			goto err_aq;
+	}
 	if (i40evf_check_api_version(dev) != 0) {
 		PMD_INIT_LOG(ERR, "check_api version failed");
 		goto err_aq;
@@ -1279,6 +1240,8 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 		i40evf_dev_close(dev);
 	rte_free(vf->vf_res);
 	vf->vf_res = NULL;
+	rte_free(vf->aq_resp);
+	vf->aq_resp = NULL;
 
 	return 0;
 }
-- 
2.4.0

[PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Jingjing Wu]

Interrupt based request of PF reset from PF is supported by
enabling the adminq event process in VF driver.
Users can register a callback for this interrupt event to get
informed, when a PF reset request detected like:
  rte_eth_dev_callback_register(portid,
		RTE_ETH_EVENT_INTR_RESET,
		reset_event_callback,
		arg);

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 doc/guides/rel_notes/release_2_3.rst |   1 +
 drivers/net/i40e/i40e_ethdev_vf.c    | 274 +++++++++++++++++++++++++++++++----
 lib/librte_ether/rte_ethdev.h        |   1 +
 3 files changed, 246 insertions(+), 30 deletions(-)

diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 99de186..73d5f76 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -4,6 +4,7 @@ DPDK Release 2.3
 New Features
 ------------
 
+* **Added pf reset event reported in i40e vf PMD driver.
 
 Resolved Issues
 ---------------
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 64e6957..1ffe64e 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -74,8 +74,6 @@
 #define I40EVF_BUSY_WAIT_DELAY 10
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
-/*ITR index for NOITR*/
-#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
 
 struct i40evf_arq_msg_info {
 	enum i40e_virtchnl_ops ops;
@@ -151,6 +149,9 @@ static int
 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 static int
 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   uint16_t msglen);
 
 /* Default hash key buffer for RSS */
 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
@@ -357,20 +358,42 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, &info);
-		if (ret == I40EVF_MSG_CMD) {
-			err = 0;
-			break;
-		} else if (ret == I40EVF_MSG_ERR) {
-			err = -1;
-			break;
-		}
-		/* If don't read msg or read sys event, continue */
-	} while (i++ < MAX_TRY_TIMES);
-	_clear_cmd(vf);
+	switch (args->ops) {
+	case I40E_VIRTCHNL_OP_RESET_VF:
+		/*no need to process in this function */
+		break;
+	case I40E_VIRTCHNL_OP_VERSION:
+	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
+		/* for init adminq commands, need to poll the response */
+		do {
+			/* Delay some time first */
+			rte_delay_ms(ASQ_DELAY_MS);
+			ret = i40evf_read_pfmsg(dev, &info);
+			if (ret == I40EVF_MSG_CMD) {
+				err = 0;
+				break;
+			} else if (ret == I40EVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		_clear_cmd(vf);
+		break;
+
+	default:
+		/* for other adminq in running time, waiting the cmd done flag */
+		do {
+			/* Delay some time first */
+			rte_delay_ms(ASQ_DELAY_MS);
+			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
+				err = 0;
+				break;
+			}
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		break;
+	}
 
 	return (err | vf->cmd_retval);
 }
@@ -719,7 +742,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 
 	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = 1;
-	map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
+	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 	/* Alway use default dynamic MSIX interrupt */
 	map_info->vecmap[0].vector_id = vector_id;
@@ -1093,6 +1116,38 @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/* Disable IRQ0 */
+static inline void
+i40evf_disable_irq0(struct i40e_hw *hw)
+{
+	/* Disable all interrupt types */
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	I40EVF_WRITE_FLUSH(hw);
+}
+
+/* Enable IRQ0 */
+static inline void
+i40evf_enable_irq0(struct i40e_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	uint32_t val;
+
+	i40evf_disable_irq0(hw);
+	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
+	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
+		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
+
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		I40E_VFINT_DYN_CTL01_INTENA_MASK |
+		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	I40EVF_WRITE_FLUSH(hw);
+}
+
 static int
 i40evf_reset_vf(struct i40e_hw *hw)
 {
@@ -1137,6 +1192,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	int i, err, bufsz;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	uint16_t interval =
+		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1218,6 +1275,15 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	ether_addr_copy((struct ether_addr *)vf->vsi_res->default_mac_addr,
 					(struct ether_addr *)hw->mac.addr);
 
+	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
+	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       (I40E_ITR_INDEX_DEFAULT <<
+				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
+			       (interval <<
+				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
+	}
+
 	return 0;
 
 err_alloc:
@@ -1246,11 +1312,142 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   __rte_unused uint16_t msglen)
+{
+	struct i40e_virtchnl_pf_event *pf_msg =
+			(struct i40e_virtchnl_pf_event *)msg;
+
+	switch (pf_msg->event) {
+	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
+		break;
+	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
+		break;
+	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
+		break;
+	default:
+		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
+		break;
+	}
+}
+
+static void
+i40evf_handle_aq_msg(struct rte_eth_dev *dev)
+{
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info info;
+	struct i40e_virtchnl_msg *v_msg;
+	uint16_t pending, opcode;
+	int ret;
+
+	info.buf_len = I40E_AQ_BUF_SZ;
+	if (!vf->aq_resp) {
+		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
+		return;
+	}
+	info.msg_buf = vf->aq_resp;
+	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
+
+	pending = 1;
+	while (pending) {
+		ret = i40e_clean_arq_element(hw, &info, &pending);
+
+		if (ret != I40E_SUCCESS) {
+			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
+				    "ret: %d", ret);
+			break;
+		}
+		opcode = rte_le_to_cpu_16(info.desc.opcode);
+
+		switch (opcode) {
+		case i40e_aqc_opc_send_msg_to_vf:
+			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
+				/* process event*/
+				i40evf_handle_pf_event(dev, info.msg_buf,
+							info.msg_len);
+			else {
+				/* read message and it's expected one */
+				if (v_msg->v_opcode == vf->pend_cmd) {
+					vf->cmd_retval = v_msg->v_retval;
+					/* prevent compiler reordering */
+					rte_compiler_barrier();
+					_clear_cmd(vf);
+				} else
+					PMD_DRV_LOG(ERR, "command mismatch,"
+						"expect %u, get %u",
+						vf->pend_cmd, v_msg->v_opcode);
+				 PMD_DRV_LOG(DEBUG, "adminq response is received,"
+					     " opcode = %d\n", v_msg->v_opcode);
+			}
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
+				    opcode);
+			break;
+		}
+	}
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt. Only adminq interrupt is processed in VF.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
+			   void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	i40evf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
+		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
+		goto done;
+	}
+
+	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
+		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
+		i40evf_handle_aq_msg(dev);
+	}
+
+	/* Link Status Change interrupt */
+	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
+		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
+				   " do nothing\n");
+
+done:
+	i40evf_enable_irq0(hw);
+	rte_intr_enable(&(dev->pci_dev->intr_handle));
+}
+
+
 static int
 i40evf_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
 			eth_dev->data->dev_private);
+	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1285,6 +1482,16 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
+	/* register callback func to eal lib */
+	rte_intr_callback_register(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)eth_dev);
+
+	/* configure and enable device interrupt */
+	i40evf_enable_irq0(hw);
+	/* intr is enabled in i40evf_enable_queues_intr when dev_start */
+
+	/* enable uio intr after callback register */
+	rte_intr_enable(&(pci_dev->intr_handle));
 	/* copy mac addr */
 	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
 					ETHER_ADDR_LEN, 0);
@@ -1662,7 +1869,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 		I40E_WRITE_REG(hw,
 			       I40E_VFINT_DYN_CTL01,
 			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
-			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1673,11 +1881,10 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
 			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
-	else
-		/* To support Linux PF host */
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
-				I40E_VFINT_DYN_CTL01_INTENA_MASK |
-				I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't set I40E_VFINT_DYN_CTL01,
+	 * because it is already done in i40evf_enable_irq0.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1690,7 +1897,8 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 
 	if (!rte_intr_allow_others(intr_handle)) {
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1700,8 +1908,10 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
 						    - 1),
 			       0);
-	else
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't zero I40E_VFINT_DYN_CTL01,
+	 * because interrupt 0 is also used for adminq processing.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1825,10 +2035,6 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* vf don't allow intr except for rxq intr */
-	if (dev->data->dev_conf.intr_conf.rxq != 0)
-		rte_intr_enable(intr_handle);
-
 	i40evf_enable_queues_intr(dev);
 	return 0;
 
@@ -2020,12 +2226,20 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = dev->pci_dev;
 
 	i40evf_dev_stop(dev);
 	hw->adapter_stopped = 1;
 	i40e_dev_free_queues(dev);
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(&(pci_dev->intr_handle));
+
+	/* unregister callback func from eal lib */
+	rte_intr_callback_unregister(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)dev);
+	i40evf_disable_irq0(hw);
 }
 
 static int
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index bada8ad..1be1783 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -2666,6 +2666,7 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
 enum rte_eth_event_type {
 	RTE_ETH_EVENT_UNKNOWN,  /**< unknown event type */
 	RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
+	RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.4.0

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[David Marchand]

Hello Jingjing,

On Wed, Jan 27, 2016 at 2:49 AM, Jingjing Wu <jingjing.wu@intel.com> wrote:
> Interrupt based request of PF reset from PF is supported by
> enabling the adminq event process in VF driver.
> Users can register a callback for this interrupt event to get
> informed, when a PF reset request detected like:
>   rte_eth_dev_callback_register(portid,
>                 RTE_ETH_EVENT_INTR_RESET,
>                 reset_event_callback,
>                 arg);
>
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>

Just adding my previous comment in this thread.

Having this infrastructure is one thing, but the initial problem was
that the driver did not recover from this reset event.
The linux i40e vf driver handles this kind of event itself.
Could we have something similar ?

Thanks.

-- 
David Marchand

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Tao, Zhe]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jingjing Wu
> Sent: Wednesday, January 27, 2016 9:50 AM
> To: dev@dpdk.org
> Subject: [dpdk-dev] [PATCH v2 2/2] i40evf: support interrupt based pf reset
> request
> 
> Interrupt based request of PF reset from PF is supported by enabling the
> adminq event process in VF driver.
> Users can register a callback for this interrupt event to get informed, when a
> PF reset request detected like:
>   rte_eth_dev_callback_register(portid,
> 		RTE_ETH_EVENT_INTR_RESET,
> 		reset_event_callback,
> 		arg);
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> ---
>  doc/guides/rel_notes/release_2_3.rst |   1 +
>  drivers/net/i40e/i40e_ethdev_vf.c    | 274
> +++++++++++++++++++++++++++++++----
>  lib/librte_ether/rte_ethdev.h        |   1 +
>  3 files changed, 246 insertions(+), 30 deletions(-)
> 
> diff --git a/doc/guides/rel_notes/release_2_3.rst
> b/doc/guides/rel_notes/release_2_3.rst
> index 99de186..73d5f76 100644
> --- a/doc/guides/rel_notes/release_2_3.rst
> +++ b/doc/guides/rel_notes/release_2_3.rst
> @@ -4,6 +4,7 @@ DPDK Release 2.3
>  New Features
>  ------------
> 
> +* **Added pf reset event reported in i40e vf PMD driver.
> 
>  Resolved Issues
>  ---------------
> diff --git a/drivers/net/i40e/i40e_ethdev_vf.c
> b/drivers/net/i40e/i40e_ethdev_vf.c
> index 64e6957..1ffe64e 100644
> --- a/drivers/net/i40e/i40e_ethdev_vf.c
> +++ b/drivers/net/i40e/i40e_ethdev_vf.c
> @@ -74,8 +74,6 @@
> +static void
> @@ -1662,7 +1869,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev
> *dev)
>  		I40E_WRITE_REG(hw,
>  			       I40E_VFINT_DYN_CTL01,
>  			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
> -			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
> +			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
> +			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
What the usage for ITR bits here?
>  		I40EVF_WRITE_FLUSH(hw);
>  		return;
>  	}
> @@ -1673,11 +1881,10 @@ i40evf_enable_queues_intr(struct rte_eth_dev
> *dev)
> 
> 	I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
>  			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
>  			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
> -	else
> -		/* To support Linux PF host */
> -		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
> 
> --
> 2.4.0

Re: [PATCH v2 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Tao, Zhe]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jingjing Wu
> Sent: Wednesday, January 27, 2016 9:50 AM
> To: dev@dpdk.org
> Subject: [dpdk-dev] [PATCH v2 1/2] i40evf: allocate virtchnl cmd buffer for
> each vf
> 
> Currently, i40evf PMD uses a global static buffer to send virtchnl command to
> host driver. It is shared by multi VFs.
> This patch changed to allocate virtchnl cmd buffer for each VF.
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> ---
>  drivers/net/i40e/i40e_ethdev.h    |   2 +
>  drivers/net/i40e/i40e_ethdev_vf.c | 181 +++++++++++++++-------------------
> ----
>  2 files changed, 74 insertions(+), 109 deletions(-)
> 
> diff --git a/drivers/net/i40e/i40e_ethdev.h
> b/drivers/net/i40e/i40e_ethdev.h index 1f9792b..93122ad 100644
> --- a/drivers/net/i40e/i40e_ethdev.h
> +++ b/drivers/net/i40e/i40e_ethdev.h
> @@ -494,7 +494,9 @@ struct i40e_vf {
>  	bool link_up;
>  	bool vf_reset;
>  	volatile uint32_t pend_cmd; /* pending command not finished yet */
> +	uint32_t cmd_retval; /* return value of the cmd response from PF */
>  	u16 pend_msg; /* flags indicates events from pf not handled yet */
> +	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
> 
>  	/* VSI info */
>  	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */ diff --git
> a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
> index 14d2a50..64e6957 100644
> --- a/drivers/net/i40e/i40e_ethdev_vf.c
> +++ b/drivers/net/i40e/i40e_ethdev_vf.c
> @@ -103,9 +103,6 @@ enum i40evf_aq_result {
>  	I40EVF_MSG_CMD,      /* Read async command result */
>  };
> 
> -/* A share buffer to store the command result from PF driver */ -static
> uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
> -
>  static int i40evf_dev_configure(struct rte_eth_dev *dev);  static int
> i40evf_dev_start(struct rte_eth_dev *dev);  static void
> i40evf_dev_stop(struct rte_eth_dev *dev); @@ -237,31 +234,39 @@
> i40evf_set_mac_type(struct i40e_hw *hw)  }
> 
>  /*
> - * Parse admin queue message.
> - *
> - * return value:
> - *  < 0: meet error
> - *  0: read sys msg
> - *  > 0: read cmd result
> + * Read data in admin queue to get msg from pf driver
>   */
>  static enum i40evf_aq_result
> -i40evf_parse_pfmsg(struct i40e_vf *vf,
> -		   struct i40e_arq_event_info *event,
> -		   struct i40evf_arq_msg_info *data)
> +i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info
> +*data)
>  {
> -	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
> -			rte_le_to_cpu_32(event->desc.cookie_high);
> -	enum i40e_status_code retval = (enum i40e_status_code)\
> -			rte_le_to_cpu_32(event->desc.cookie_low);
> -	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
> +	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> +	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> >dev_private);
> +	struct i40e_arq_event_info event;
> +	enum i40e_virtchnl_ops opcode;
> +	enum i40e_status_code retval;
> +	int ret;
> +	enum i40evf_aq_result result = I40EVF_MSG_NON;
> 
> +	event.buf_len = data->buf_len;
> +	event.msg_buf = data->msg;
> +	ret = i40e_clean_arq_element(hw, &event, NULL);
> +	/* Can't read any msg from adminQ */
> +	if (ret) {
> +		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
> +			result = I40EVF_MSG_NON;
> +		else
> +			result = I40EVF_MSG_ERR;
> +		return result;
> +	}
> +
> +	opcode = (enum
> i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
> +	retval = (enum
> +i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
>  	/* pf sys event */
>  	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
>  		struct i40e_virtchnl_pf_event *vpe =
> -			(struct i40e_virtchnl_pf_event *)event->msg_buf;
> +			(struct i40e_virtchnl_pf_event *)event.msg_buf;
> 
> -		/* Initialize ret to sys event */
> -		ret = I40EVF_MSG_SYS;
> +		result = I40EVF_MSG_SYS;
>  		switch (vpe->event) {
>  		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
>  			vf->link_up =
> @@ -286,74 +291,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
>  		}
>  	} else {
>  		/* async reply msg on command issued by vf previously */
> -		ret = I40EVF_MSG_CMD;
> +		result = I40EVF_MSG_CMD;
>  		/* Actual data length read from PF */
> -		data->msg_len = event->msg_len;
> +		data->msg_len = event.msg_len;
>  	}
> -	/* fill the ops and result to notify VF */
> +
>  	data->result = retval;
>  	data->ops = opcode;
> 
> -	return ret;
> -}
> -
> -/*
> - * Read data in admin queue to get msg from pf driver
> - */
> -static enum i40evf_aq_result
> -i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info
> *data) -{
> -	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> -	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> >dev_private);
> -	struct i40e_arq_event_info event;
> -	int ret;
> -	enum i40evf_aq_result result = I40EVF_MSG_NON;
> -
> -	event.buf_len = data->buf_len;
> -	event.msg_buf = data->msg;
> -	ret = i40e_clean_arq_element(hw, &event, NULL);
> -	/* Can't read any msg from adminQ */
> -	if (ret) {
> -		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
> -			result = I40EVF_MSG_NON;
> -		else
> -			result = I40EVF_MSG_ERR;
> -		return result;
> -	}
> -
> -	/* Parse the event */
> -	result = i40evf_parse_pfmsg(vf, &event, data);
> -
>  	return result;
>  }
> 
> -/*
> - * Polling read until command result return from pf driver or meet error.
> - */
> -static int
> -i40evf_wait_cmd_done(struct rte_eth_dev *dev,
> -		     struct i40evf_arq_msg_info *data)
> -{
> -	int i = 0;
> -	enum i40evf_aq_result ret;
> -
> -#define MAX_TRY_TIMES 20
> -#define ASQ_DELAY_MS  100
> -	do {
> -		/* Delay some time first */
> -		rte_delay_ms(ASQ_DELAY_MS);
> -		ret = i40evf_read_pfmsg(dev, data);
> -		if (ret == I40EVF_MSG_CMD)
> -			return 0;
> -		else if (ret == I40EVF_MSG_ERR)
> -			return -1;
> -
> -		/* If don't read msg or read sys event, continue */
> -	} while(i++ < MAX_TRY_TIMES);
> -
> -	return -1;
> -}
> -
>  /**
>   * clear current command. Only call in case execute
>   * _atomic_set_cmd successfully.
> @@ -380,13 +328,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum
> i40e_virtchnl_ops ops)
>  	return !ret;
>  }
> 
> +#define MAX_TRY_TIMES 20
> +#define ASQ_DELAY_MS  100
> +
>  static int
>  i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
> {
>  	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
>  	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> >dev_private);
> -	int err = -1;
>  	struct i40evf_arq_msg_info info;
> +	enum i40evf_aq_result ret;
> +	int err = -1;
> +	int i = 0;
> 
>  	if (_atomic_set_cmd(vf, args->ops))
>  		return -1;
> @@ -404,19 +357,22 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev,
> struct vf_cmd_info *args)
>  		return err;
>  	}
> 
> -	err = i40evf_wait_cmd_done(dev, &info);
> -	/* read message and it's expected one */
> -	if (!err && args->ops == info.ops)
> -		_clear_cmd(vf);
> -	else if (err) {
> -		PMD_DRV_LOG(ERR, "Failed to read message from
> AdminQ");
> -		_clear_cmd(vf);
> -	}
> -	else if (args->ops != info.ops)
> -		PMD_DRV_LOG(ERR, "command mismatch, expect %u,
> get %u",
> -			    args->ops, info.ops);
> +	do {
> +		/* Delay some time first */
> +		rte_delay_ms(ASQ_DELAY_MS);
> +		ret = i40evf_read_pfmsg(dev, &info);
> +		if (ret == I40EVF_MSG_CMD) {
> +			err = 0;
> +			break;
> +		} else if (ret == I40EVF_MSG_ERR) {
> +			err = -1;
> +			break;
> +		}
> +		/* If don't read msg or read sys event, continue */
> +	} while (i++ < MAX_TRY_TIMES);
> +	_clear_cmd(vf);
> 
> -	return (err | info.result);
> +	return (err | vf->cmd_retval);
>  }
> 
Why combine the i40evf_parse_pfmsg and  i40evf_read_pfmsg into one function i40evf_read_pfmsg?

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Tao, Zhe]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jingjing Wu
> Sent: Wednesday, January 27, 2016 9:50 AM
> To: dev@dpdk.org
> Subject: [dpdk-dev] [PATCH v2 2/2] i40evf: support interrupt based pf reset
> request
> 
> Interrupt based request of PF reset from PF is supported by enabling the
> adminq event process in VF driver.
> Users can register a callback for this interrupt event to get informed, when a
> PF reset request detected like:
>   rte_eth_dev_callback_register(portid,
> 		RTE_ETH_EVENT_INTR_RESET,
> 		reset_event_callback,
> 		arg);
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Two questions,
1.If the VF RX/TX using msix 1 and Admin Queue using msix 0, 
how the two interrupts can both be read in user spaces, in VM VFIO not supported,

2.But if we want to  run l3fwd-power in VF, we can only assign the rx/tx intr to msix0,
but both thread will using epoll to wait the msix0 event, 
and intr thread may miss the vf reset event if l3fwd-power thread clean the msix0 related fd firstly
 if there are no more packets and msg response come in, the intr thread will not be wake up again

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Wu, Jingjing]

> -----Original Message-----
> From: Tao, Zhe
> Sent: Thursday, January 28, 2016 3:03 PM
> To: Wu, Jingjing
> Cc: dev@dpdk.org
> Subject: RE: [dpdk-dev] [PATCH v2 2/2] i40evf: support interrupt based pf
> reset request
> 
> > @@ -74,8 +74,6 @@
> > +static void
> > @@ -1662,7 +1869,8 @@ (struct rte_eth_dev
> > *dev)
> >  		I40E_WRITE_REG(hw,
> >  			       I40E_VFINT_DYN_CTL01,
> >  			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
> > -			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
> > +			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
> > +			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
> What the usage for ITR bits here?
According to the access type of register I40E_VFINT_DYN_CTL01, the ITR_INDX_MASK
here means don't update the ITR index.
> >  		I40EVF_WRITE_FLUSH(hw);
> >  		return;
> >  	}

Re: [PATCH v2 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Wu, Jingjing]

> -----Original Message-----
> From: Tao, Zhe
> Sent: Friday, January 29, 2016 3:28 PM
> To: Wu, Jingjing; dev@dpdk.org
> Subject: RE: [dpdk-dev] [PATCH v2 1/2] i40evf: allocate virtchnl cmd buffer
> for each vf
> 
> 
> 
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jingjing Wu
> > Sent: Wednesday, January 27, 2016 9:50 AM
> > To: dev@dpdk.org
> > Subject: [dpdk-dev] [PATCH v2 1/2] i40evf: allocate virtchnl cmd
> > buffer for each vf
> >
> > Currently, i40evf PMD uses a global static buffer to send virtchnl
> > command to host driver. It is shared by multi VFs.
> > This patch changed to allocate virtchnl cmd buffer for each VF.
> >
> > Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> > ---
> >  drivers/net/i40e/i40e_ethdev.h    |   2 +
> >  drivers/net/i40e/i40e_ethdev_vf.c | 181
> > +++++++++++++++-------------------
> > ----
> >  2 files changed, 74 insertions(+), 109 deletions(-)
> >
> > diff --git a/drivers/net/i40e/i40e_ethdev.h
> > b/drivers/net/i40e/i40e_ethdev.h index 1f9792b..93122ad 100644
> > --- a/drivers/net/i40e/i40e_ethdev.h
> > +++ b/drivers/net/i40e/i40e_ethdev.h
> > @@ -494,7 +494,9 @@ struct i40e_vf {
> >  	bool link_up;
> >  	bool vf_reset;
> >  	volatile uint32_t pend_cmd; /* pending command not finished yet */
> > +	uint32_t cmd_retval; /* return value of the cmd response from PF */
> >  	u16 pend_msg; /* flags indicates events from pf not handled yet */
> > +	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
> >
> >  	/* VSI info */
> >  	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */ diff --git
> > a/drivers/net/i40e/i40e_ethdev_vf.c
> > b/drivers/net/i40e/i40e_ethdev_vf.c
> > index 14d2a50..64e6957 100644
> > --- a/drivers/net/i40e/i40e_ethdev_vf.c
> > +++ b/drivers/net/i40e/i40e_ethdev_vf.c
> > @@ -103,9 +103,6 @@ enum i40evf_aq_result {
> >  	I40EVF_MSG_CMD,      /* Read async command result */
> >  };
> >
> > -/* A share buffer to store the command result from PF driver */
> > -static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
> > -
> >  static int i40evf_dev_configure(struct rte_eth_dev *dev);  static int
> > i40evf_dev_start(struct rte_eth_dev *dev);  static void
> > i40evf_dev_stop(struct rte_eth_dev *dev); @@ -237,31 +234,39 @@
> > i40evf_set_mac_type(struct i40e_hw *hw)  }
> >
> >  /*
> > - * Parse admin queue message.
> > - *
> > - * return value:
> > - *  < 0: meet error
> > - *  0: read sys msg
> > - *  > 0: read cmd result
> > + * Read data in admin queue to get msg from pf driver
> >   */
> >  static enum i40evf_aq_result
> > -i40evf_parse_pfmsg(struct i40e_vf *vf,
> > -		   struct i40e_arq_event_info *event,
> > -		   struct i40evf_arq_msg_info *data)
> > +i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info
> > +*data)
> >  {
> > -	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
> > -			rte_le_to_cpu_32(event->desc.cookie_high);
> > -	enum i40e_status_code retval = (enum i40e_status_code)\
> > -			rte_le_to_cpu_32(event->desc.cookie_low);
> > -	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
> > +	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> > >dev_private);
> > +	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> > >dev_private);
> > +	struct i40e_arq_event_info event;
> > +	enum i40e_virtchnl_ops opcode;
> > +	enum i40e_status_code retval;
> > +	int ret;
> > +	enum i40evf_aq_result result = I40EVF_MSG_NON;
> >
> > +	event.buf_len = data->buf_len;
> > +	event.msg_buf = data->msg;
> > +	ret = i40e_clean_arq_element(hw, &event, NULL);
> > +	/* Can't read any msg from adminQ */
> > +	if (ret) {
> > +		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
> > +			result = I40EVF_MSG_NON;
> > +		else
> > +			result = I40EVF_MSG_ERR;
> > +		return result;
> > +	}
> > +
> > +	opcode = (enum
> > i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
> > +	retval = (enum
> > +i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
> >  	/* pf sys event */
> >  	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
> >  		struct i40e_virtchnl_pf_event *vpe =
> > -			(struct i40e_virtchnl_pf_event *)event->msg_buf;
> > +			(struct i40e_virtchnl_pf_event *)event.msg_buf;
> >
> > -		/* Initialize ret to sys event */
> > -		ret = I40EVF_MSG_SYS;
> > +		result = I40EVF_MSG_SYS;
> >  		switch (vpe->event) {
> >  		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
> >  			vf->link_up =
> > @@ -286,74 +291,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
> >  		}
> >  	} else {
> >  		/* async reply msg on command issued by vf previously */
> > -		ret = I40EVF_MSG_CMD;
> > +		result = I40EVF_MSG_CMD;
> >  		/* Actual data length read from PF */
> > -		data->msg_len = event->msg_len;
> > +		data->msg_len = event.msg_len;
> >  	}
> > -	/* fill the ops and result to notify VF */
> > +
> >  	data->result = retval;
> >  	data->ops = opcode;
> >
> > -	return ret;
> > -}
> > -
> > -/*
> > - * Read data in admin queue to get msg from pf driver
> > - */
> > -static enum i40evf_aq_result
> > -i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info
> > *data) -{
> > -	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> > >dev_private);
> > -	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> > >dev_private);
> > -	struct i40e_arq_event_info event;
> > -	int ret;
> > -	enum i40evf_aq_result result = I40EVF_MSG_NON;
> > -
> > -	event.buf_len = data->buf_len;
> > -	event.msg_buf = data->msg;
> > -	ret = i40e_clean_arq_element(hw, &event, NULL);
> > -	/* Can't read any msg from adminQ */
> > -	if (ret) {
> > -		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
> > -			result = I40EVF_MSG_NON;
> > -		else
> > -			result = I40EVF_MSG_ERR;
> > -		return result;
> > -	}
> > -
> > -	/* Parse the event */
> > -	result = i40evf_parse_pfmsg(vf, &event, data);
> > -
> >  	return result;
> >  }
> >
> > -/*
> > - * Polling read until command result return from pf driver or meet error.
> > - */
> > -static int
> > -i40evf_wait_cmd_done(struct rte_eth_dev *dev,
> > -		     struct i40evf_arq_msg_info *data)
> > -{
> > -	int i = 0;
> > -	enum i40evf_aq_result ret;
> > -
> > -#define MAX_TRY_TIMES 20
> > -#define ASQ_DELAY_MS  100
> > -	do {
> > -		/* Delay some time first */
> > -		rte_delay_ms(ASQ_DELAY_MS);
> > -		ret = i40evf_read_pfmsg(dev, data);
> > -		if (ret == I40EVF_MSG_CMD)
> > -			return 0;
> > -		else if (ret == I40EVF_MSG_ERR)
> > -			return -1;
> > -
> > -		/* If don't read msg or read sys event, continue */
> > -	} while(i++ < MAX_TRY_TIMES);
> > -
> > -	return -1;
> > -}
> > -
> >  /**
> >   * clear current command. Only call in case execute
> >   * _atomic_set_cmd successfully.
> > @@ -380,13 +328,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum
> > i40e_virtchnl_ops ops)
> >  	return !ret;
> >  }
> >
> > +#define MAX_TRY_TIMES 20
> > +#define ASQ_DELAY_MS  100
> > +
> >  static int
> >  i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info
> > *args) {
> >  	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> > >dev_private);
> >  	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> > >dev_private);
> > -	int err = -1;
> >  	struct i40evf_arq_msg_info info;
> > +	enum i40evf_aq_result ret;
> > +	int err = -1;
> > +	int i = 0;
> >
> >  	if (_atomic_set_cmd(vf, args->ops))
> >  		return -1;
> > @@ -404,19 +357,22 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev,
> > struct vf_cmd_info *args)
> >  		return err;
> >  	}
> >
> > -	err = i40evf_wait_cmd_done(dev, &info);
> > -	/* read message and it's expected one */
> > -	if (!err && args->ops == info.ops)
> > -		_clear_cmd(vf);
> > -	else if (err) {
> > -		PMD_DRV_LOG(ERR, "Failed to read message from
> > AdminQ");
> > -		_clear_cmd(vf);
> > -	}
> > -	else if (args->ops != info.ops)
> > -		PMD_DRV_LOG(ERR, "command mismatch, expect %u,
> > get %u",
> > -			    args->ops, info.ops);
> > +	do {
> > +		/* Delay some time first */
> > +		rte_delay_ms(ASQ_DELAY_MS);
> > +		ret = i40evf_read_pfmsg(dev, &info);
> > +		if (ret == I40EVF_MSG_CMD) {
> > +			err = 0;
> > +			break;
> > +		} else if (ret == I40EVF_MSG_ERR) {
> > +			err = -1;
> > +			break;
> > +		}
> > +		/* If don't read msg or read sys event, continue */
> > +	} while (i++ < MAX_TRY_TIMES);
> > +	_clear_cmd(vf);
> >
> > -	return (err | info.result);
> > +	return (err | vf->cmd_retval);
> >  }
> >
> Why combine the i40evf_parse_pfmsg and  i40evf_read_pfmsg into one
> function i40evf_read_pfmsg?

Because what the i40evf_parse_pfmsg did is processing the pf event message, and
it is meaningless for checking whether the vf request message is done.
And the pf event message will be processed in i40evf_handle_pf_event which defined
in the next patch of the same patch set.

Thanks
Jingjing

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Wu, Jingjing]

> -----Original Message-----
> From: Tao, Zhe
> Sent: Friday, January 29, 2016 4:51 PM
> To: Wu, Jingjing; dev@dpdk.org
> Subject: RE: [dpdk-dev] [PATCH v2 2/2] i40evf: support interrupt based pf
> reset request
> 
> 
> 
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jingjing Wu
> > Sent: Wednesday, January 27, 2016 9:50 AM
> > To: dev@dpdk.org
> > Subject: [dpdk-dev] [PATCH v2 2/2] i40evf: support interrupt based pf
> > reset request
> >
> > Interrupt based request of PF reset from PF is supported by enabling
> > the adminq event process in VF driver.
> > Users can register a callback for this interrupt event to get
> > informed, when a PF reset request detected like:
> >   rte_eth_dev_callback_register(portid,
> > 		RTE_ETH_EVENT_INTR_RESET,
> > 		reset_event_callback,
> > 		arg);
> >
> > Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> Two questions,
> 1.If the VF RX/TX using msix 1 and Admin Queue using msix 0, how the two
> interrupts can both be read in user spaces, in VM VFIO not supported,
> 
If i40e kernel driver is used as host driver, no matter VM VFIO is supported or not,
only msix 0 is reserved for DPDK i40e VF.
> 2.But if we want to  run l3fwd-power in VF, we can only assign the rx/tx intr
> to msix0, but both thread will using epoll to wait the msix0 event, and intr
> thread may miss the vf reset event if l3fwd-power thread clean the msix0
> related fd firstly  if there are no more packets and msg response come in, the
> intr thread will not be wake up again
Yes, you are right. The same as above answer, if i40e kernel driver is used as
host driver, then RX interrupt is not supported.

Thanks
Jingjing 

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Wu, Jingjing]

> -----Original Message-----
> From: David Marchand [mailto:david.marchand@6wind.com]
> Sent: Wednesday, January 27, 2016 4:34 PM
> To: Wu, Jingjing
> Cc: dev@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH v2 2/2] i40evf: support interrupt based pf
> reset request
> 
> Hello Jingjing,
> 
> On Wed, Jan 27, 2016 at 2:49 AM, Jingjing Wu <jingjing.wu@intel.com> wrote:
> > Interrupt based request of PF reset from PF is supported by enabling
> > the adminq event process in VF driver.
> > Users can register a callback for this interrupt event to get
> > informed, when a PF reset request detected like:
> >   rte_eth_dev_callback_register(portid,
> >                 RTE_ETH_EVENT_INTR_RESET,
> >                 reset_event_callback,
> >                 arg);
> >
> > Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> 
> Just adding my previous comment in this thread.
> 
> Having this infrastructure is one thing, but the initial problem was that the
> driver did not recover from this reset event.
> The linux i40e vf driver handles this kind of event itself.
> Could we have something similar ?
> 

Hi, David

Considering about the how to use DPDK PMD, and how to setup resource, we can
know that lots of resources are managed by application. I think based on current
PMD driver framework, driver cannot reset without application's help.
If we need to support driver recovery automatically, we'd better to find a way to do that.
Do you have any idea?
 
However, this patch can notify the reset event to application, even it is not a perfect
solution as you hoped.

Thanks
Jingjing
 
> Thanks.
> 
> --
> David Marchand

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[David Marchand]

Hello,

On Sun, Feb 14, 2016 at 4:25 AM, Wu, Jingjing <jingjing.wu@intel.com> wrote:
>> -----Original Message-----
>> From: David Marchand [mailto:david.marchand@6wind.com]
>> Having this infrastructure is one thing, but the initial problem was that the
>> driver did not recover from this reset event.
>> The linux i40e vf driver handles this kind of event itself.
>> Could we have something similar ?
>>
>
> Considering about the how to use DPDK PMD, and how to setup resource, we can
> know that lots of resources are managed by application. I think based on current
> PMD driver framework, driver cannot reset without application's help.

I reported an issue on ixgbe.
What you provide here is a workaround for i40e.
I am not even sure this can be applied to ixgbe.

Does it mean that anytime we have a problem with drivers, workarounds
should be applied to ethdev / eal ... so that you don't have to handle
anything in the drivers ?
This is not the first time I complain about this kind of design issues.


> If we need to support driver recovery automatically, we'd better to find a way to do that.
> Do you have any idea?

First, list those "lots of resources" that "are managed by application".
If your driver needs to keep track of those, this is i40e driver job
to do this internally without requiring ethdev to be modified.

If this proves to be generic enough, maybe moving part of this to
ethdev will then make sense.


Thanks.

-- 
David Marchand

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Zhe Tao]

On Mon, Feb 15, 2016 at 02:16:16PM +0100, David Marchand wrote:
Hello,
> 
> On Sun, Feb 14, 2016 at 4:25 AM, Wu, Jingjing <jingjing.wu@intel.com> wrote:
> >> -----Original Message-----
> >> From: David Marchand [mailto:david.marchand@6wind.com]
> >> Having this infrastructure is one thing, but the initial problem was that the
> >> driver did not recover from this reset event.
> >> The linux i40e vf driver handles this kind of event itself.
> >> Could we have something similar ?
> >>
> >
> > Considering about the how to use DPDK PMD, and how to setup resource, we can
> > know that lots of resources are managed by application. I think based on current
> > PMD driver framework, driver cannot reset without application's help.
> 
> I reported an issue on ixgbe.
> What you provide here is a workaround for i40e.
> I am not even sure this can be applied to ixgbe.
> 
> Does it mean that anytime we have a problem with drivers, workarounds
> should be applied to ethdev / eal ... so that you don't have to handle
> anything in the drivers ?
I think this patch provides a necessary framework for i40e VF to handle
the asynchronous event, maybe Jingjing can help to change the description of
this patch to let it does not limit to the VF reset event. 
> This is not the first time I complain about this kind of design issues.
> 
> 
> > If we need to support driver recovery automatically, we'd better to find a way to do that.
> > Do you have any idea?
> 
> First, list those "lots of resources" that "are managed by application".
> If your driver needs to keep track of those, this is i40e driver job
> to do this internally without requiring ethdev to be modified.
> 
> If this proves to be generic enough, maybe moving part of this to
> ethdev will then make sense.
> 
> 
> Thanks.
> 
> -- 
> David Marchand
Thanks,

Zhe Tao

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Wu, Jingjing]

> I reported an issue on ixgbe.
Yes, thanks, we also notice such issue on ixgbe.

> What you provide here is a workaround for i40e.
> I am not even sure this can be applied to ixgbe.
>
Yes, not just workaround, also a basic one, without the patch, DPDK VF even
doesn't know the pf reset happened. I think ixgbe also need to know that.

> Does it mean that anytime we have a problem with drivers, workarounds
> should be applied to ethdev / eal ... so that you don't have to handle
> anything in the drivers ?

Currently as my understanding DPDK PMD driver is part of DPDK library.
Even the driver loading is in the thread which is created by application. From this
side, there is no a task which managed by driver internally. In fact, we also help
the reset process can be down automatically or at least provide an simple API to
application to help them recovery simply. Maybe the latter one is following the
current DPDK's framework. Otherwise, we need a thread for each driver?

And back to this patch, the patch just make the interrupt of pf reset can be received
by i40e vf PMD driver. It didn't change the ethdev/eal..... 
I don't think you have objection to it, right?

About how to process the reset event, we can raise another thread to discuss?

> This is not the first time I complain about this kind of design issues.
> 
> 
> > If we need to support driver recovery automatically, we'd better to find a way to do that.
> > Do you have any idea?
> 
> First, list those "lots of resources" that "are managed by application".
> If your driver needs to keep track of those, this is i40e driver job
> to do this internally without requiring ethdev to be modified.
>
Agree about the resource listing. But again, about the "internally", can you share your idea about it?
As you know, pmd driver even have no internal thread.

> If this proves to be generic enough, maybe moving part of this to
> ethdev will then make sense.
>
We can discuss, I think most NICs may have such issue. We need to make agreement on that.

Thanks
Jingjing
> 
> Thanks.
> 
> --
> David Marchand

Re: [PATCH v2 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Zhang, Helin]

> -----Original Message-----
> From: Wu, Jingjing
> Sent: Wednesday, January 27, 2016 9:50 AM
> To: dev@dpdk.org
> Cc: Wu, Jingjing; Zhang, Helin; Lu, Wenzhuo; Pei, Yulong
> Subject: [PATCH v2 1/2] i40evf: allocate virtchnl cmd buffer for each vf
> 
> Currently, i40evf PMD uses a global static buffer to send virtchnl command to
> host driver. It is shared by multi VFs.
> This patch changed to allocate virtchnl cmd buffer for each VF.
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> ---
>  drivers/net/i40e/i40e_ethdev.h    |   2 +
>  drivers/net/i40e/i40e_ethdev_vf.c | 181 +++++++++++++++-------------------
> ----
>  2 files changed, 74 insertions(+), 109 deletions(-)
> 
> diff --git a/drivers/net/i40e/i40e_ethdev.h
> b/drivers/net/i40e/i40e_ethdev.h index 1f9792b..93122ad 100644
> --- a/drivers/net/i40e/i40e_ethdev.h
> +++ b/drivers/net/i40e/i40e_ethdev.h
> @@ -494,7 +494,9 @@ struct i40e_vf {
>  	bool link_up;
>  	bool vf_reset;
>  	volatile uint32_t pend_cmd; /* pending command not finished yet */
> +	uint32_t cmd_retval; /* return value of the cmd response from PF */
>  	u16 pend_msg; /* flags indicates events from pf not handled yet */
> +	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
> 
>  	/* VSI info */
>  	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */ diff --git
> a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
> index 14d2a50..64e6957 100644
> --- a/drivers/net/i40e/i40e_ethdev_vf.c
> +++ b/drivers/net/i40e/i40e_ethdev_vf.c
> @@ -103,9 +103,6 @@ enum i40evf_aq_result {
>  	I40EVF_MSG_CMD,      /* Read async command result */
>  };
> 
> -/* A share buffer to store the command result from PF driver */ -static
> uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
> -
>  static int i40evf_dev_configure(struct rte_eth_dev *dev);  static int
> i40evf_dev_start(struct rte_eth_dev *dev);  static void
> i40evf_dev_stop(struct rte_eth_dev *dev); @@ -237,31 +234,39 @@
> i40evf_set_mac_type(struct i40e_hw *hw)  }
> 
>  /*
> - * Parse admin queue message.
> - *
> - * return value:
> - *  < 0: meet error
> - *  0: read sys msg
> - *  > 0: read cmd result
> + * Read data in admin queue to get msg from pf driver
>   */
>  static enum i40evf_aq_result
> -i40evf_parse_pfmsg(struct i40e_vf *vf,
> -		   struct i40e_arq_event_info *event,
> -		   struct i40evf_arq_msg_info *data)
> +i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info
> +*data)
>  {
> -	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
> -			rte_le_to_cpu_32(event->desc.cookie_high);
> -	enum i40e_status_code retval = (enum i40e_status_code)\
> -			rte_le_to_cpu_32(event->desc.cookie_low);
> -	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
> +	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> +	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> >dev_private);
> +	struct i40e_arq_event_info event;
> +	enum i40e_virtchnl_ops opcode;
> +	enum i40e_status_code retval;
> +	int ret;
> +	enum i40evf_aq_result result = I40EVF_MSG_NON;
> 
> +	event.buf_len = data->buf_len;
> +	event.msg_buf = data->msg;
> +	ret = i40e_clean_arq_element(hw, &event, NULL);
> +	/* Can't read any msg from adminQ */
> +	if (ret) {
> +		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
> +			result = I40EVF_MSG_NON;
No need to assign I40EVF_MSG_NON, as it was initialized at first.

> +		else
> +			result = I40EVF_MSG_ERR;
> +		return result;
> +	}
> +
> +	opcode = (enum
> i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
> +	retval = (enum
> +i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
>  	/* pf sys event */
>  	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
>  		struct i40e_virtchnl_pf_event *vpe =
> -			(struct i40e_virtchnl_pf_event *)event->msg_buf;
> +			(struct i40e_virtchnl_pf_event *)event.msg_buf;
> 
> -		/* Initialize ret to sys event */
> -		ret = I40EVF_MSG_SYS;
> +		result = I40EVF_MSG_SYS;
>  		switch (vpe->event) {
>  		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
>  			vf->link_up =
> @@ -286,74 +291,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
>  		}
>  	} else {
>  		/* async reply msg on command issued by vf previously */
> -		ret = I40EVF_MSG_CMD;
> +		result = I40EVF_MSG_CMD;
>  		/* Actual data length read from PF */
> -		data->msg_len = event->msg_len;
> +		data->msg_len = event.msg_len;
>  	}
> -	/* fill the ops and result to notify VF */
> +
>  	data->result = retval;
>  	data->ops = opcode;
> 
> -	return ret;
> -}
> -
> -/*
> - * Read data in admin queue to get msg from pf driver
> - */
> -static enum i40evf_aq_result
> -i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info
> *data) -{
> -	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
> -	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> >dev_private);
> -	struct i40e_arq_event_info event;
> -	int ret;
> -	enum i40evf_aq_result result = I40EVF_MSG_NON;
> -
> -	event.buf_len = data->buf_len;
> -	event.msg_buf = data->msg;
> -	ret = i40e_clean_arq_element(hw, &event, NULL);
> -	/* Can't read any msg from adminQ */
> -	if (ret) {
> -		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
> -			result = I40EVF_MSG_NON;
> -		else
> -			result = I40EVF_MSG_ERR;
> -		return result;
> -	}
> -
> -	/* Parse the event */
> -	result = i40evf_parse_pfmsg(vf, &event, data);
> -
>  	return result;
>  }
> 
> -/*
> - * Polling read until command result return from pf driver or meet error.
> - */
> -static int
> -i40evf_wait_cmd_done(struct rte_eth_dev *dev,
> -		     struct i40evf_arq_msg_info *data)
> -{
> -	int i = 0;
> -	enum i40evf_aq_result ret;
> -
> -#define MAX_TRY_TIMES 20
> -#define ASQ_DELAY_MS  100
> -	do {
> -		/* Delay some time first */
> -		rte_delay_ms(ASQ_DELAY_MS);
> -		ret = i40evf_read_pfmsg(dev, data);
> -		if (ret == I40EVF_MSG_CMD)
> -			return 0;
> -		else if (ret == I40EVF_MSG_ERR)
> -			return -1;
> -
> -		/* If don't read msg or read sys event, continue */
> -	} while(i++ < MAX_TRY_TIMES);
> -
> -	return -1;
> -}
> -
>  /**
>   * clear current command. Only call in case execute
>   * _atomic_set_cmd successfully.
> @@ -380,13 +328,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum
> i40e_virtchnl_ops ops)
>  	return !ret;
>  }
> 
> +#define MAX_TRY_TIMES 20
> +#define ASQ_DELAY_MS  100
> +
>  static int
>  i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
> {
>  	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
>  	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> >dev_private);
> -	int err = -1;
>  	struct i40evf_arq_msg_info info;
> +	enum i40evf_aq_result ret;
> +	int err = -1;
> +	int i = 0;
> 
>  	if (_atomic_set_cmd(vf, args->ops))
>  		return -1;
> @@ -404,19 +357,22 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev,
> struct vf_cmd_info *args)
>  		return err;
>  	}
> 
> -	err = i40evf_wait_cmd_done(dev, &info);
> -	/* read message and it's expected one */
> -	if (!err && args->ops == info.ops)
> -		_clear_cmd(vf);
> -	else if (err) {
> -		PMD_DRV_LOG(ERR, "Failed to read message from
> AdminQ");
> -		_clear_cmd(vf);
> -	}
> -	else if (args->ops != info.ops)
> -		PMD_DRV_LOG(ERR, "command mismatch, expect %u,
> get %u",
> -			    args->ops, info.ops);
> +	do {
> +		/* Delay some time first */
> +		rte_delay_ms(ASQ_DELAY_MS);
It would be better to read first, but not delay first.
In addition, the value of ASG_DELAY_MS shouldn't be 100ms.
It is a bit too long, try 5ms or 10ms, or even shorter, and see if it is good.
I remember that there is a complaint here for the interval.
Note that if ASG_DELAY_MS changed, MAX_TRY_TIMES should also be modified.

> +		ret = i40evf_read_pfmsg(dev, &info);
> +		if (ret == I40EVF_MSG_CMD) {
> +			err = 0;
> +			break;
> +		} else if (ret == I40EVF_MSG_ERR) {
> +			err = -1;
> +			break;
> +		}
> +		/* If don't read msg or read sys event, continue */
> +	} while (i++ < MAX_TRY_TIMES);
> +	_clear_cmd(vf);
> 
> -	return (err | info.result);
> +	return (err | vf->cmd_retval);
>  }
> 
>  /*

Re: [PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Zhang, Helin]

> -----Original Message-----
> From: Wu, Jingjing
> Sent: Wednesday, January 27, 2016 9:50 AM
> To: dev@dpdk.org
> Cc: Wu, Jingjing; Zhang, Helin; Lu, Wenzhuo; Pei, Yulong
> Subject: [PATCH v2 2/2] i40evf: support interrupt based pf reset request
> 
> Interrupt based request of PF reset from PF is supported by enabling the
> adminq event process in VF driver.
> Users can register a callback for this interrupt event to get informed, when a
> PF reset request detected like:
>   rte_eth_dev_callback_register(portid,
> 		RTE_ETH_EVENT_INTR_RESET,
> 		reset_event_callback,
> 		arg);
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> ---
>  doc/guides/rel_notes/release_2_3.rst |   1 +
>  drivers/net/i40e/i40e_ethdev_vf.c    | 274
> +++++++++++++++++++++++++++++++----
>  lib/librte_ether/rte_ethdev.h        |   1 +
>  3 files changed, 246 insertions(+), 30 deletions(-)
> 
> diff --git a/doc/guides/rel_notes/release_2_3.rst
> b/doc/guides/rel_notes/release_2_3.rst
> index 99de186..73d5f76 100644
> --- a/doc/guides/rel_notes/release_2_3.rst
> +++ b/doc/guides/rel_notes/release_2_3.rst
> @@ -4,6 +4,7 @@ DPDK Release 2.3
>  New Features
>  ------------
> 
> +* **Added pf reset event reported in i40e vf PMD driver.
> 
>  Resolved Issues
>  ---------------
> diff --git a/drivers/net/i40e/i40e_ethdev_vf.c
> b/drivers/net/i40e/i40e_ethdev_vf.c
> index 64e6957..1ffe64e 100644
> --- a/drivers/net/i40e/i40e_ethdev_vf.c
> +++ b/drivers/net/i40e/i40e_ethdev_vf.c
> @@ -74,8 +74,6 @@
>  #define I40EVF_BUSY_WAIT_DELAY 10
>  #define I40EVF_BUSY_WAIT_COUNT 50
>  #define MAX_RESET_WAIT_CNT     20
> -/*ITR index for NOITR*/
> -#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
> 
>  struct i40evf_arq_msg_info {
>  	enum i40e_virtchnl_ops ops;
> @@ -151,6 +149,9 @@ static int
>  i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t
> queue_id);  static int  i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev
> *dev, uint16_t queue_id);
> +static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
> +			   uint8_t *msg,
> +			   uint16_t msglen);
> 
>  /* Default hash key buffer for RSS */
>  static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1]; @@ -
> 357,20 +358,42 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct
> vf_cmd_info *args)
>  		return err;
>  	}
> 
> -	do {
> -		/* Delay some time first */
> -		rte_delay_ms(ASQ_DELAY_MS);
> -		ret = i40evf_read_pfmsg(dev, &info);
> -		if (ret == I40EVF_MSG_CMD) {
> -			err = 0;
> -			break;
> -		} else if (ret == I40EVF_MSG_ERR) {
> -			err = -1;
> -			break;
> -		}
> -		/* If don't read msg or read sys event, continue */
> -	} while (i++ < MAX_TRY_TIMES);
> -	_clear_cmd(vf);
> +	switch (args->ops) {
> +	case I40E_VIRTCHNL_OP_RESET_VF:
> +		/*no need to process in this function */
> +		break;
> +	case I40E_VIRTCHNL_OP_VERSION:
> +	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
> +		/* for init adminq commands, need to poll the response */
> +		do {
> +			/* Delay some time first */
> +			rte_delay_ms(ASQ_DELAY_MS);
> +			ret = i40evf_read_pfmsg(dev, &info);
> +			if (ret == I40EVF_MSG_CMD) {
> +				err = 0;
> +				break;
> +			} else if (ret == I40EVF_MSG_ERR) {
> +				err = -1;
> +				break;
> +			}
> +			/* If don't read msg or read sys event, continue */
> +		} while (i++ < MAX_TRY_TIMES);
> +		_clear_cmd(vf);
> +		break;
> +
> +	default:
> +		/* for other adminq in running time, waiting the cmd done
> flag */
> +		do {
> +			/* Delay some time first */
> +			rte_delay_ms(ASQ_DELAY_MS);
> +			if (vf->pend_cmd ==
> I40E_VIRTCHNL_OP_UNKNOWN) {
> +				err = 0;
> +				break;
> +			}
> +			/* If don't read msg or read sys event, continue */
> +		} while (i++ < MAX_TRY_TIMES);
> +		break;
> +	}
> 
>  	return (err | vf->cmd_retval);
>  }
> @@ -719,7 +742,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
> 
>  	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
>  	map_info->num_vectors = 1;
> -	map_info->vecmap[0].rxitr_idx =
> I40E_QINT_RQCTL_MSIX_INDX_NOITR;
> +	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
>  	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
>  	/* Alway use default dynamic MSIX interrupt */
>  	map_info->vecmap[0].vector_id = vector_id; @@ -1093,6 +1116,38
> @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
>  	return 0;
>  }
> 
> +/* Disable IRQ0 */
> +static inline void
> +i40evf_disable_irq0(struct i40e_hw *hw) {
> +	/* Disable all interrupt types */
> +	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
> +	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
> +		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
> +	I40EVF_WRITE_FLUSH(hw);
> +}
> +
> +/* Enable IRQ0 */
> +static inline void
> +i40evf_enable_irq0(struct i40e_hw *hw)
> +{
> +	/* Enable admin queue interrupt trigger */
> +	uint32_t val;
> +
> +	i40evf_disable_irq0(hw);
> +	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
> +	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
> +		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
> +	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
> +
> +	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
> +		I40E_VFINT_DYN_CTL01_INTENA_MASK |
> +		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
> +		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
> +
> +	I40EVF_WRITE_FLUSH(hw);
> +}
> +
>  static int
>  i40evf_reset_vf(struct i40e_hw *hw)
>  {
> @@ -1137,6 +1192,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
>  	int i, err, bufsz;
>  	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data-
> >dev_private);
>  	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data-
> >dev_private);
> +	uint16_t interval =
> +		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
> 
>  	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data-
> >dev_private);
>  	vf->dev_data = dev->data;
> @@ -1218,6 +1275,15 @@ i40evf_init_vf(struct rte_eth_dev *dev)
>  	ether_addr_copy((struct ether_addr *)vf->vsi_res-
> >default_mac_addr,
>  					(struct ether_addr *)hw->mac.addr);
> 
> +	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
> +	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
> +		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
> +			       (I40E_ITR_INDEX_DEFAULT <<
> +				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
> +			       (interval <<
> +				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
> +	}
A write flush might be needed here?

[PATCH v3 0/2] i40evf: pf reset event report

[Jingjing Wu]

v3 changes:
 - commit log doc rewording.
 - rebase on latest dpdk-next-net/rel_16_04 branch.
 - remove few useless line
 - adjust interval and increase times for waiting pf msg

v2 changes:
 - remove the change on vf reset status checking
 - add pf event report support in release note

When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
interrupt request will go via adminq event, VF need be informed, a callback
mechanism is introduced by VF. This will allow VF to invoke callback when
reset happens.
Users can register a callback for this interrupt event like:
    rte_eth_dev_callback_register(portid,
                RTE_ETH_EVENT_INTR_RESET,
                reset_event_callback,
                arg);

Jingjing Wu (2):
  i40evf: allocate virtchnl cmd buffer for each vf
  i40evf: support to report pf reset event

 doc/guides/rel_notes/release_16_04.rst |   1 +
 drivers/net/i40e/i40e_ethdev.h         |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 420 +++++++++++++++++++++++----------
 lib/librte_ether/rte_ethdev.h          |   1 +
 4 files changed, 301 insertions(+), 123 deletions(-)

-- 
2.4.0

[PATCH v3 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Jingjing Wu]

Currently, i40evf PMD uses a global static buffer to send virtchnl
command to host driver. It is shared by multi VFs.
This patch changed to allocate virtchnl cmd buffer for each VF.

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 drivers/net/i40e/i40e_ethdev.h    |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c | 178 +++++++++++++++-----------------------
 2 files changed, 71 insertions(+), 109 deletions(-)

diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index db6173a..93adc49 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -494,7 +494,9 @@ struct i40e_vf {
 	bool link_up;
 	bool vf_reset;
 	volatile uint32_t pend_cmd; /* pending command not finished yet */
+	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	u16 pend_msg; /* flags indicates events from pf not handled yet */
+	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
 
 	/* VSI info */
 	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 13c5b3d..4b15ed6 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -103,9 +103,6 @@ enum i40evf_aq_result {
 	I40EVF_MSG_CMD,      /* Read async command result */
 };
 
-/* A share buffer to store the command result from PF driver */
-static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
-
 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 static int i40evf_dev_start(struct rte_eth_dev *dev);
 static void i40evf_dev_stop(struct rte_eth_dev *dev);
@@ -237,31 +234,37 @@ i40evf_set_mac_type(struct i40e_hw *hw)
 }
 
 /*
- * Parse admin queue message.
- *
- * return value:
- *  < 0: meet error
- *  0: read sys msg
- *  > 0: read cmd result
+ * Read data in admin queue to get msg from pf driver
  */
 static enum i40evf_aq_result
-i40evf_parse_pfmsg(struct i40e_vf *vf,
-		   struct i40e_arq_event_info *event,
-		   struct i40evf_arq_msg_info *data)
+i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 {
-	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
-			rte_le_to_cpu_32(event->desc.cookie_high);
-	enum i40e_status_code retval = (enum i40e_status_code)\
-			rte_le_to_cpu_32(event->desc.cookie_low);
-	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info event;
+	enum i40e_virtchnl_ops opcode;
+	enum i40e_status_code retval;
+	int ret;
+	enum i40evf_aq_result result = I40EVF_MSG_NON;
+
+	event.buf_len = data->buf_len;
+	event.msg_buf = data->msg;
+	ret = i40e_clean_arq_element(hw, &event, NULL);
+	/* Can't read any msg from adminQ */
+	if (ret) {
+		if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
+			result = I40EVF_MSG_ERR;
+		return result;
+	}
 
+	opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
+	retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 	/* pf sys event */
 	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 		struct i40e_virtchnl_pf_event *vpe =
-			(struct i40e_virtchnl_pf_event *)event->msg_buf;
+			(struct i40e_virtchnl_pf_event *)event.msg_buf;
 
-		/* Initialize ret to sys event */
-		ret = I40EVF_MSG_SYS;
+		result = I40EVF_MSG_SYS;
 		switch (vpe->event) {
 		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 			vf->link_up =
@@ -286,74 +289,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
 		}
 	} else {
 		/* async reply msg on command issued by vf previously */
-		ret = I40EVF_MSG_CMD;
+		result = I40EVF_MSG_CMD;
 		/* Actual data length read from PF */
-		data->msg_len = event->msg_len;
+		data->msg_len = event.msg_len;
 	}
-	/* fill the ops and result to notify VF */
+
 	data->result = retval;
 	data->ops = opcode;
 
-	return ret;
-}
-
-/*
- * Read data in admin queue to get msg from pf driver
- */
-static enum i40evf_aq_result
-i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
-{
-	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	struct i40e_arq_event_info event;
-	int ret;
-	enum i40evf_aq_result result = I40EVF_MSG_NON;
-
-	event.buf_len = data->buf_len;
-	event.msg_buf = data->msg;
-	ret = i40e_clean_arq_element(hw, &event, NULL);
-	/* Can't read any msg from adminQ */
-	if (ret) {
-		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
-			result = I40EVF_MSG_NON;
-		else
-			result = I40EVF_MSG_ERR;
-		return result;
-	}
-
-	/* Parse the event */
-	result = i40evf_parse_pfmsg(vf, &event, data);
-
 	return result;
 }
 
-/*
- * Polling read until command result return from pf driver or meet error.
- */
-static int
-i40evf_wait_cmd_done(struct rte_eth_dev *dev,
-		     struct i40evf_arq_msg_info *data)
-{
-	int i = 0;
-	enum i40evf_aq_result ret;
-
-#define MAX_TRY_TIMES 20
-#define ASQ_DELAY_MS  100
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, data);
-		if (ret == I40EVF_MSG_CMD)
-			return 0;
-		else if (ret == I40EVF_MSG_ERR)
-			return -1;
-
-		/* If don't read msg or read sys event, continue */
-	} while(i++ < MAX_TRY_TIMES);
-
-	return -1;
-}
-
 /**
  * clear current command. Only call in case execute
  * _atomic_set_cmd successfully.
@@ -380,13 +326,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 	return !ret;
 }
 
+#define MAX_TRY_TIMES 200
+#define ASQ_DELAY_MS  10
+
 static int
 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	int err = -1;
 	struct i40evf_arq_msg_info info;
+	enum i40evf_aq_result ret;
+	int err = -1;
+	int i = 0;
 
 	if (_atomic_set_cmd(vf, args->ops))
 		return -1;
@@ -404,19 +355,21 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	err = i40evf_wait_cmd_done(dev, &info);
-	/* read message and it's expected one */
-	if (!err && args->ops == info.ops)
-		_clear_cmd(vf);
-	else if (err) {
-		PMD_DRV_LOG(ERR, "Failed to read message from AdminQ");
-		_clear_cmd(vf);
-	}
-	else if (args->ops != info.ops)
-		PMD_DRV_LOG(ERR, "command mismatch, expect %u, get %u",
-			    args->ops, info.ops);
+	do {
+		ret = i40evf_read_pfmsg(dev, &info);
+		if (ret == I40EVF_MSG_CMD) {
+			err = 0;
+			break;
+		} else if (ret == I40EVF_MSG_ERR) {
+			err = -1;
+			break;
+		}
+		rte_delay_ms(ASQ_DELAY_MS);
+		/* If don't read msg or read sys event, continue */
+	} while (i++ < MAX_TRY_TIMES);
+	_clear_cmd(vf);
 
-	return err | info.result;
+	return err | vf->cmd_retval;
 }
 
 /*
@@ -436,7 +389,7 @@ i40evf_check_api_version(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_VERSION;
 	args.in_args = (uint8_t *)&version;
 	args.in_args_size = sizeof(version);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -474,7 +427,7 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev)
 	uint32_t caps, len;
 
 	args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	if (PF_IS_V11(vf)) {
 		caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
@@ -527,7 +480,7 @@ i40evf_config_promisc(struct rte_eth_dev *dev,
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 	args.in_args = (uint8_t *)&promisc;
 	args.in_args_size = sizeof(promisc);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -554,7 +507,7 @@ i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 	args.in_args = (uint8_t *)&offload;
 	args.in_args_size = sizeof(offload);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -585,7 +538,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 	args.in_args = (uint8_t *)&tpid_info;
 	args.in_args_size = sizeof(tpid_info);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -664,7 +617,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 	args.in_args = (uint8_t *)vc_vqci;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -717,7 +670,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 		(enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 	args.in_args = (uint8_t *)vc_vqcei;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -779,7 +732,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 	args.in_args = (u8 *)cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -810,7 +763,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 		args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 	args.in_args = (u8 *)&queue_select;
 	args.in_args_size = sizeof(queue_select);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -902,7 +855,7 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -939,7 +892,7 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -962,7 +915,7 @@ i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 	args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 	args.in_args = (u8 *)&q_stats;
 	args.in_args_size = sizeof(q_stats);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -1056,7 +1009,7 @@ i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1083,7 +1036,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1095,6 +1048,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 static int
 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 {
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	int err;
 	struct vf_cmd_info args;
 	struct rte_eth_link *new_link;
@@ -1102,7 +1056,7 @@ i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
 	args.in_args = NULL;
 	args.in_args_size = 0;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err) {
@@ -1196,7 +1150,6 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-
 	/* Reset VF and wait until it's complete */
 	if (i40evf_reset_vf(hw)) {
 		PMD_INIT_LOG(ERR, "reset NIC failed");
@@ -1214,6 +1167,11 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "init_adminq failed");
 		return -1;
 	}
+	vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
+	if (!vf->aq_resp) {
+		PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
+			goto err_aq;
+	}
 	if (i40evf_check_api_version(dev) != 0) {
 		PMD_INIT_LOG(ERR, "check_api version failed");
 		goto err_aq;
@@ -1279,6 +1237,8 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 		i40evf_dev_close(dev);
 	rte_free(vf->vf_res);
 	vf->vf_res = NULL;
+	rte_free(vf->aq_resp);
+	vf->aq_resp = NULL;
 
 	return 0;
 }
-- 
2.4.0

[PATCH v3 2/2] i40evf: support to report pf reset event

[Jingjing Wu]

When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
interrupt will go via adminq event, VF need be informed the event,
a callback mechanism is introduced by VF. This will allow VF to
invoke callback when reset happens.
Users can register a callback for this interrupt event like:
  rte_eth_dev_callback_register(portid,
		RTE_ETH_EVENT_INTR_RESET,
		reset_event_callback,
		arg);

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 doc/guides/rel_notes/release_16_04.rst |   1 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 272 +++++++++++++++++++++++++++++----
 lib/librte_ether/rte_ethdev.h          |   1 +
 3 files changed, 245 insertions(+), 29 deletions(-)

diff --git a/doc/guides/rel_notes/release_16_04.rst b/doc/guides/rel_notes/release_16_04.rst
index 3036f7d..596b6e2 100644
--- a/doc/guides/rel_notes/release_16_04.rst
+++ b/doc/guides/rel_notes/release_16_04.rst
@@ -70,6 +70,7 @@ This section should contain new features added in this release. Sample format:
 
 * **szedata2: Add functions for setting link up/down.**
 
+* **Added pf reset event reported in i40e vf PMD driver.
 
 Resolved Issues
 ---------------
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 4b15ed6..b6e360b 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -74,8 +74,6 @@
 #define I40EVF_BUSY_WAIT_DELAY 10
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
-/*ITR index for NOITR*/
-#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
 
 struct i40evf_arq_msg_info {
 	enum i40e_virtchnl_ops ops;
@@ -151,6 +149,9 @@ static int
 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 static int
 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   uint16_t msglen);
 
 /* Default hash key buffer for RSS */
 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
@@ -355,19 +356,40 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	do {
-		ret = i40evf_read_pfmsg(dev, &info);
-		if (ret == I40EVF_MSG_CMD) {
-			err = 0;
-			break;
-		} else if (ret == I40EVF_MSG_ERR) {
-			err = -1;
-			break;
-		}
-		rte_delay_ms(ASQ_DELAY_MS);
-		/* If don't read msg or read sys event, continue */
-	} while (i++ < MAX_TRY_TIMES);
-	_clear_cmd(vf);
+	switch (args->ops) {
+	case I40E_VIRTCHNL_OP_RESET_VF:
+		/*no need to process in this function */
+		break;
+	case I40E_VIRTCHNL_OP_VERSION:
+	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
+		/* for init adminq commands, need to poll the response */
+		do {
+			ret = i40evf_read_pfmsg(dev, &info);
+			if (ret == I40EVF_MSG_CMD) {
+				err = 0;
+				break;
+			} else if (ret == I40EVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		_clear_cmd(vf);
+		break;
+
+	default:
+		/* for other adminq in running time, waiting the cmd done flag */
+		do {
+			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
+				err = 0;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		break;
+	}
 
 	return err | vf->cmd_retval;
 }
@@ -716,7 +738,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 
 	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = 1;
-	map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
+	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 	/* Alway use default dynamic MSIX interrupt */
 	map_info->vecmap[0].vector_id = vector_id;
@@ -1090,6 +1112,38 @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/* Disable IRQ0 */
+static inline void
+i40evf_disable_irq0(struct i40e_hw *hw)
+{
+	/* Disable all interrupt types */
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	I40EVF_WRITE_FLUSH(hw);
+}
+
+/* Enable IRQ0 */
+static inline void
+i40evf_enable_irq0(struct i40e_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	uint32_t val;
+
+	i40evf_disable_irq0(hw);
+	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
+	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
+		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
+
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		I40E_VFINT_DYN_CTL01_INTENA_MASK |
+		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	I40EVF_WRITE_FLUSH(hw);
+}
+
 static int
 i40evf_reset_vf(struct i40e_hw *hw)
 {
@@ -1134,6 +1188,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	int i, err, bufsz;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	uint16_t interval =
+		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1215,6 +1271,16 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	ether_addr_copy((struct ether_addr *)vf->vsi_res->default_mac_addr,
 					(struct ether_addr *)hw->mac.addr);
 
+	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
+	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       (I40E_ITR_INDEX_DEFAULT <<
+				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
+			       (interval <<
+				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
+		I40EVF_WRITE_FLUSH(hw);
+	}
+
 	return 0;
 
 err_alloc:
@@ -1243,11 +1309,142 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   __rte_unused uint16_t msglen)
+{
+	struct i40e_virtchnl_pf_event *pf_msg =
+			(struct i40e_virtchnl_pf_event *)msg;
+
+	switch (pf_msg->event) {
+	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
+		break;
+	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
+		break;
+	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
+		break;
+	default:
+		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
+		break;
+	}
+}
+
+static void
+i40evf_handle_aq_msg(struct rte_eth_dev *dev)
+{
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info info;
+	struct i40e_virtchnl_msg *v_msg;
+	uint16_t pending, opcode;
+	int ret;
+
+	info.buf_len = I40E_AQ_BUF_SZ;
+	if (!vf->aq_resp) {
+		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
+		return;
+	}
+	info.msg_buf = vf->aq_resp;
+	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
+
+	pending = 1;
+	while (pending) {
+		ret = i40e_clean_arq_element(hw, &info, &pending);
+
+		if (ret != I40E_SUCCESS) {
+			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
+				    "ret: %d", ret);
+			break;
+		}
+		opcode = rte_le_to_cpu_16(info.desc.opcode);
+
+		switch (opcode) {
+		case i40e_aqc_opc_send_msg_to_vf:
+			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
+				/* process event*/
+				i40evf_handle_pf_event(dev, info.msg_buf,
+							info.msg_len);
+			else {
+				/* read message and it's expected one */
+				if (v_msg->v_opcode == vf->pend_cmd) {
+					vf->cmd_retval = v_msg->v_retval;
+					/* prevent compiler reordering */
+					rte_compiler_barrier();
+					_clear_cmd(vf);
+				} else
+					PMD_DRV_LOG(ERR, "command mismatch,"
+						"expect %u, get %u",
+						vf->pend_cmd, v_msg->v_opcode);
+				PMD_DRV_LOG(DEBUG, "adminq response is received,"
+					     " opcode = %d\n", v_msg->v_opcode);
+			}
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
+				    opcode);
+			break;
+		}
+	}
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt. Only adminq interrupt is processed in VF.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
+			   void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	i40evf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
+		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
+		goto done;
+	}
+
+	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
+		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
+		i40evf_handle_aq_msg(dev);
+	}
+
+	/* Link Status Change interrupt */
+	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
+		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
+				   " do nothing\n");
+
+done:
+	i40evf_enable_irq0(hw);
+	rte_intr_enable(&(dev->pci_dev->intr_handle));
+}
+
+
 static int
 i40evf_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
 			eth_dev->data->dev_private);
+	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1282,6 +1479,16 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
+	/* register callback func to eal lib */
+	rte_intr_callback_register(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)eth_dev);
+
+	/* enable uio intr after callback register */
+	rte_intr_enable(&(pci_dev->intr_handle));
+
+	/* configure and enable device interrupt */
+	i40evf_enable_irq0(hw);
+
 	/* copy mac addr */
 	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
 					ETHER_ADDR_LEN, 0);
@@ -1659,7 +1866,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 		I40E_WRITE_REG(hw,
 			       I40E_VFINT_DYN_CTL01,
 			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
-			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1670,11 +1878,10 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
 			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
-	else
-		/* To support Linux PF host */
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
-				I40E_VFINT_DYN_CTL01_INTENA_MASK |
-				I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't set I40E_VFINT_DYN_CTL01,
+	 * because it is already done in i40evf_enable_irq0.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1687,7 +1894,8 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 
 	if (!rte_intr_allow_others(intr_handle)) {
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1697,8 +1905,10 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
 						    - 1),
 			       0);
-	else
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't zero I40E_VFINT_DYN_CTL01,
+	 * because interrupt 0 is also used for adminq processing.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1822,10 +2032,6 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* vf don't allow intr except for rxq intr */
-	if (dev->data->dev_conf.intr_conf.rxq != 0)
-		rte_intr_enable(intr_handle);
-
 	i40evf_enable_queues_intr(dev);
 	return 0;
 
@@ -2017,12 +2223,20 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = dev->pci_dev;
 
 	i40evf_dev_stop(dev);
 	hw->adapter_stopped = 1;
 	i40e_dev_free_queues(dev);
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(&(pci_dev->intr_handle));
+
+	/* unregister callback func from eal lib */
+	rte_intr_callback_unregister(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)dev);
+	i40evf_disable_irq0(hw);
 }
 
 static int
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index 16da821..cdff3b7 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -2661,6 +2661,7 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
 enum rte_eth_event_type {
 	RTE_ETH_EVENT_UNKNOWN,  /**< unknown event type */
 	RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
+	RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.4.0

Re: [PATCH v3 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Zhang, Helin]

> -----Original Message-----
> From: Wu, Jingjing
> Sent: Friday, February 26, 2016 2:52 PM
> To: Richardson, Bruce
> Cc: dev@dpdk.org; Wu, Jingjing; Zhang, Helin; Tao, Zhe
> Subject: [PATCH v3 1/2] i40evf: allocate virtchnl cmd buffer for each vf
> 
> Currently, i40evf PMD uses a global static buffer to send virtchnl command to
> host driver. It is shared by multi VFs.
> This patch changed to allocate virtchnl cmd buffer for each VF.
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>

Re: [PATCH v3 2/2] i40evf: support to report pf reset event

[Zhang, Helin]

> -----Original Message-----
> From: Wu, Jingjing
> Sent: Friday, February 26, 2016 2:52 PM
> To: Richardson, Bruce
> Cc: dev@dpdk.org; Wu, Jingjing; Zhang, Helin; Tao, Zhe
> Subject: [PATCH v3 2/2] i40evf: support to report pf reset event
> 
> When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
> interrupt will go via adminq event, VF need be informed the event, a callback
> mechanism is introduced by VF. This will allow VF to invoke callback when reset
> happens.
> Users can register a callback for this interrupt event like:
>   rte_eth_dev_callback_register(portid,
> 		RTE_ETH_EVENT_INTR_RESET,
> 		reset_event_callback,
> 		arg);
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>

Re: [PATCH v3 0/2] i40evf: pf reset event report

[Bruce Richardson]

On Fri, Feb 26, 2016 at 02:51:52PM +0800, Jingjing Wu wrote:
> v3 changes:
>  - commit log doc rewording.
>  - rebase on latest dpdk-next-net/rel_16_04 branch.
>  - remove few useless line
>  - adjust interval and increase times for waiting pf msg
> 
> v2 changes:
>  - remove the change on vf reset status checking
>  - add pf event report support in release note
> 
> When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
> interrupt request will go via adminq event, VF need be informed, a callback
> mechanism is introduced by VF. This will allow VF to invoke callback when
> reset happens.
> Users can register a callback for this interrupt event like:
>     rte_eth_dev_callback_register(portid,
>                 RTE_ETH_EVENT_INTR_RESET,
>                 reset_event_callback,
>                 arg);
> 
> Jingjing Wu (2):
>   i40evf: allocate virtchnl cmd buffer for each vf
>   i40evf: support to report pf reset event
> 
>  doc/guides/rel_notes/release_16_04.rst |   1 +
>  drivers/net/i40e/i40e_ethdev.h         |   2 +
>  drivers/net/i40e/i40e_ethdev_vf.c      | 420 +++++++++++++++++++++++----------
>  lib/librte_ether/rte_ethdev.h          |   1 +
>  4 files changed, 301 insertions(+), 123 deletions(-)

Hi Jingjing,

I'm planning on merging this patch set into dpdk-next-net, but it doesn't apply
cleanly. Can you perhaps rebase it on top of dpdk-next-net/rel_16_04 branch, and
do a V4 keeping Helin's ack on each patch.

Thanks,
/Bruce

Re: [PATCH v3 0/2] i40evf: pf reset event report

[Zhe Tao]

On Fri, Feb 26, 2016 at 02:51:52PM +0800, Jingjing Wu wrote:
> v3 changes:
>  - commit log doc rewording.
>  - rebase on latest dpdk-next-net/rel_16_04 branch.
>  - remove few useless line
>  - adjust interval and increase times for waiting pf msg
> 
> v2 changes:
>  - remove the change on vf reset status checking
>  - add pf event report support in release note
> 
> When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
> interrupt request will go via adminq event, VF need be informed, a callback
> mechanism is introduced by VF. This will allow VF to invoke callback when
> reset happens.
> Users can register a callback for this interrupt event like:
>     rte_eth_dev_callback_register(portid,
>                 RTE_ETH_EVENT_INTR_RESET,
>                 reset_event_callback,
>                 arg);
> 
> Jingjing Wu (2):
>   i40evf: allocate virtchnl cmd buffer for each vf
>   i40evf: support to report pf reset event
> 
>  doc/guides/rel_notes/release_16_04.rst |   1 +
>  drivers/net/i40e/i40e_ethdev.h         |   2 +
>  drivers/net/i40e/i40e_ethdev_vf.c      | 420 +++++++++++++++++++++++----------
>  lib/librte_ether/rte_ethdev.h          |   1 +
>  4 files changed, 301 insertions(+), 123 deletions(-)
> 
> -- 
> 2.4.0
Acked-by: Zhe Tao <zhe.tao@intel.com>

[PATCH v4 0/2] i40evf: pf reset event report

[Jingjing Wu]

v4 changes:
 - rebase on latest dpdk-next-net/rel_16_04 branch (commit 0f9564a0e4f2)

v3 changes:
 - commit log doc rewording.
 - rebase on latest dpdk-next-net/rel_16_04 branch.
 - remove few useless line.
 - adjust interval and increase times for waiting pf msg

v2 changes:
 - remove the change on vf reset status checking
 - add pf event report support in release note

When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset, 
interrupt request will go via adminq event, VF need be informed, a 
callback mechanism is introduced by VF. This will allow VF to invoke 
callback when reset happens.
Users can register a callback for this interrupt event like:
    rte_eth_dev_callback_register(portid,
                 RTE_ETH_EVENT_INTR_RESET,
                 reset_event_callback,
                 arg); 

Jingjing Wu (2):
  i40evf: allocate virtchnl cmd buffer for each vf
  i40evf: support to report pf reset event

 doc/guides/rel_notes/release_16_04.rst |   2 +
 drivers/net/i40e/i40e_ethdev.h         |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 420 +++++++++++++++++++++++----------
 lib/librte_ether/rte_ethdev.h          |   1 +
 4 files changed, 302 insertions(+), 123 deletions(-)

-- 
2.4.0

[PATCH v4 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Jingjing Wu]

Currently, i40evf PMD uses a global static buffer to send virtchnl
command to host driver. It is shared by multi VFs.
This patch changed to allocate virtchnl cmd buffer for each VF.

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>
---
 drivers/net/i40e/i40e_ethdev.h    |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c | 178 +++++++++++++++-----------------------
 2 files changed, 71 insertions(+), 109 deletions(-)

diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index a9b805e..8cd6126 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -497,7 +497,9 @@ struct i40e_vf {
 	bool link_up;
 	bool vf_reset;
 	volatile uint32_t pend_cmd; /* pending command not finished yet */
+	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	u16 pend_msg; /* flags indicates events from pf not handled yet */
+	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
 
 	/* VSI info */
 	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 6185ee8..90edeb5 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -103,9 +103,6 @@ enum i40evf_aq_result {
 	I40EVF_MSG_CMD,      /* Read async command result */
 };
 
-/* A share buffer to store the command result from PF driver */
-static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
-
 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 static int i40evf_dev_start(struct rte_eth_dev *dev);
 static void i40evf_dev_stop(struct rte_eth_dev *dev);
@@ -217,31 +214,37 @@ static const struct eth_dev_ops i40evf_eth_dev_ops = {
 };
 
 /*
- * Parse admin queue message.
- *
- * return value:
- *  < 0: meet error
- *  0: read sys msg
- *  > 0: read cmd result
+ * Read data in admin queue to get msg from pf driver
  */
 static enum i40evf_aq_result
-i40evf_parse_pfmsg(struct i40e_vf *vf,
-		   struct i40e_arq_event_info *event,
-		   struct i40evf_arq_msg_info *data)
+i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 {
-	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
-			rte_le_to_cpu_32(event->desc.cookie_high);
-	enum i40e_status_code retval = (enum i40e_status_code)\
-			rte_le_to_cpu_32(event->desc.cookie_low);
-	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info event;
+	enum i40e_virtchnl_ops opcode;
+	enum i40e_status_code retval;
+	int ret;
+	enum i40evf_aq_result result = I40EVF_MSG_NON;
+
+	event.buf_len = data->buf_len;
+	event.msg_buf = data->msg;
+	ret = i40e_clean_arq_element(hw, &event, NULL);
+	/* Can't read any msg from adminQ */
+	if (ret) {
+		if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
+			result = I40EVF_MSG_ERR;
+		return result;
+	}
 
+	opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
+	retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 	/* pf sys event */
 	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 		struct i40e_virtchnl_pf_event *vpe =
-			(struct i40e_virtchnl_pf_event *)event->msg_buf;
+			(struct i40e_virtchnl_pf_event *)event.msg_buf;
 
-		/* Initialize ret to sys event */
-		ret = I40EVF_MSG_SYS;
+		result = I40EVF_MSG_SYS;
 		switch (vpe->event) {
 		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 			vf->link_up =
@@ -266,74 +269,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
 		}
 	} else {
 		/* async reply msg on command issued by vf previously */
-		ret = I40EVF_MSG_CMD;
+		result = I40EVF_MSG_CMD;
 		/* Actual data length read from PF */
-		data->msg_len = event->msg_len;
+		data->msg_len = event.msg_len;
 	}
-	/* fill the ops and result to notify VF */
+
 	data->result = retval;
 	data->ops = opcode;
 
-	return ret;
-}
-
-/*
- * Read data in admin queue to get msg from pf driver
- */
-static enum i40evf_aq_result
-i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
-{
-	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	struct i40e_arq_event_info event;
-	int ret;
-	enum i40evf_aq_result result = I40EVF_MSG_NON;
-
-	event.buf_len = data->buf_len;
-	event.msg_buf = data->msg;
-	ret = i40e_clean_arq_element(hw, &event, NULL);
-	/* Can't read any msg from adminQ */
-	if (ret) {
-		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
-			result = I40EVF_MSG_NON;
-		else
-			result = I40EVF_MSG_ERR;
-		return result;
-	}
-
-	/* Parse the event */
-	result = i40evf_parse_pfmsg(vf, &event, data);
-
 	return result;
 }
 
-/*
- * Polling read until command result return from pf driver or meet error.
- */
-static int
-i40evf_wait_cmd_done(struct rte_eth_dev *dev,
-		     struct i40evf_arq_msg_info *data)
-{
-	int i = 0;
-	enum i40evf_aq_result ret;
-
-#define MAX_TRY_TIMES 20
-#define ASQ_DELAY_MS  100
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, data);
-		if (ret == I40EVF_MSG_CMD)
-			return 0;
-		else if (ret == I40EVF_MSG_ERR)
-			return -1;
-
-		/* If don't read msg or read sys event, continue */
-	} while(i++ < MAX_TRY_TIMES);
-
-	return -1;
-}
-
 /**
  * clear current command. Only call in case execute
  * _atomic_set_cmd successfully.
@@ -360,13 +306,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 	return !ret;
 }
 
+#define MAX_TRY_TIMES 200
+#define ASQ_DELAY_MS  10
+
 static int
 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	int err = -1;
 	struct i40evf_arq_msg_info info;
+	enum i40evf_aq_result ret;
+	int err = -1;
+	int i = 0;
 
 	if (_atomic_set_cmd(vf, args->ops))
 		return -1;
@@ -384,19 +335,21 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	err = i40evf_wait_cmd_done(dev, &info);
-	/* read message and it's expected one */
-	if (!err && args->ops == info.ops)
-		_clear_cmd(vf);
-	else if (err) {
-		PMD_DRV_LOG(ERR, "Failed to read message from AdminQ");
-		_clear_cmd(vf);
-	}
-	else if (args->ops != info.ops)
-		PMD_DRV_LOG(ERR, "command mismatch, expect %u, get %u",
-			    args->ops, info.ops);
+	do {
+		ret = i40evf_read_pfmsg(dev, &info);
+		if (ret == I40EVF_MSG_CMD) {
+			err = 0;
+			break;
+		} else if (ret == I40EVF_MSG_ERR) {
+			err = -1;
+			break;
+		}
+		rte_delay_ms(ASQ_DELAY_MS);
+		/* If don't read msg or read sys event, continue */
+	} while (i++ < MAX_TRY_TIMES);
+	_clear_cmd(vf);
 
-	return err | info.result;
+	return err | vf->cmd_retval;
 }
 
 /*
@@ -416,7 +369,7 @@ i40evf_check_api_version(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_VERSION;
 	args.in_args = (uint8_t *)&version;
 	args.in_args_size = sizeof(version);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -454,7 +407,7 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev)
 	uint32_t caps, len;
 
 	args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	if (PF_IS_V11(vf)) {
 		caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
@@ -507,7 +460,7 @@ i40evf_config_promisc(struct rte_eth_dev *dev,
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 	args.in_args = (uint8_t *)&promisc;
 	args.in_args_size = sizeof(promisc);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -534,7 +487,7 @@ i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 	args.in_args = (uint8_t *)&offload;
 	args.in_args_size = sizeof(offload);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -565,7 +518,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 	args.in_args = (uint8_t *)&tpid_info;
 	args.in_args_size = sizeof(tpid_info);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -644,7 +597,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 	args.in_args = (uint8_t *)vc_vqci;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -697,7 +650,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 		(enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 	args.in_args = (uint8_t *)vc_vqcei;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -759,7 +712,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 	args.in_args = (u8 *)cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -790,7 +743,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 		args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 	args.in_args = (u8 *)&queue_select;
 	args.in_args_size = sizeof(queue_select);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -882,7 +835,7 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -919,7 +872,7 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -942,7 +895,7 @@ i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 	args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 	args.in_args = (u8 *)&q_stats;
 	args.in_args_size = sizeof(q_stats);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -1036,7 +989,7 @@ i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1063,7 +1016,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1075,6 +1028,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 static int
 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 {
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	int err;
 	struct vf_cmd_info args;
 	struct rte_eth_link *new_link;
@@ -1082,7 +1036,7 @@ i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
 	args.in_args = NULL;
 	args.in_args_size = 0;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err) {
@@ -1177,7 +1131,6 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-
 	/* Reset VF and wait until it's complete */
 	if (i40evf_reset_vf(hw)) {
 		PMD_INIT_LOG(ERR, "reset NIC failed");
@@ -1195,6 +1148,11 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "init_adminq failed");
 		return -1;
 	}
+	vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
+	if (!vf->aq_resp) {
+		PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
+			goto err_aq;
+	}
 	if (i40evf_check_api_version(dev) != 0) {
 		PMD_INIT_LOG(ERR, "check_api version failed");
 		goto err_aq;
@@ -1259,6 +1217,8 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 		i40evf_dev_close(dev);
 	rte_free(vf->vf_res);
 	vf->vf_res = NULL;
+	rte_free(vf->aq_resp);
+	vf->aq_resp = NULL;
 
 	return 0;
 }
-- 
2.4.0

[PATCH v4 2/2] i40evf: support to report pf reset event

[Jingjing Wu]

When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
interrupt will go via adminq event, VF need be informed the event,
a callback mechanism is introduced by VF. This will allow VF to
invoke callback when reset happens.
Users can register a callback for this interrupt event like:
  rte_eth_dev_callback_register(portid,
		RTE_ETH_EVENT_INTR_RESET,
		reset_event_callback,
		arg);

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>
---
 doc/guides/rel_notes/release_16_04.rst |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 272 +++++++++++++++++++++++++++++----
 lib/librte_ether/rte_ethdev.h          |   1 +
 3 files changed, 246 insertions(+), 29 deletions(-)

diff --git a/doc/guides/rel_notes/release_16_04.rst b/doc/guides/rel_notes/release_16_04.rst
index eab5f92..a872d40 100644
--- a/doc/guides/rel_notes/release_16_04.rst
+++ b/doc/guides/rel_notes/release_16_04.rst
@@ -105,6 +105,8 @@ This section should contain new features added in this release. Sample format:
   be down.
   We added the support of auto-neg by SW to avoid this link down issue.
 
+* **Added pf reset event reported in i40e vf PMD driver.
+
 
 Resolved Issues
 ---------------
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 90edeb5..ede4bc2 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -74,8 +74,6 @@
 #define I40EVF_BUSY_WAIT_DELAY 10
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
-/*ITR index for NOITR*/
-#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
 
 struct i40evf_arq_msg_info {
 	enum i40e_virtchnl_ops ops;
@@ -151,6 +149,9 @@ static int
 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 static int
 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   uint16_t msglen);
 
 /* Default hash key buffer for RSS */
 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
@@ -335,19 +336,40 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	do {
-		ret = i40evf_read_pfmsg(dev, &info);
-		if (ret == I40EVF_MSG_CMD) {
-			err = 0;
-			break;
-		} else if (ret == I40EVF_MSG_ERR) {
-			err = -1;
-			break;
-		}
-		rte_delay_ms(ASQ_DELAY_MS);
-		/* If don't read msg or read sys event, continue */
-	} while (i++ < MAX_TRY_TIMES);
-	_clear_cmd(vf);
+	switch (args->ops) {
+	case I40E_VIRTCHNL_OP_RESET_VF:
+		/*no need to process in this function */
+		break;
+	case I40E_VIRTCHNL_OP_VERSION:
+	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
+		/* for init adminq commands, need to poll the response */
+		do {
+			ret = i40evf_read_pfmsg(dev, &info);
+			if (ret == I40EVF_MSG_CMD) {
+				err = 0;
+				break;
+			} else if (ret == I40EVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		_clear_cmd(vf);
+		break;
+
+	default:
+		/* for other adminq in running time, waiting the cmd done flag */
+		do {
+			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
+				err = 0;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		break;
+	}
 
 	return err | vf->cmd_retval;
 }
@@ -696,7 +718,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 
 	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = 1;
-	map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
+	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 	/* Alway use default dynamic MSIX interrupt */
 	map_info->vecmap[0].vector_id = vector_id;
@@ -1070,6 +1092,38 @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/* Disable IRQ0 */
+static inline void
+i40evf_disable_irq0(struct i40e_hw *hw)
+{
+	/* Disable all interrupt types */
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	I40EVF_WRITE_FLUSH(hw);
+}
+
+/* Enable IRQ0 */
+static inline void
+i40evf_enable_irq0(struct i40e_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	uint32_t val;
+
+	i40evf_disable_irq0(hw);
+	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
+	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
+		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
+
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		I40E_VFINT_DYN_CTL01_INTENA_MASK |
+		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	I40EVF_WRITE_FLUSH(hw);
+}
+
 static int
 i40evf_reset_vf(struct i40e_hw *hw)
 {
@@ -1115,6 +1169,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct ether_addr *p_mac_addr;
+	uint16_t interval =
+		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1195,6 +1251,16 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	else
 		eth_random_addr(hw->mac.addr); /* Generate a random one */
 
+	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
+	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       (I40E_ITR_INDEX_DEFAULT <<
+				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
+			       (interval <<
+				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
+		I40EVF_WRITE_FLUSH(hw);
+	}
+
 	return 0;
 
 err_alloc:
@@ -1223,11 +1289,142 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   __rte_unused uint16_t msglen)
+{
+	struct i40e_virtchnl_pf_event *pf_msg =
+			(struct i40e_virtchnl_pf_event *)msg;
+
+	switch (pf_msg->event) {
+	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
+		break;
+	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
+		break;
+	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
+		break;
+	default:
+		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
+		break;
+	}
+}
+
+static void
+i40evf_handle_aq_msg(struct rte_eth_dev *dev)
+{
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info info;
+	struct i40e_virtchnl_msg *v_msg;
+	uint16_t pending, opcode;
+	int ret;
+
+	info.buf_len = I40E_AQ_BUF_SZ;
+	if (!vf->aq_resp) {
+		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
+		return;
+	}
+	info.msg_buf = vf->aq_resp;
+	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
+
+	pending = 1;
+	while (pending) {
+		ret = i40e_clean_arq_element(hw, &info, &pending);
+
+		if (ret != I40E_SUCCESS) {
+			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
+				    "ret: %d", ret);
+			break;
+		}
+		opcode = rte_le_to_cpu_16(info.desc.opcode);
+
+		switch (opcode) {
+		case i40e_aqc_opc_send_msg_to_vf:
+			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
+				/* process event*/
+				i40evf_handle_pf_event(dev, info.msg_buf,
+							info.msg_len);
+			else {
+				/* read message and it's expected one */
+				if (v_msg->v_opcode == vf->pend_cmd) {
+					vf->cmd_retval = v_msg->v_retval;
+					/* prevent compiler reordering */
+					rte_compiler_barrier();
+					_clear_cmd(vf);
+				} else
+					PMD_DRV_LOG(ERR, "command mismatch,"
+						"expect %u, get %u",
+						vf->pend_cmd, v_msg->v_opcode);
+				PMD_DRV_LOG(DEBUG, "adminq response is received,"
+					     " opcode = %d\n", v_msg->v_opcode);
+			}
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
+				    opcode);
+			break;
+		}
+	}
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt. Only adminq interrupt is processed in VF.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
+			   void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	i40evf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
+		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
+		goto done;
+	}
+
+	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
+		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
+		i40evf_handle_aq_msg(dev);
+	}
+
+	/* Link Status Change interrupt */
+	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
+		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
+				   " do nothing\n");
+
+done:
+	i40evf_enable_irq0(hw);
+	rte_intr_enable(&(dev->pci_dev->intr_handle));
+}
+
+
 static int
 i40evf_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
 			eth_dev->data->dev_private);
+	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1262,6 +1459,16 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
+	/* register callback func to eal lib */
+	rte_intr_callback_register(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)eth_dev);
+
+	/* enable uio intr after callback register */
+	rte_intr_enable(&(pci_dev->intr_handle));
+
+	/* configure and enable device interrupt */
+	i40evf_enable_irq0(hw);
+
 	/* copy mac addr */
 	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
 					ETHER_ADDR_LEN, 0);
@@ -1639,7 +1846,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 		I40E_WRITE_REG(hw,
 			       I40E_VFINT_DYN_CTL01,
 			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
-			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1650,11 +1858,10 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
 			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
-	else
-		/* To support Linux PF host */
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
-				I40E_VFINT_DYN_CTL01_INTENA_MASK |
-				I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't set I40E_VFINT_DYN_CTL01,
+	 * because it is already done in i40evf_enable_irq0.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1667,7 +1874,8 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 
 	if (!rte_intr_allow_others(intr_handle)) {
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1677,8 +1885,10 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
 						    - 1),
 			       0);
-	else
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't zero I40E_VFINT_DYN_CTL01,
+	 * because interrupt 0 is also used for adminq processing.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1802,10 +2012,6 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* vf don't allow intr except for rxq intr */
-	if (dev->data->dev_conf.intr_conf.rxq != 0)
-		rte_intr_enable(intr_handle);
-
 	i40evf_enable_queues_intr(dev);
 	return 0;
 
@@ -1997,12 +2203,20 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = dev->pci_dev;
 
 	i40evf_dev_stop(dev);
 	hw->adapter_stopped = 1;
 	i40e_dev_free_queues(dev);
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(&(pci_dev->intr_handle));
+
+	/* unregister callback func from eal lib */
+	rte_intr_callback_unregister(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)dev);
+	i40evf_disable_irq0(hw);
 }
 
 static int
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index e2893ba..0c81435 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -2663,6 +2663,7 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
 enum rte_eth_event_type {
 	RTE_ETH_EVENT_UNKNOWN,  /**< unknown event type */
 	RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
+	RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.4.0

Re: [PATCH v4 2/2] i40evf: support to report pf reset event

[Thomas Monjalon]

2016-03-09 14:00, Jingjing Wu:
> When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
> interrupt will go via adminq event, VF need be informed the event,
> a callback mechanism is introduced by VF. This will allow VF to
> invoke callback when reset happens.
> Users can register a callback for this interrupt event like:
>   rte_eth_dev_callback_register(portid,
> 		RTE_ETH_EVENT_INTR_RESET,
> 		reset_event_callback,
> 		arg);
[...]
> --- a/doc/guides/rel_notes/release_16_04.rst
> +++ b/doc/guides/rel_notes/release_16_04.rst
> @@ -105,6 +105,8 @@ This section should contain new features added in this release. Sample format:
>    be down.
>    We added the support of auto-neg by SW to avoid this link down issue.
>  
> +* **Added pf reset event reported in i40e vf PMD driver.

Bruce, is this comment clear enough for an user?

Beware of the wrong formating (missing **)

Re: [PATCH v4 2/2] i40evf: support to report pf reset event

[Bruce Richardson]

On Wed, Mar 09, 2016 at 10:59:56AM +0100, Thomas Monjalon wrote:
> 2016-03-09 14:00, Jingjing Wu:
> > When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
> > interrupt will go via adminq event, VF need be informed the event,
> > a callback mechanism is introduced by VF. This will allow VF to
> > invoke callback when reset happens.
> > Users can register a callback for this interrupt event like:
> >   rte_eth_dev_callback_register(portid,
> > 		RTE_ETH_EVENT_INTR_RESET,
> > 		reset_event_callback,
> > 		arg);
> [...]
> > --- a/doc/guides/rel_notes/release_16_04.rst
> > +++ b/doc/guides/rel_notes/release_16_04.rst
> > @@ -105,6 +105,8 @@ This section should contain new features added in this release. Sample format:
> >    be down.
> >    We added the support of auto-neg by SW to avoid this link down issue.
> >  
> > +* **Added pf reset event reported in i40e vf PMD driver.
> 
> Bruce, is this comment clear enough for an user?
> 
> Beware of the wrong formating (missing **)

It's pretty ok, though I might word it a little differently myself. "reported"
should be "reporting".

Your question, though, does bring up the issue of scope and reviews again. I, as
committer, spend a lot of time tweaking commit messages, sanity checking
patches for compilation errors under various settings, and running checkpatch
etc. before applying them. However, IMHO it is up to the maintainers of the
various subsystems to enforce proper documentation in the patches submitted.
The maintainers are the primary gatekeepers here, and I, for one, don't want to
end up having to review all patches in detail before I apply them - otherwise
we'll be limited to a very small number of driver patches per release :)

In this case, if the submitter of the patch and the maintainer of the driver in
question are happy with the documentation, then who am I to go querying that. :-)

Having committers do full review on apply will only have two possible effects:
1. make the maintainers less conscientious about their job, since they know the
  committers will catch any real bugs or issues on apply
2. cause a lot of problems for submitters as they see a lot of issues being
  flagged at the last minute by committers, when they thought their patch was
  safely acked and ready for commit for some time.

We certainly see lots of the second issue occurring right now, I believe - [I'm
obvously not going to comment on the former :-)]

I'd be very much in favour of having a rule that once a patch is acked by a
maintainer, then it must be applied. We may suffer a bit from slightly lower
quality patches getting applied, but the speed of applying patches should
increase, and the patch contents can always be fixed by subsequent patches later.
[Unlike commit message which can't be fixed later without rewriting git history]
In this case, I feel that phrase "the perfect is the enemy of the good" applies.
https://en.wikipedia.org/wiki/Perfect_is_the_enemy_of_good

Just my 2c on this. I'm sure you have a different view, Thomas, so it's probably
a discussion worth having.

/Bruce

Re: expectations on maintainer's review

[Thomas Monjalon]

I've changed the title for this discussion.

2016-03-09 11:01, Bruce Richardson:
[snip comments about minor issue in release notes]

> Your question, though, does bring up the issue of scope and reviews again. I, as
> committer, spend a lot of time tweaking commit messages, sanity checking
> patches for compilation errors under various settings, and running checkpatch
> etc. before applying them. However, IMHO it is up to the maintainers of the
> various subsystems to enforce proper documentation in the patches submitted.
> The maintainers are the primary gatekeepers here, and I, for one, don't want to
> end up having to review all patches in detail before I apply them - otherwise
> we'll be limited to a very small number of driver patches per release :)

Yes that's a problem.

> In this case, if the submitter of the patch and the maintainer of the driver in
> question are happy with the documentation, then who am I to go querying that. :-)
> 
> Having committers do full review on apply will only have two possible effects:
> 1. make the maintainers less conscientious about their job, since they know the
>   committers will catch any real bugs or issues on apply

Yes we need maintainers to be conscientious on every parts of the patches.
One problem about the release notes and doc, is that not a lot of maintainers
have the "english skills".
Note that it would be easier if we would allow to write in Irish, Chinese or
French languages ;)
Unfortunately we took the constraints of writing in C and English.

> 2. cause a lot of problems for submitters as they see a lot of issues being
>   flagged at the last minute by committers, when they thought their patch was
>   safely acked and ready for commit for some time.
> 
> We certainly see lots of the second issue occurring right now, I believe - [I'm
> obvously not going to comment on the former :-)]
> 
> I'd be very much in favour of having a rule that once a patch is acked by a
> maintainer, then it must be applied. We may suffer a bit from slightly lower
> quality patches getting applied, but the speed of applying patches should
> increase, and the patch contents can always be fixed by subsequent patches later.
> [Unlike commit message which can't be fixed later without rewriting git history]
> In this case, I feel that phrase "the perfect is the enemy of the good" applies.
> https://en.wikipedia.org/wiki/Perfect_is_the_enemy_of_good

Yes but I don't think saying we are OK to decrease the quality is a good message.
The reality is that people never rework what was been committed.
That's why we must be very careful on API and documentation.
About the release notes, decreasing its quality mean we don't care wether it is
read and understood. So maybe we can shrink it to less details and have only a
title with a git/author reference.

> Just my 2c on this. I'm sure you have a different view, Thomas, so it's probably
> a discussion worth having.

Thanks for bringing the discussion.

Re: expectations on maintainer's review

[Bruce Richardson]

On Wed, Mar 09, 2016 at 12:26:58PM +0100, Thomas Monjalon wrote:
> I've changed the title for this discussion.
> 
> 2016-03-09 11:01, Bruce Richardson:
> [snip comments about minor issue in release notes]
> 
> > Your question, though, does bring up the issue of scope and reviews again. I, as
> > committer, spend a lot of time tweaking commit messages, sanity checking
> > patches for compilation errors under various settings, and running checkpatch
> > etc. before applying them. However, IMHO it is up to the maintainers of the
> > various subsystems to enforce proper documentation in the patches submitted.
> > The maintainers are the primary gatekeepers here, and I, for one, don't want to
> > end up having to review all patches in detail before I apply them - otherwise
> > we'll be limited to a very small number of driver patches per release :)
> 
> Yes that's a problem.
> 
> > In this case, if the submitter of the patch and the maintainer of the driver in
> > question are happy with the documentation, then who am I to go querying that. :-)
> > 
> > Having committers do full review on apply will only have two possible effects:
> > 1. make the maintainers less conscientious about their job, since they know the
> >   committers will catch any real bugs or issues on apply
> 
> Yes we need maintainers to be conscientious on every parts of the patches.

Definite +1

> One problem about the release notes and doc, is that not a lot of maintainers
> have the "english skills".
> Note that it would be easier if we would allow to write in Irish, Chinese or
> French languages ;)
> Unfortunately we took the constraints of writing in C and English.
> 

Yes, language is a good point, and I'm ok with helping to clean up grammar and 
minor language issues e.g. the one word correction I suggested at the start of
this discussion. For the scope of the text, and whether it contains enough
information, I would tend to push that responsibility back on the maintainer
though.


> > 2. cause a lot of problems for submitters as they see a lot of issues being
> >   flagged at the last minute by committers, when they thought their patch was
> >   safely acked and ready for commit for some time.
> > 
> > We certainly see lots of the second issue occurring right now, I believe - [I'm
> > obvously not going to comment on the former :-)]
> > 
> > I'd be very much in favour of having a rule that once a patch is acked by a
> > maintainer, then it must be applied. We may suffer a bit from slightly lower
> > quality patches getting applied, but the speed of applying patches should
> > increase, and the patch contents can always be fixed by subsequent patches later.
> > [Unlike commit message which can't be fixed later without rewriting git history]
> > In this case, I feel that phrase "the perfect is the enemy of the good" applies.
> > https://en.wikipedia.org/wiki/Perfect_is_the_enemy_of_good
> 
> Yes but I don't think saying we are OK to decrease the quality is a good message.
> The reality is that people never rework what was been committed.

Yes, point taken.

> That's why we must be very careful on API and documentation.
> About the release notes, decreasing its quality mean we don't care wether it is
> read and understood. So maybe we can shrink it to less details and have only a
> title with a git/author reference.

I don't think I agree with that. I think the doc should be readable independently
of having the git repo.

> 
> > Just my 2c on this. I'm sure you have a different view, Thomas, so it's probably
> > a discussion worth having.
> 
> Thanks for bringing the discussion.

Indeed. So I would summarise this as:
* an ask to the maintainers to pay increased attention to documentation side of
patches when reviewing and acking.
* on my end, I will do some doc reviews as part of applying commits, but on a
best-effort basis. The primary responsibility is with the maintainers to ensure
documentation quantity before patch application stage.

Does that seem reasonable?

Regards,
/Bruce

Re: expectations on maintainer's review

[Thomas Monjalon]

2016-03-09 14:15, Bruce Richardson:
> On Wed, Mar 09, 2016 at 12:26:58PM +0100, Thomas Monjalon wrote:
> > I've changed the title for this discussion.
> > 
> > 2016-03-09 11:01, Bruce Richardson:
> > [snip comments about minor issue in release notes]
> > 
> > > Your question, though, does bring up the issue of scope and reviews again. I, as
> > > committer, spend a lot of time tweaking commit messages, sanity checking
> > > patches for compilation errors under various settings, and running checkpatch
> > > etc. before applying them. However, IMHO it is up to the maintainers of the
> > > various subsystems to enforce proper documentation in the patches submitted.
> > > The maintainers are the primary gatekeepers here, and I, for one, don't want to
> > > end up having to review all patches in detail before I apply them - otherwise
> > > we'll be limited to a very small number of driver patches per release :)
> > 
> > Yes that's a problem.
> > 
> > > In this case, if the submitter of the patch and the maintainer of the driver in
> > > question are happy with the documentation, then who am I to go querying that. :-)
> > > 
> > > Having committers do full review on apply will only have two possible effects:
> > > 1. make the maintainers less conscientious about their job, since they know the
> > >   committers will catch any real bugs or issues on apply
> > 
> > Yes we need maintainers to be conscientious on every parts of the patches.
> 
> Definite +1
> 
> > One problem about the release notes and doc, is that not a lot of maintainers
> > have the "english skills".
> > Note that it would be easier if we would allow to write in Irish, Chinese or
> > French languages ;)
> > Unfortunately we took the constraints of writing in C and English.
> > 
> 
> Yes, language is a good point, and I'm ok with helping to clean up grammar and 
> minor language issues e.g. the one word correction I suggested at the start of
> this discussion. For the scope of the text, and whether it contains enough
> information, I would tend to push that responsibility back on the maintainer
> though.
> 
> 
> > > 2. cause a lot of problems for submitters as they see a lot of issues being
> > >   flagged at the last minute by committers, when they thought their patch was
> > >   safely acked and ready for commit for some time.
> > > 
> > > We certainly see lots of the second issue occurring right now, I believe - [I'm
> > > obvously not going to comment on the former :-)]
> > > 
> > > I'd be very much in favour of having a rule that once a patch is acked by a
> > > maintainer, then it must be applied. We may suffer a bit from slightly lower
> > > quality patches getting applied, but the speed of applying patches should
> > > increase, and the patch contents can always be fixed by subsequent patches later.
> > > [Unlike commit message which can't be fixed later without rewriting git history]
> > > In this case, I feel that phrase "the perfect is the enemy of the good" applies.
> > > https://en.wikipedia.org/wiki/Perfect_is_the_enemy_of_good
> > 
> > Yes but I don't think saying we are OK to decrease the quality is a good message.
> > The reality is that people never rework what was been committed.
> 
> Yes, point taken.
> 
> > That's why we must be very careful on API and documentation.
> > About the release notes, decreasing its quality mean we don't care wether it is
> > read and understood. So maybe we can shrink it to less details and have only a
> > title with a git/author reference.
> 
> I don't think I agree with that. I think the doc should be readable independently
> of having the git repo.
> 
> > 
> > > Just my 2c on this. I'm sure you have a different view, Thomas, so it's probably
> > > a discussion worth having.
> > 
> > Thanks for bringing the discussion.
> 
> Indeed. So I would summarise this as:
> * an ask to the maintainers to pay increased attention to documentation side of
> patches when reviewing and acking.
> * on my end, I will do some doc reviews as part of applying commits, but on a
> best-effort basis. The primary responsibility is with the maintainers to ensure
> documentation quantity before patch application stage.
> 
> Does that seem reasonable?

Yes and I'd like to hear some maintainers to comment or commit this summary.

[PATCH v5 0/2] i40evf: pf reset event report

[Jingjing Wu]

v5 changes:
 - doc rewording and format fixing.

v4 changes:
 - rebase on latest dpdk-next-net/rel_16_04 branch (commit 0f9564a0e4f2)

v3 changes:
 - commit log doc rewording.
 - rebase on latest dpdk-next-net/rel_16_04 branch.
 - remove few useless line.
 - adjust interval and increase times for waiting pf msg

v2 changes:
 - remove the change on vf reset status checking
 - add pf event report support in release note

When Linux PF and DPDK VF are used for i40e PMD, In case of PF
reset, interrupt request will go via adminq event, VF need be
informed, a callback mechanism is introduced by VF. This will
allow VF to invoke callback when reset happens.
Users can register a callback for this interrupt event like:
    rte_eth_dev_callback_register(portid,
                 RTE_ETH_EVENT_INTR_RESET,
                 reset_event_callback,
                 arg);


Jingjing Wu (2):
  i40evf: allocate virtchnl cmd buffer for each vf
  i40evf: support to report pf reset event

 doc/guides/rel_notes/release_16_04.rst |   2 +
 drivers/net/i40e/i40e_ethdev.h         |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 420 +++++++++++++++++++++++----------
 lib/librte_ether/rte_ethdev.h          |   1 +
 4 files changed, 302 insertions(+), 123 deletions(-)

-- 
2.4.0

[PATCH v5 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Jingjing Wu]

Currently, i40evf PMD uses a global static buffer to send virtchnl
command to host driver. It is shared by multi VFs.
This patch changed to allocate virtchnl cmd buffer for each VF.

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>
---
 drivers/net/i40e/i40e_ethdev.h    |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c | 178 +++++++++++++++-----------------------
 2 files changed, 71 insertions(+), 109 deletions(-)

diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index a9b805e..8cd6126 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -497,7 +497,9 @@ struct i40e_vf {
 	bool link_up;
 	bool vf_reset;
 	volatile uint32_t pend_cmd; /* pending command not finished yet */
+	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	u16 pend_msg; /* flags indicates events from pf not handled yet */
+	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
 
 	/* VSI info */
 	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 6185ee8..90edeb5 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -103,9 +103,6 @@ enum i40evf_aq_result {
 	I40EVF_MSG_CMD,      /* Read async command result */
 };
 
-/* A share buffer to store the command result from PF driver */
-static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
-
 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 static int i40evf_dev_start(struct rte_eth_dev *dev);
 static void i40evf_dev_stop(struct rte_eth_dev *dev);
@@ -217,31 +214,37 @@ static const struct eth_dev_ops i40evf_eth_dev_ops = {
 };
 
 /*
- * Parse admin queue message.
- *
- * return value:
- *  < 0: meet error
- *  0: read sys msg
- *  > 0: read cmd result
+ * Read data in admin queue to get msg from pf driver
  */
 static enum i40evf_aq_result
-i40evf_parse_pfmsg(struct i40e_vf *vf,
-		   struct i40e_arq_event_info *event,
-		   struct i40evf_arq_msg_info *data)
+i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 {
-	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
-			rte_le_to_cpu_32(event->desc.cookie_high);
-	enum i40e_status_code retval = (enum i40e_status_code)\
-			rte_le_to_cpu_32(event->desc.cookie_low);
-	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info event;
+	enum i40e_virtchnl_ops opcode;
+	enum i40e_status_code retval;
+	int ret;
+	enum i40evf_aq_result result = I40EVF_MSG_NON;
+
+	event.buf_len = data->buf_len;
+	event.msg_buf = data->msg;
+	ret = i40e_clean_arq_element(hw, &event, NULL);
+	/* Can't read any msg from adminQ */
+	if (ret) {
+		if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
+			result = I40EVF_MSG_ERR;
+		return result;
+	}
 
+	opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
+	retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 	/* pf sys event */
 	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 		struct i40e_virtchnl_pf_event *vpe =
-			(struct i40e_virtchnl_pf_event *)event->msg_buf;
+			(struct i40e_virtchnl_pf_event *)event.msg_buf;
 
-		/* Initialize ret to sys event */
-		ret = I40EVF_MSG_SYS;
+		result = I40EVF_MSG_SYS;
 		switch (vpe->event) {
 		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 			vf->link_up =
@@ -266,74 +269,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
 		}
 	} else {
 		/* async reply msg on command issued by vf previously */
-		ret = I40EVF_MSG_CMD;
+		result = I40EVF_MSG_CMD;
 		/* Actual data length read from PF */
-		data->msg_len = event->msg_len;
+		data->msg_len = event.msg_len;
 	}
-	/* fill the ops and result to notify VF */
+
 	data->result = retval;
 	data->ops = opcode;
 
-	return ret;
-}
-
-/*
- * Read data in admin queue to get msg from pf driver
- */
-static enum i40evf_aq_result
-i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
-{
-	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	struct i40e_arq_event_info event;
-	int ret;
-	enum i40evf_aq_result result = I40EVF_MSG_NON;
-
-	event.buf_len = data->buf_len;
-	event.msg_buf = data->msg;
-	ret = i40e_clean_arq_element(hw, &event, NULL);
-	/* Can't read any msg from adminQ */
-	if (ret) {
-		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
-			result = I40EVF_MSG_NON;
-		else
-			result = I40EVF_MSG_ERR;
-		return result;
-	}
-
-	/* Parse the event */
-	result = i40evf_parse_pfmsg(vf, &event, data);
-
 	return result;
 }
 
-/*
- * Polling read until command result return from pf driver or meet error.
- */
-static int
-i40evf_wait_cmd_done(struct rte_eth_dev *dev,
-		     struct i40evf_arq_msg_info *data)
-{
-	int i = 0;
-	enum i40evf_aq_result ret;
-
-#define MAX_TRY_TIMES 20
-#define ASQ_DELAY_MS  100
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, data);
-		if (ret == I40EVF_MSG_CMD)
-			return 0;
-		else if (ret == I40EVF_MSG_ERR)
-			return -1;
-
-		/* If don't read msg or read sys event, continue */
-	} while(i++ < MAX_TRY_TIMES);
-
-	return -1;
-}
-
 /**
  * clear current command. Only call in case execute
  * _atomic_set_cmd successfully.
@@ -360,13 +306,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 	return !ret;
 }
 
+#define MAX_TRY_TIMES 200
+#define ASQ_DELAY_MS  10
+
 static int
 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	int err = -1;
 	struct i40evf_arq_msg_info info;
+	enum i40evf_aq_result ret;
+	int err = -1;
+	int i = 0;
 
 	if (_atomic_set_cmd(vf, args->ops))
 		return -1;
@@ -384,19 +335,21 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	err = i40evf_wait_cmd_done(dev, &info);
-	/* read message and it's expected one */
-	if (!err && args->ops == info.ops)
-		_clear_cmd(vf);
-	else if (err) {
-		PMD_DRV_LOG(ERR, "Failed to read message from AdminQ");
-		_clear_cmd(vf);
-	}
-	else if (args->ops != info.ops)
-		PMD_DRV_LOG(ERR, "command mismatch, expect %u, get %u",
-			    args->ops, info.ops);
+	do {
+		ret = i40evf_read_pfmsg(dev, &info);
+		if (ret == I40EVF_MSG_CMD) {
+			err = 0;
+			break;
+		} else if (ret == I40EVF_MSG_ERR) {
+			err = -1;
+			break;
+		}
+		rte_delay_ms(ASQ_DELAY_MS);
+		/* If don't read msg or read sys event, continue */
+	} while (i++ < MAX_TRY_TIMES);
+	_clear_cmd(vf);
 
-	return err | info.result;
+	return err | vf->cmd_retval;
 }
 
 /*
@@ -416,7 +369,7 @@ i40evf_check_api_version(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_VERSION;
 	args.in_args = (uint8_t *)&version;
 	args.in_args_size = sizeof(version);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -454,7 +407,7 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev)
 	uint32_t caps, len;
 
 	args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	if (PF_IS_V11(vf)) {
 		caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
@@ -507,7 +460,7 @@ i40evf_config_promisc(struct rte_eth_dev *dev,
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 	args.in_args = (uint8_t *)&promisc;
 	args.in_args_size = sizeof(promisc);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -534,7 +487,7 @@ i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 	args.in_args = (uint8_t *)&offload;
 	args.in_args_size = sizeof(offload);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -565,7 +518,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 	args.in_args = (uint8_t *)&tpid_info;
 	args.in_args_size = sizeof(tpid_info);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -644,7 +597,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 	args.in_args = (uint8_t *)vc_vqci;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -697,7 +650,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 		(enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 	args.in_args = (uint8_t *)vc_vqcei;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -759,7 +712,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 	args.in_args = (u8 *)cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -790,7 +743,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 		args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 	args.in_args = (u8 *)&queue_select;
 	args.in_args_size = sizeof(queue_select);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -882,7 +835,7 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -919,7 +872,7 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 	args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -942,7 +895,7 @@ i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 	args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 	args.in_args = (u8 *)&q_stats;
 	args.in_args_size = sizeof(q_stats);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -1036,7 +989,7 @@ i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1063,7 +1016,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1075,6 +1028,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 static int
 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 {
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	int err;
 	struct vf_cmd_info args;
 	struct rte_eth_link *new_link;
@@ -1082,7 +1036,7 @@ i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
 	args.in_args = NULL;
 	args.in_args_size = 0;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err) {
@@ -1177,7 +1131,6 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-
 	/* Reset VF and wait until it's complete */
 	if (i40evf_reset_vf(hw)) {
 		PMD_INIT_LOG(ERR, "reset NIC failed");
@@ -1195,6 +1148,11 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "init_adminq failed");
 		return -1;
 	}
+	vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
+	if (!vf->aq_resp) {
+		PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
+			goto err_aq;
+	}
 	if (i40evf_check_api_version(dev) != 0) {
 		PMD_INIT_LOG(ERR, "check_api version failed");
 		goto err_aq;
@@ -1259,6 +1217,8 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 		i40evf_dev_close(dev);
 	rte_free(vf->vf_res);
 	vf->vf_res = NULL;
+	rte_free(vf->aq_resp);
+	vf->aq_resp = NULL;
 
 	return 0;
 }
-- 
2.4.0

[PATCH v5 2/2] i40evf: support to report pf reset event

[Jingjing Wu]

When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
interrupt will go via adminq event, VF need be informed the event,
a callback mechanism is introduced by VF. This will allow VF to
invoke callback when reset happens.
Users can register a callback for this interrupt event like:
  rte_eth_dev_callback_register(portid,
		RTE_ETH_EVENT_INTR_RESET,
		reset_event_callback,
		arg);

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>
---
 doc/guides/rel_notes/release_16_04.rst |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 272 +++++++++++++++++++++++++++++----
 lib/librte_ether/rte_ethdev.h          |   1 +
 3 files changed, 246 insertions(+), 29 deletions(-)

diff --git a/doc/guides/rel_notes/release_16_04.rst b/doc/guides/rel_notes/release_16_04.rst
index 5abf48a..68a35c9 100644
--- a/doc/guides/rel_notes/release_16_04.rst
+++ b/doc/guides/rel_notes/release_16_04.rst
@@ -119,6 +119,8 @@ This section should contain new features added in this release. Sample format:
 
   Only available with Mellanox OFED >= 3.2.
 
+* **Added pf reset event reporting in i40e vf PMD driver.**
+
 
 Resolved Issues
 ---------------
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 90edeb5..ede4bc2 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -74,8 +74,6 @@
 #define I40EVF_BUSY_WAIT_DELAY 10
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
-/*ITR index for NOITR*/
-#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
 
 struct i40evf_arq_msg_info {
 	enum i40e_virtchnl_ops ops;
@@ -151,6 +149,9 @@ static int
 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 static int
 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   uint16_t msglen);
 
 /* Default hash key buffer for RSS */
 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
@@ -335,19 +336,40 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	do {
-		ret = i40evf_read_pfmsg(dev, &info);
-		if (ret == I40EVF_MSG_CMD) {
-			err = 0;
-			break;
-		} else if (ret == I40EVF_MSG_ERR) {
-			err = -1;
-			break;
-		}
-		rte_delay_ms(ASQ_DELAY_MS);
-		/* If don't read msg or read sys event, continue */
-	} while (i++ < MAX_TRY_TIMES);
-	_clear_cmd(vf);
+	switch (args->ops) {
+	case I40E_VIRTCHNL_OP_RESET_VF:
+		/*no need to process in this function */
+		break;
+	case I40E_VIRTCHNL_OP_VERSION:
+	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
+		/* for init adminq commands, need to poll the response */
+		do {
+			ret = i40evf_read_pfmsg(dev, &info);
+			if (ret == I40EVF_MSG_CMD) {
+				err = 0;
+				break;
+			} else if (ret == I40EVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		_clear_cmd(vf);
+		break;
+
+	default:
+		/* for other adminq in running time, waiting the cmd done flag */
+		do {
+			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
+				err = 0;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		break;
+	}
 
 	return err | vf->cmd_retval;
 }
@@ -696,7 +718,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 
 	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = 1;
-	map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
+	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 	/* Alway use default dynamic MSIX interrupt */
 	map_info->vecmap[0].vector_id = vector_id;
@@ -1070,6 +1092,38 @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/* Disable IRQ0 */
+static inline void
+i40evf_disable_irq0(struct i40e_hw *hw)
+{
+	/* Disable all interrupt types */
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	I40EVF_WRITE_FLUSH(hw);
+}
+
+/* Enable IRQ0 */
+static inline void
+i40evf_enable_irq0(struct i40e_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	uint32_t val;
+
+	i40evf_disable_irq0(hw);
+	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
+	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
+		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
+
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		I40E_VFINT_DYN_CTL01_INTENA_MASK |
+		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	I40EVF_WRITE_FLUSH(hw);
+}
+
 static int
 i40evf_reset_vf(struct i40e_hw *hw)
 {
@@ -1115,6 +1169,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct ether_addr *p_mac_addr;
+	uint16_t interval =
+		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1195,6 +1251,16 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	else
 		eth_random_addr(hw->mac.addr); /* Generate a random one */
 
+	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
+	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       (I40E_ITR_INDEX_DEFAULT <<
+				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
+			       (interval <<
+				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
+		I40EVF_WRITE_FLUSH(hw);
+	}
+
 	return 0;
 
 err_alloc:
@@ -1223,11 +1289,142 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   __rte_unused uint16_t msglen)
+{
+	struct i40e_virtchnl_pf_event *pf_msg =
+			(struct i40e_virtchnl_pf_event *)msg;
+
+	switch (pf_msg->event) {
+	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
+		break;
+	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
+		break;
+	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
+		break;
+	default:
+		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
+		break;
+	}
+}
+
+static void
+i40evf_handle_aq_msg(struct rte_eth_dev *dev)
+{
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info info;
+	struct i40e_virtchnl_msg *v_msg;
+	uint16_t pending, opcode;
+	int ret;
+
+	info.buf_len = I40E_AQ_BUF_SZ;
+	if (!vf->aq_resp) {
+		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
+		return;
+	}
+	info.msg_buf = vf->aq_resp;
+	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
+
+	pending = 1;
+	while (pending) {
+		ret = i40e_clean_arq_element(hw, &info, &pending);
+
+		if (ret != I40E_SUCCESS) {
+			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
+				    "ret: %d", ret);
+			break;
+		}
+		opcode = rte_le_to_cpu_16(info.desc.opcode);
+
+		switch (opcode) {
+		case i40e_aqc_opc_send_msg_to_vf:
+			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
+				/* process event*/
+				i40evf_handle_pf_event(dev, info.msg_buf,
+							info.msg_len);
+			else {
+				/* read message and it's expected one */
+				if (v_msg->v_opcode == vf->pend_cmd) {
+					vf->cmd_retval = v_msg->v_retval;
+					/* prevent compiler reordering */
+					rte_compiler_barrier();
+					_clear_cmd(vf);
+				} else
+					PMD_DRV_LOG(ERR, "command mismatch,"
+						"expect %u, get %u",
+						vf->pend_cmd, v_msg->v_opcode);
+				PMD_DRV_LOG(DEBUG, "adminq response is received,"
+					     " opcode = %d\n", v_msg->v_opcode);
+			}
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
+				    opcode);
+			break;
+		}
+	}
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt. Only adminq interrupt is processed in VF.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
+			   void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	i40evf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
+		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
+		goto done;
+	}
+
+	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
+		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
+		i40evf_handle_aq_msg(dev);
+	}
+
+	/* Link Status Change interrupt */
+	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
+		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
+				   " do nothing\n");
+
+done:
+	i40evf_enable_irq0(hw);
+	rte_intr_enable(&(dev->pci_dev->intr_handle));
+}
+
+
 static int
 i40evf_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
 			eth_dev->data->dev_private);
+	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1262,6 +1459,16 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
+	/* register callback func to eal lib */
+	rte_intr_callback_register(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)eth_dev);
+
+	/* enable uio intr after callback register */
+	rte_intr_enable(&(pci_dev->intr_handle));
+
+	/* configure and enable device interrupt */
+	i40evf_enable_irq0(hw);
+
 	/* copy mac addr */
 	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
 					ETHER_ADDR_LEN, 0);
@@ -1639,7 +1846,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 		I40E_WRITE_REG(hw,
 			       I40E_VFINT_DYN_CTL01,
 			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
-			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1650,11 +1858,10 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
 			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
-	else
-		/* To support Linux PF host */
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
-				I40E_VFINT_DYN_CTL01_INTENA_MASK |
-				I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't set I40E_VFINT_DYN_CTL01,
+	 * because it is already done in i40evf_enable_irq0.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1667,7 +1874,8 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 
 	if (!rte_intr_allow_others(intr_handle)) {
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1677,8 +1885,10 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
 						    - 1),
 			       0);
-	else
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't zero I40E_VFINT_DYN_CTL01,
+	 * because interrupt 0 is also used for adminq processing.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1802,10 +2012,6 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* vf don't allow intr except for rxq intr */
-	if (dev->data->dev_conf.intr_conf.rxq != 0)
-		rte_intr_enable(intr_handle);
-
 	i40evf_enable_queues_intr(dev);
 	return 0;
 
@@ -1997,12 +2203,20 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = dev->pci_dev;
 
 	i40evf_dev_stop(dev);
 	hw->adapter_stopped = 1;
 	i40e_dev_free_queues(dev);
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(&(pci_dev->intr_handle));
+
+	/* unregister callback func from eal lib */
+	rte_intr_callback_unregister(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)dev);
+	i40evf_disable_irq0(hw);
 }
 
 static int
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index e2893ba..0c81435 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -2663,6 +2663,7 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
 enum rte_eth_event_type {
 	RTE_ETH_EVENT_UNKNOWN,  /**< unknown event type */
 	RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
+	RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.4.0

Re: [PATCH v5 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Bruce Richardson]

On Thu, Mar 10, 2016 at 11:41:10AM +0800, Jingjing Wu wrote:
> Currently, i40evf PMD uses a global static buffer to send virtchnl
> command to host driver. It is shared by multi VFs.
> This patch changed to allocate virtchnl cmd buffer for each VF.
> 
> Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> Acked-by: Helin Zhang <helin.zhang@intel.com>
> ---
>  drivers/net/i40e/i40e_ethdev.h    |   2 +
>  drivers/net/i40e/i40e_ethdev_vf.c | 178 +++++++++++++++-----------------------
>  2 files changed, 71 insertions(+), 109 deletions(-)
> 
Hi Jingjing,

I get some compilation errors when I apply this to dpdk-next-net.

/home/bruce/next-net/dpdk-next-net/drivers/net/i40e/i40e_ethdev_vf.c: In function ‘i40evf_add_del_all_mac_addr’:
/home/bruce/next-net/dpdk-next-net/drivers/net/i40e/i40e_ethdev_vf.c:1801:21: error: ‘cmd_result_buffer’ undeclared (first use in this function)
   args.out_buffer = cmd_result_buffer;
                        ^

This is with Fedora 23 with gcc 5.3.1

/Bruce

Re: [PATCH v5 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Wu, Jingjing]

Hi, Bruce

That is because another patch was applied before this one. I will rework it, and send it soon.

Thanks
Jingjing

> -----Original Message-----
> From: Richardson, Bruce
> Sent: Monday, March 14, 2016 8:22 PM
> To: Wu, Jingjing
> Cc: dev@dpdk.org; Zhang, Helin; Tao, Zhe
> Subject: Re: [PATCH v5 1/2] i40evf: allocate virtchnl cmd buffer for each vf
> 
> On Thu, Mar 10, 2016 at 11:41:10AM +0800, Jingjing Wu wrote:
> > Currently, i40evf PMD uses a global static buffer to send virtchnl
> > command to host driver. It is shared by multi VFs.
> > This patch changed to allocate virtchnl cmd buffer for each VF.
> >
> > Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
> > Acked-by: Helin Zhang <helin.zhang@intel.com>
> > ---
> >  drivers/net/i40e/i40e_ethdev.h    |   2 +
> >  drivers/net/i40e/i40e_ethdev_vf.c | 178
> > +++++++++++++++-----------------------
> >  2 files changed, 71 insertions(+), 109 deletions(-)
> >
> Hi Jingjing,
> 
> I get some compilation errors when I apply this to dpdk-next-net.
> 
> /home/bruce/next-net/dpdk-next-net/drivers/net/i40e/i40e_ethdev_vf.c:
> In function ‘i40evf_add_del_all_mac_addr’:
> /home/bruce/next-net/dpdk-next-
> net/drivers/net/i40e/i40e_ethdev_vf.c:1801:21: error: ‘cmd_result_buffer’
> undeclared (first use in this function)
>    args.out_buffer = cmd_result_buffer;
>                         ^
> 
> This is with Fedora 23 with gcc 5.3.1
> 
> /Bruce

[PATCH v6 0/2] i40evf: pf reset event report

[Jingjing Wu]

v6 changes:
 - rebase on latest dpdk-next-net/rel_16_04 branch (commit 94d8f4bf963a)
 - fix complie issue

v5 changes:
 - doc rewording and format fixing.

v4 changes:
 - rebase on latest dpdk-next-net/rel_16_04 branch (commit 0f9564a0e4f2)

v3 changes:
 - commit log doc rewording.
 - rebase on latest dpdk-next-net/rel_16_04 branch.
 - remove few useless line.
 - adjust interval and increase times for waiting pf msg

v2 changes:
 - remove the change on vf reset status checking
 - add pf event report support in release note

When Linux PF and DPDK VF are used for i40e PMD, In case of PF
reset, interrupt request will go via adminq event, VF need be
informed, a callback mechanism is introduced by VF. This will
allow VF to invoke callback when reset happens.
Users can register a callback for this interrupt event like:
    rte_eth_dev_callback_register(portid,
                 RTE_ETH_EVENT_INTR_RESET,
                 reset_event_callback,
                 arg);


Jingjing Wu (2):
  i40evf: allocate virtchnl cmd buffer for each vf
  i40evf: support to report pf reset event

 doc/guides/rel_notes/release_16_04.rst |   3 +
 drivers/net/i40e/i40e_ethdev.h         |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 425 +++++++++++++++++++++++----------
 lib/librte_ether/rte_ethdev.h          |   1 +
 4 files changed, 306 insertions(+), 125 deletions(-)

-- 
2.4.0

[PATCH v6 1/2] i40evf: allocate virtchnl cmd buffer for each vf

[Jingjing Wu]

Currently, i40evf PMD uses a global static buffer to send virtchnl
command to host driver. It is shared by multi VFs.
This patch changed to allocate virtchnl cmd buffer for each VF.

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>
---
 drivers/net/i40e/i40e_ethdev.h    |   2 +
 drivers/net/i40e/i40e_ethdev_vf.c | 180 +++++++++++++++-----------------------
 2 files changed, 72 insertions(+), 110 deletions(-)

diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index 1c75672..b641882 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -500,7 +500,9 @@ struct i40e_vf {
 	bool link_up;
 	bool vf_reset;
 	volatile uint32_t pend_cmd; /* pending command not finished yet */
+	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	u16 pend_msg; /* flags indicates events from pf not handled yet */
+	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
 
 	/* VSI info */
 	struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 6b7b350..7944ccc 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -103,9 +103,6 @@ enum i40evf_aq_result {
 	I40EVF_MSG_CMD,      /* Read async command result */
 };
 
-/* A share buffer to store the command result from PF driver */
-static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
-
 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 static int i40evf_dev_start(struct rte_eth_dev *dev);
 static void i40evf_dev_stop(struct rte_eth_dev *dev);
@@ -224,31 +221,37 @@ static const struct eth_dev_ops i40evf_eth_dev_ops = {
 };
 
 /*
- * Parse admin queue message.
- *
- * return value:
- *  < 0: meet error
- *  0: read sys msg
- *  > 0: read cmd result
+ * Read data in admin queue to get msg from pf driver
  */
 static enum i40evf_aq_result
-i40evf_parse_pfmsg(struct i40e_vf *vf,
-		   struct i40e_arq_event_info *event,
-		   struct i40evf_arq_msg_info *data)
+i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 {
-	enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
-			rte_le_to_cpu_32(event->desc.cookie_high);
-	enum i40e_status_code retval = (enum i40e_status_code)\
-			rte_le_to_cpu_32(event->desc.cookie_low);
-	enum i40evf_aq_result ret = I40EVF_MSG_CMD;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info event;
+	enum i40e_virtchnl_ops opcode;
+	enum i40e_status_code retval;
+	int ret;
+	enum i40evf_aq_result result = I40EVF_MSG_NON;
+
+	event.buf_len = data->buf_len;
+	event.msg_buf = data->msg;
+	ret = i40e_clean_arq_element(hw, &event, NULL);
+	/* Can't read any msg from adminQ */
+	if (ret) {
+		if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
+			result = I40EVF_MSG_ERR;
+		return result;
+	}
 
+	opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
+	retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 	/* pf sys event */
 	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 		struct i40e_virtchnl_pf_event *vpe =
-			(struct i40e_virtchnl_pf_event *)event->msg_buf;
+			(struct i40e_virtchnl_pf_event *)event.msg_buf;
 
-		/* Initialize ret to sys event */
-		ret = I40EVF_MSG_SYS;
+		result = I40EVF_MSG_SYS;
 		switch (vpe->event) {
 		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 			vf->link_up =
@@ -273,74 +276,17 @@ i40evf_parse_pfmsg(struct i40e_vf *vf,
 		}
 	} else {
 		/* async reply msg on command issued by vf previously */
-		ret = I40EVF_MSG_CMD;
+		result = I40EVF_MSG_CMD;
 		/* Actual data length read from PF */
-		data->msg_len = event->msg_len;
+		data->msg_len = event.msg_len;
 	}
-	/* fill the ops and result to notify VF */
+
 	data->result = retval;
 	data->ops = opcode;
 
-	return ret;
-}
-
-/*
- * Read data in admin queue to get msg from pf driver
- */
-static enum i40evf_aq_result
-i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
-{
-	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	struct i40e_arq_event_info event;
-	int ret;
-	enum i40evf_aq_result result = I40EVF_MSG_NON;
-
-	event.buf_len = data->buf_len;
-	event.msg_buf = data->msg;
-	ret = i40e_clean_arq_element(hw, &event, NULL);
-	/* Can't read any msg from adminQ */
-	if (ret) {
-		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
-			result = I40EVF_MSG_NON;
-		else
-			result = I40EVF_MSG_ERR;
-		return result;
-	}
-
-	/* Parse the event */
-	result = i40evf_parse_pfmsg(vf, &event, data);
-
 	return result;
 }
 
-/*
- * Polling read until command result return from pf driver or meet error.
- */
-static int
-i40evf_wait_cmd_done(struct rte_eth_dev *dev,
-		     struct i40evf_arq_msg_info *data)
-{
-	int i = 0;
-	enum i40evf_aq_result ret;
-
-#define MAX_TRY_TIMES 20
-#define ASQ_DELAY_MS  100
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, data);
-		if (ret == I40EVF_MSG_CMD)
-			return 0;
-		else if (ret == I40EVF_MSG_ERR)
-			return -1;
-
-		/* If don't read msg or read sys event, continue */
-	} while(i++ < MAX_TRY_TIMES);
-
-	return -1;
-}
-
 /**
  * clear current command. Only call in case execute
  * _atomic_set_cmd successfully.
@@ -367,13 +313,18 @@ _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 	return !ret;
 }
 
+#define MAX_TRY_TIMES 200
+#define ASQ_DELAY_MS  10
+
 static int
 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-	int err = -1;
 	struct i40evf_arq_msg_info info;
+	enum i40evf_aq_result ret;
+	int err = -1;
+	int i = 0;
 
 	if (_atomic_set_cmd(vf, args->ops))
 		return -1;
@@ -391,19 +342,21 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	err = i40evf_wait_cmd_done(dev, &info);
-	/* read message and it's expected one */
-	if (!err && args->ops == info.ops)
-		_clear_cmd(vf);
-	else if (err) {
-		PMD_DRV_LOG(ERR, "Failed to read message from AdminQ");
-		_clear_cmd(vf);
-	}
-	else if (args->ops != info.ops)
-		PMD_DRV_LOG(ERR, "command mismatch, expect %u, get %u",
-			    args->ops, info.ops);
+	do {
+		ret = i40evf_read_pfmsg(dev, &info);
+		if (ret == I40EVF_MSG_CMD) {
+			err = 0;
+			break;
+		} else if (ret == I40EVF_MSG_ERR) {
+			err = -1;
+			break;
+		}
+		rte_delay_ms(ASQ_DELAY_MS);
+		/* If don't read msg or read sys event, continue */
+	} while (i++ < MAX_TRY_TIMES);
+	_clear_cmd(vf);
 
-	return err | info.result;
+	return err | vf->cmd_retval;
 }
 
 /*
@@ -423,7 +376,7 @@ i40evf_check_api_version(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_VERSION;
 	args.in_args = (uint8_t *)&version;
 	args.in_args_size = sizeof(version);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -461,7 +414,7 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev)
 	uint32_t caps, len;
 
 	args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	if (PF_IS_V11(vf)) {
 		caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
@@ -514,7 +467,7 @@ i40evf_config_promisc(struct rte_eth_dev *dev,
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 	args.in_args = (uint8_t *)&promisc;
 	args.in_args_size = sizeof(promisc);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -541,7 +494,7 @@ i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 	args.in_args = (uint8_t *)&offload;
 	args.in_args_size = sizeof(offload);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -572,7 +525,7 @@ i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 	args.in_args = (uint8_t *)&tpid_info;
 	args.in_args_size = sizeof(tpid_info);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -651,7 +604,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 	args.in_args = (uint8_t *)vc_vqci;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -704,7 +657,7 @@ i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 		(enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 	args.in_args = (uint8_t *)vc_vqcei;
 	args.in_args_size = size;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	ret = i40evf_execute_vf_cmd(dev, &args);
 	if (ret)
@@ -766,7 +719,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 	args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 	args.in_args = (u8 *)cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -797,7 +750,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 		args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 	args.in_args = (u8 *)&queue_select;
 	args.in_args_size = sizeof(queue_select);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -892,7 +845,7 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev,
 	args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -933,7 +886,7 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index)
 	args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 	args.in_args = cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -955,7 +908,7 @@ i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 	args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 	args.in_args = (u8 *)&q_stats;
 	args.in_args_size = sizeof(q_stats);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 
 	err = i40evf_execute_vf_cmd(dev, &args);
@@ -1049,7 +1002,7 @@ i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1076,7 +1029,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 	args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
 	args.in_args = (u8 *)&cmd_buffer;
 	args.in_args_size = sizeof(cmd_buffer);
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err)
@@ -1088,6 +1041,7 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 static int
 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 {
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	int err;
 	struct vf_cmd_info args;
 	struct rte_eth_link *new_link;
@@ -1095,7 +1049,7 @@ i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
 	args.in_args = NULL;
 	args.in_args_size = 0;
-	args.out_buffer = cmd_result_buffer;
+	args.out_buffer = vf->aq_resp;
 	args.out_size = I40E_AQ_BUF_SZ;
 	err = i40evf_execute_vf_cmd(dev, &args);
 	if (err) {
@@ -1190,7 +1144,6 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-
 	/* Reset VF and wait until it's complete */
 	if (i40evf_reset_vf(hw)) {
 		PMD_INIT_LOG(ERR, "reset NIC failed");
@@ -1208,6 +1161,11 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "init_adminq failed");
 		return -1;
 	}
+	vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
+	if (!vf->aq_resp) {
+		PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
+			goto err_aq;
+	}
 	if (i40evf_check_api_version(dev) != 0) {
 		PMD_INIT_LOG(ERR, "check_api version failed");
 		goto err_aq;
@@ -1272,6 +1230,8 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 		i40evf_dev_close(dev);
 	rte_free(vf->vf_res);
 	vf->vf_res = NULL;
+	rte_free(vf->aq_resp);
+	vf->aq_resp = NULL;
 
 	return 0;
 }
@@ -1838,7 +1798,7 @@ i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add)
 			   I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 		args.in_args = (uint8_t *)list;
 		args.in_args_size = len;
-		args.out_buffer = cmd_result_buffer;
+		args.out_buffer = vf->aq_resp;
 		args.out_size = I40E_AQ_BUF_SZ;
 		err = i40evf_execute_vf_cmd(dev, &args);
 		if (err)
-- 
2.4.0

[PATCH v6 2/2] i40evf: support to report pf reset event

[Jingjing Wu]

When Linux PF and DPDK VF are used for i40e PMD, In case of PF reset,
interrupt will go via adminq event, VF need be informed the event,
a callback mechanism is introduced by VF. This will allow VF to
invoke callback when reset happens.
Users can register a callback for this interrupt event like:
  rte_eth_dev_callback_register(portid,
		RTE_ETH_EVENT_INTR_RESET,
		reset_event_callback,
		arg);

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
Acked-by: Helin Zhang <helin.zhang@intel.com>
---
 doc/guides/rel_notes/release_16_04.rst |   3 +
 drivers/net/i40e/i40e_ethdev_vf.c      | 275 +++++++++++++++++++++++++++++----
 lib/librte_ether/rte_ethdev.h          |   1 +
 3 files changed, 249 insertions(+), 30 deletions(-)

diff --git a/doc/guides/rel_notes/release_16_04.rst b/doc/guides/rel_notes/release_16_04.rst
index 5f9eb3e..c1a8456 100644
--- a/doc/guides/rel_notes/release_16_04.rst
+++ b/doc/guides/rel_notes/release_16_04.rst
@@ -139,6 +139,9 @@ This section should contain new features added in this release. Sample format:
   Added support for linking multi-segment buffers together to
   handle Jumbo packets.
 
+* **Added pf reset event reporting in i40e vf PMD driver.**
+
+
 Resolved Issues
 ---------------
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 7944ccc..1ce174c 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -74,8 +74,6 @@
 #define I40EVF_BUSY_WAIT_DELAY 10
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
-/*ITR index for NOITR*/
-#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
 
 struct i40evf_arq_msg_info {
 	enum i40e_virtchnl_ops ops;
@@ -156,6 +154,9 @@ static int
 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 static int
 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+				   uint8_t *msg,
+				   uint16_t msglen);
 
 /* Default hash key buffer for RSS */
 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
@@ -342,19 +343,40 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	do {
-		ret = i40evf_read_pfmsg(dev, &info);
-		if (ret == I40EVF_MSG_CMD) {
-			err = 0;
-			break;
-		} else if (ret == I40EVF_MSG_ERR) {
-			err = -1;
-			break;
-		}
-		rte_delay_ms(ASQ_DELAY_MS);
-		/* If don't read msg or read sys event, continue */
-	} while (i++ < MAX_TRY_TIMES);
-	_clear_cmd(vf);
+	switch (args->ops) {
+	case I40E_VIRTCHNL_OP_RESET_VF:
+		/*no need to process in this function */
+		break;
+	case I40E_VIRTCHNL_OP_VERSION:
+	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
+		/* for init adminq commands, need to poll the response */
+		do {
+			ret = i40evf_read_pfmsg(dev, &info);
+			if (ret == I40EVF_MSG_CMD) {
+				err = 0;
+				break;
+			} else if (ret == I40EVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		_clear_cmd(vf);
+		break;
+
+	default:
+		/* for other adminq in running time, waiting the cmd done flag */
+		do {
+			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
+				err = 0;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		break;
+	}
 
 	return err | vf->cmd_retval;
 }
@@ -703,7 +725,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 
 	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = 1;
-	map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
+	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 	/* Alway use default dynamic MSIX interrupt */
 	map_info->vecmap[0].vector_id = vector_id;
@@ -1083,6 +1105,38 @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/* Disable IRQ0 */
+static inline void
+i40evf_disable_irq0(struct i40e_hw *hw)
+{
+	/* Disable all interrupt types */
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	I40EVF_WRITE_FLUSH(hw);
+}
+
+/* Enable IRQ0 */
+static inline void
+i40evf_enable_irq0(struct i40e_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	uint32_t val;
+
+	i40evf_disable_irq0(hw);
+	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
+	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
+		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
+
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		I40E_VFINT_DYN_CTL01_INTENA_MASK |
+		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	I40EVF_WRITE_FLUSH(hw);
+}
+
 static int
 i40evf_reset_vf(struct i40e_hw *hw)
 {
@@ -1128,6 +1182,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct ether_addr *p_mac_addr;
+	uint16_t interval =
+		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1208,6 +1264,16 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	else
 		eth_random_addr(hw->mac.addr); /* Generate a random one */
 
+	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
+	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       (I40E_ITR_INDEX_DEFAULT <<
+				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
+			       (interval <<
+				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
+		I40EVF_WRITE_FLUSH(hw);
+	}
+
 	return 0;
 
 err_alloc:
@@ -1236,11 +1302,141 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   __rte_unused uint16_t msglen)
+{
+	struct i40e_virtchnl_pf_event *pf_msg =
+			(struct i40e_virtchnl_pf_event *)msg;
+
+	switch (pf_msg->event) {
+	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
+		break;
+	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
+		break;
+	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
+		break;
+	default:
+		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
+		break;
+	}
+}
+
+static void
+i40evf_handle_aq_msg(struct rte_eth_dev *dev)
+{
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info info;
+	struct i40e_virtchnl_msg *v_msg;
+	uint16_t pending, opcode;
+	int ret;
+
+	info.buf_len = I40E_AQ_BUF_SZ;
+	if (!vf->aq_resp) {
+		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
+		return;
+	}
+	info.msg_buf = vf->aq_resp;
+	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
+
+	pending = 1;
+	while (pending) {
+		ret = i40e_clean_arq_element(hw, &info, &pending);
+
+		if (ret != I40E_SUCCESS) {
+			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
+				    "ret: %d", ret);
+			break;
+		}
+		opcode = rte_le_to_cpu_16(info.desc.opcode);
+
+		switch (opcode) {
+		case i40e_aqc_opc_send_msg_to_vf:
+			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
+				/* process event*/
+				i40evf_handle_pf_event(dev, info.msg_buf,
+						       info.msg_len);
+			else {
+				/* read message and it's expected one */
+				if (v_msg->v_opcode == vf->pend_cmd) {
+					vf->cmd_retval = v_msg->v_retval;
+					/* prevent compiler reordering */
+					rte_compiler_barrier();
+					_clear_cmd(vf);
+				} else
+					PMD_DRV_LOG(ERR, "command mismatch,"
+						"expect %u, get %u",
+						vf->pend_cmd, v_msg->v_opcode);
+				PMD_DRV_LOG(DEBUG, "adminq response is received,"
+					     " opcode = %d\n", v_msg->v_opcode);
+			}
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
+				    opcode);
+			break;
+		}
+	}
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt. Only adminq interrupt is processed in VF.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
+			     void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	i40evf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
+		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
+		goto done;
+	}
+
+	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
+		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
+		i40evf_handle_aq_msg(dev);
+	}
+
+	/* Link Status Change interrupt */
+	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
+		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
+				   " do nothing\n");
+
+done:
+	i40evf_enable_irq0(hw);
+	rte_intr_enable(&dev->pci_dev->intr_handle);
+}
+
 static int
 i40evf_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
 			eth_dev->data->dev_private);
+	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1275,7 +1471,17 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
-	/* allocate memory for mac addr storage */
+	/* register callback func to eal lib */
+	rte_intr_callback_register(&pci_dev->intr_handle,
+		i40evf_dev_interrupt_handler, (void *)eth_dev);
+
+	/* enable uio intr after callback register */
+	rte_intr_enable(&pci_dev->intr_handle);
+
+	/* configure and enable device interrupt */
+	i40evf_enable_irq0(hw);
+
+	/* copy mac addr */
 	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
 					ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX,
 					0);
@@ -1654,7 +1860,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 		I40E_WRITE_REG(hw,
 			       I40E_VFINT_DYN_CTL01,
 			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
-			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1665,11 +1872,11 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
 			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
-	else
-		/* To support Linux PF host */
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
-				I40E_VFINT_DYN_CTL01_INTENA_MASK |
-				I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't set
+	 * I40E_VFINT_DYN_CTL01,
+	 * because it is already done in i40evf_enable_irq0.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1682,7 +1889,8 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 
 	if (!rte_intr_allow_others(intr_handle)) {
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1692,8 +1900,11 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
 						    - 1),
 			       0);
-	else
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't zero
+	 * I40E_VFINT_DYN_CTL01,
+	 * because interrupt 0 is also used for adminq processing.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1868,10 +2079,6 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* vf don't allow intr except for rxq intr */
-	if (dev->data->dev_conf.intr_conf.rxq != 0)
-		rte_intr_enable(intr_handle);
-
 	i40evf_enable_queues_intr(dev);
 	return 0;
 
@@ -2060,12 +2267,20 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = dev->pci_dev;
 
 	i40evf_dev_stop(dev);
 	hw->adapter_stopped = 1;
 	i40e_dev_free_queues(dev);
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(&pci_dev->intr_handle);
+
+	/* unregister callback func from eal lib */
+	rte_intr_callback_unregister(&pci_dev->intr_handle,
+		i40evf_dev_interrupt_handler, (void *)dev);
+	i40evf_disable_irq0(hw);
 }
 
 static int
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index e2893ba..0c81435 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -2663,6 +2663,7 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
 enum rte_eth_event_type {
 	RTE_ETH_EVENT_UNKNOWN,  /**< unknown event type */
 	RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
+	RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.4.0

Re: [PATCH v6 0/2] i40evf: pf reset event report

[Bruce Richardson]

On Tue, Mar 15, 2016 at 09:59:38AM +0800, Jingjing Wu wrote:
> v6 changes:
>  - rebase on latest dpdk-next-net/rel_16_04 branch (commit 94d8f4bf963a)
>  - fix complie issue
> 
> v5 changes:
>  - doc rewording and format fixing.
> 
> v4 changes:
>  - rebase on latest dpdk-next-net/rel_16_04 branch (commit 0f9564a0e4f2)
> 
> v3 changes:
>  - commit log doc rewording.
>  - rebase on latest dpdk-next-net/rel_16_04 branch.
>  - remove few useless line.
>  - adjust interval and increase times for waiting pf msg
> 
> v2 changes:
>  - remove the change on vf reset status checking
>  - add pf event report support in release note
> 
> When Linux PF and DPDK VF are used for i40e PMD, In case of PF
> reset, interrupt request will go via adminq event, VF need be
> informed, a callback mechanism is introduced by VF. This will
> allow VF to invoke callback when reset happens.
> Users can register a callback for this interrupt event like:
>     rte_eth_dev_callback_register(portid,
>                  RTE_ETH_EVENT_INTR_RESET,
>                  reset_event_callback,
>                  arg);
> 
> 
> Jingjing Wu (2):
>   i40evf: allocate virtchnl cmd buffer for each vf
>   i40evf: support to report pf reset event
> 
Applied to dpdk-next-net/rel_16_04.

/Bruce

[PATCH v2 2/2] i40evf: support interrupt based pf reset request

[Jingjing Wu]

Interrupt based request of PF reset from PF is supported by
enabling the adminq event process in VF driver.
Users can register a callback for this interrupt event to get
informed, when a PF reset request detected like:
  rte_eth_dev_callback_register(portid,
		RTE_ETH_EVENT_INTR_RESET,
		reset_event_callback,
		arg);

Signed-off-by: Jingjing Wu <jingjing.wu@intel.com>
---
 doc/guides/rel_notes/release_2_3.rst |   1 +
 drivers/net/i40e/i40e_ethdev_vf.c    | 274 +++++++++++++++++++++++++++++++----
 lib/librte_ether/rte_ethdev.h        |   1 +
 3 files changed, 246 insertions(+), 30 deletions(-)

diff --git a/doc/guides/rel_notes/release_2_3.rst b/doc/guides/rel_notes/release_2_3.rst
index 99de186..73d5f76 100644
--- a/doc/guides/rel_notes/release_2_3.rst
+++ b/doc/guides/rel_notes/release_2_3.rst
@@ -4,6 +4,7 @@ DPDK Release 2.3
 New Features
 ------------
 
+* **Added pf reset event reported in i40e vf PMD driver.
 
 Resolved Issues
 ---------------
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 64e6957..1ffe64e 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -74,8 +74,6 @@
 #define I40EVF_BUSY_WAIT_DELAY 10
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
-/*ITR index for NOITR*/
-#define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
 
 struct i40evf_arq_msg_info {
 	enum i40e_virtchnl_ops ops;
@@ -151,6 +149,9 @@ static int
 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 static int
 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
+static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   uint16_t msglen);
 
 /* Default hash key buffer for RSS */
 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
@@ -357,20 +358,42 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 		return err;
 	}
 
-	do {
-		/* Delay some time first */
-		rte_delay_ms(ASQ_DELAY_MS);
-		ret = i40evf_read_pfmsg(dev, &info);
-		if (ret == I40EVF_MSG_CMD) {
-			err = 0;
-			break;
-		} else if (ret == I40EVF_MSG_ERR) {
-			err = -1;
-			break;
-		}
-		/* If don't read msg or read sys event, continue */
-	} while (i++ < MAX_TRY_TIMES);
-	_clear_cmd(vf);
+	switch (args->ops) {
+	case I40E_VIRTCHNL_OP_RESET_VF:
+		/*no need to process in this function */
+		break;
+	case I40E_VIRTCHNL_OP_VERSION:
+	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
+		/* for init adminq commands, need to poll the response */
+		do {
+			/* Delay some time first */
+			rte_delay_ms(ASQ_DELAY_MS);
+			ret = i40evf_read_pfmsg(dev, &info);
+			if (ret == I40EVF_MSG_CMD) {
+				err = 0;
+				break;
+			} else if (ret == I40EVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		_clear_cmd(vf);
+		break;
+
+	default:
+		/* for other adminq in running time, waiting the cmd done flag */
+		do {
+			/* Delay some time first */
+			rte_delay_ms(ASQ_DELAY_MS);
+			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
+				err = 0;
+				break;
+			}
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		break;
+	}
 
 	return (err | vf->cmd_retval);
 }
@@ -719,7 +742,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev)
 
 	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = 1;
-	map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
+	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 	/* Alway use default dynamic MSIX interrupt */
 	map_info->vecmap[0].vector_id = vector_id;
@@ -1093,6 +1116,38 @@ i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/* Disable IRQ0 */
+static inline void
+i40evf_disable_irq0(struct i40e_hw *hw)
+{
+	/* Disable all interrupt types */
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	I40EVF_WRITE_FLUSH(hw);
+}
+
+/* Enable IRQ0 */
+static inline void
+i40evf_enable_irq0(struct i40e_hw *hw)
+{
+	/* Enable admin queue interrupt trigger */
+	uint32_t val;
+
+	i40evf_disable_irq0(hw);
+	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
+	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
+		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
+	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
+
+	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+		I40E_VFINT_DYN_CTL01_INTENA_MASK |
+		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+
+	I40EVF_WRITE_FLUSH(hw);
+}
+
 static int
 i40evf_reset_vf(struct i40e_hw *hw)
 {
@@ -1137,6 +1192,8 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	int i, err, bufsz;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	uint16_t interval =
+		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1218,6 +1275,15 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	ether_addr_copy((struct ether_addr *)vf->vsi_res->default_mac_addr,
 					(struct ether_addr *)hw->mac.addr);
 
+	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
+	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       (I40E_ITR_INDEX_DEFAULT <<
+				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
+			       (interval <<
+				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
+	}
+
 	return 0;
 
 err_alloc:
@@ -1246,11 +1312,142 @@ i40evf_uninit_vf(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
+			   uint8_t *msg,
+			   __rte_unused uint16_t msglen)
+{
+	struct i40e_virtchnl_pf_event *pf_msg =
+			(struct i40e_virtchnl_pf_event *)msg;
+
+	switch (pf_msg->event) {
+	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
+		break;
+	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
+		break;
+	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
+		break;
+	default:
+		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
+		break;
+	}
+}
+
+static void
+i40evf_handle_aq_msg(struct rte_eth_dev *dev)
+{
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
+	struct i40e_arq_event_info info;
+	struct i40e_virtchnl_msg *v_msg;
+	uint16_t pending, opcode;
+	int ret;
+
+	info.buf_len = I40E_AQ_BUF_SZ;
+	if (!vf->aq_resp) {
+		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
+		return;
+	}
+	info.msg_buf = vf->aq_resp;
+	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
+
+	pending = 1;
+	while (pending) {
+		ret = i40e_clean_arq_element(hw, &info, &pending);
+
+		if (ret != I40E_SUCCESS) {
+			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
+				    "ret: %d", ret);
+			break;
+		}
+		opcode = rte_le_to_cpu_16(info.desc.opcode);
+
+		switch (opcode) {
+		case i40e_aqc_opc_send_msg_to_vf:
+			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
+				/* process event*/
+				i40evf_handle_pf_event(dev, info.msg_buf,
+							info.msg_len);
+			else {
+				/* read message and it's expected one */
+				if (v_msg->v_opcode == vf->pend_cmd) {
+					vf->cmd_retval = v_msg->v_retval;
+					/* prevent compiler reordering */
+					rte_compiler_barrier();
+					_clear_cmd(vf);
+				} else
+					PMD_DRV_LOG(ERR, "command mismatch,"
+						"expect %u, get %u",
+						vf->pend_cmd, v_msg->v_opcode);
+				 PMD_DRV_LOG(DEBUG, "adminq response is received,"
+					     " opcode = %d\n", v_msg->v_opcode);
+			}
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
+				    opcode);
+			break;
+		}
+	}
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt. Only adminq interrupt is processed in VF.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
+			   void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	i40evf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
+		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
+		goto done;
+	}
+
+	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
+		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
+		i40evf_handle_aq_msg(dev);
+	}
+
+	/* Link Status Change interrupt */
+	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
+		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
+				   " do nothing\n");
+
+done:
+	i40evf_enable_irq0(hw);
+	rte_intr_enable(&(dev->pci_dev->intr_handle));
+}
+
+
 static int
 i40evf_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
 			eth_dev->data->dev_private);
+	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1285,6 +1482,16 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
+	/* register callback func to eal lib */
+	rte_intr_callback_register(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)eth_dev);
+
+	/* configure and enable device interrupt */
+	i40evf_enable_irq0(hw);
+	/* intr is enabled in i40evf_enable_queues_intr when dev_start */
+
+	/* enable uio intr after callback register */
+	rte_intr_enable(&(pci_dev->intr_handle));
 	/* copy mac addr */
 	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
 					ETHER_ADDR_LEN, 0);
@@ -1662,7 +1869,8 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 		I40E_WRITE_REG(hw,
 			       I40E_VFINT_DYN_CTL01,
 			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
-			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1673,11 +1881,10 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev)
 			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
 			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
-	else
-		/* To support Linux PF host */
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
-				I40E_VFINT_DYN_CTL01_INTENA_MASK |
-				I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't set I40E_VFINT_DYN_CTL01,
+	 * because it is already done in i40evf_enable_irq0.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1690,7 +1897,8 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 
 	if (!rte_intr_allow_others(intr_handle)) {
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
+			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 		I40EVF_WRITE_FLUSH(hw);
 		return;
 	}
@@ -1700,8 +1908,10 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev)
 			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
 						    - 1),
 			       0);
-	else
-		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
+	/* If host driver is kernel driver, do nothing.
+	 * Interrupt 0 is used for rx packets, but don't zero I40E_VFINT_DYN_CTL01,
+	 * because interrupt 0 is also used for adminq processing.
+	 */
 
 	I40EVF_WRITE_FLUSH(hw);
 }
@@ -1825,10 +2035,6 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* vf don't allow intr except for rxq intr */
-	if (dev->data->dev_conf.intr_conf.rxq != 0)
-		rte_intr_enable(intr_handle);
-
 	i40evf_enable_queues_intr(dev);
 	return 0;
 
@@ -2020,12 +2226,20 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = dev->pci_dev;
 
 	i40evf_dev_stop(dev);
 	hw->adapter_stopped = 1;
 	i40e_dev_free_queues(dev);
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(&(pci_dev->intr_handle));
+
+	/* unregister callback func from eal lib */
+	rte_intr_callback_unregister(&(pci_dev->intr_handle),
+		i40evf_dev_interrupt_handler, (void *)dev);
+	i40evf_disable_irq0(hw);
 }
 
 static int
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index bada8ad..1be1783 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -2666,6 +2666,7 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
 enum rte_eth_event_type {
 	RTE_ETH_EVENT_UNKNOWN,  /**< unknown event type */
 	RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
+	RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.4.0