[dpdk-dev] [PATCH v4 38/58] net/txgbe: add PF module init and uninit for SRIOV

[Jiawen Wu <jiawenwu@trustnetic.com>]

Add PF module init and uninit operations with mailbox.

Signed-off-by: Jiawen Wu <jiawenwu@trustnetic.com>
---
 doc/guides/nics/features/txgbe.ini  |   1 +
 drivers/net/txgbe/base/meson.build  |   1 +
 drivers/net/txgbe/base/txgbe.h      |   1 +
 drivers/net/txgbe/base/txgbe_hw.c   |   4 +
 drivers/net/txgbe/base/txgbe_mbx.c  |  29 ++++++
 drivers/net/txgbe/base/txgbe_mbx.h  |  12 +++
 drivers/net/txgbe/base/txgbe_type.h |  14 +++
 drivers/net/txgbe/meson.build       |   1 +
 drivers/net/txgbe/txgbe_ethdev.c    |  15 +++
 drivers/net/txgbe/txgbe_ethdev.h    |  21 ++++
 drivers/net/txgbe/txgbe_pf.c        | 142 ++++++++++++++++++++++++++++
 11 files changed, 241 insertions(+)
 create mode 100644 drivers/net/txgbe/base/txgbe_mbx.c
 create mode 100644 drivers/net/txgbe/base/txgbe_mbx.h
 create mode 100644 drivers/net/txgbe/txgbe_pf.c

diff --git a/doc/guides/nics/features/txgbe.ini b/doc/guides/nics/features/txgbe.ini
index bd39d6642..1d00f3105 100644
--- a/doc/guides/nics/features/txgbe.ini
+++ b/doc/guides/nics/features/txgbe.ini
@@ -15,6 +15,7 @@ LRO                  = Y
 TSO                  = Y
 Unicast MAC filter   = Y
 Multicast MAC filter = Y
+SR-IOV               = Y
 VLAN filter          = Y
 CRC offload          = P
 VLAN offload         = P
diff --git a/drivers/net/txgbe/base/meson.build b/drivers/net/txgbe/base/meson.build
index d0100714e..6490d4ed8 100644
--- a/drivers/net/txgbe/base/meson.build
+++ b/drivers/net/txgbe/base/meson.build
@@ -4,6 +4,7 @@
 sources = [
 	'txgbe_eeprom.c',
 	'txgbe_hw.c',
+	'txgbe_mbx.c',
 	'txgbe_mng.c',
 	'txgbe_phy.c',
 ]
diff --git a/drivers/net/txgbe/base/txgbe.h b/drivers/net/txgbe/base/txgbe.h
index 764caa439..bf95d5681 100644
--- a/drivers/net/txgbe/base/txgbe.h
+++ b/drivers/net/txgbe/base/txgbe.h
@@ -7,6 +7,7 @@
 
 #include "txgbe_type.h"
 #include "txgbe_mng.h"
+#include "txgbe_mbx.h"
 #include "txgbe_eeprom.h"
 #include "txgbe_phy.h"
 #include "txgbe_hw.h"
diff --git a/drivers/net/txgbe/base/txgbe_hw.c b/drivers/net/txgbe/base/txgbe_hw.c
index 1b1d1525f..3a9e57024 100644
--- a/drivers/net/txgbe/base/txgbe_hw.c
+++ b/drivers/net/txgbe/base/txgbe_hw.c
@@ -3,6 +3,7 @@
  */
 
 #include "txgbe_type.h"
+#include "txgbe_mbx.h"
 #include "txgbe_phy.h"
 #include "txgbe_eeprom.h"
 #include "txgbe_mng.h"
@@ -1701,6 +1702,7 @@ s32 txgbe_init_ops_pf(struct txgbe_hw *hw)
 	struct txgbe_mac_info *mac = &hw->mac;
 	struct txgbe_phy_info *phy = &hw->phy;
 	struct txgbe_rom_info *rom = &hw->rom;
+	struct txgbe_mbx_info *mbx = &hw->mbx;
 
 	DEBUGFUNC("txgbe_init_ops_pf");
 
@@ -1762,6 +1764,8 @@ s32 txgbe_init_ops_pf(struct txgbe_hw *hw)
 	mac->get_thermal_sensor_data = txgbe_get_thermal_sensor_data;
 	mac->init_thermal_sensor_thresh = txgbe_init_thermal_sensor_thresh;
 
+	mbx->init_params = txgbe_init_mbx_params_pf;
+
 	/* EEPROM */
 	rom->init_params = txgbe_init_eeprom_params;
 	rom->read16 = txgbe_ee_read16;
diff --git a/drivers/net/txgbe/base/txgbe_mbx.c b/drivers/net/txgbe/base/txgbe_mbx.c
new file mode 100644
index 000000000..e53024674
--- /dev/null
+++ b/drivers/net/txgbe/base/txgbe_mbx.c
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2020
+ */
+
+#include "txgbe_type.h"
+
+#include "txgbe_mbx.h"
+
+/**
+ *  txgbe_init_mbx_params_pf - set initial values for pf mailbox
+ *  @hw: pointer to the HW structure
+ *
+ *  Initializes the hw->mbx struct to correct values for pf mailbox
+ */
+void txgbe_init_mbx_params_pf(struct txgbe_hw *hw)
+{
+	struct txgbe_mbx_info *mbx = &hw->mbx;
+
+	mbx->timeout = 0;
+	mbx->usec_delay = 0;
+
+	mbx->size = TXGBE_P2VMBX_SIZE;
+
+	mbx->stats.msgs_tx = 0;
+	mbx->stats.msgs_rx = 0;
+	mbx->stats.reqs = 0;
+	mbx->stats.acks = 0;
+	mbx->stats.rsts = 0;
+}
diff --git a/drivers/net/txgbe/base/txgbe_mbx.h b/drivers/net/txgbe/base/txgbe_mbx.h
new file mode 100644
index 000000000..ab2c5dd97
--- /dev/null
+++ b/drivers/net/txgbe/base/txgbe_mbx.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2020
+ */
+
+#ifndef _TXGBE_MBX_H_
+#define _TXGBE_MBX_H_
+
+#include "txgbe_type.h"
+
+void txgbe_init_mbx_params_pf(struct txgbe_hw *);
+
+#endif /* _TXGBE_MBX_H_ */
diff --git a/drivers/net/txgbe/base/txgbe_type.h b/drivers/net/txgbe/base/txgbe_type.h
index 539d1fb4d..234b79266 100644
--- a/drivers/net/txgbe/base/txgbe_type.h
+++ b/drivers/net/txgbe/base/txgbe_type.h
@@ -580,6 +580,15 @@ struct txgbe_phy_info {
 	u32 link_mode;
 };
 
+struct txgbe_mbx_stats {
+	u32 msgs_tx;
+	u32 msgs_rx;
+
+	u32 acks;
+	u32 reqs;
+	u32 rsts;
+};
+
 struct txgbe_mbx_info {
 	void (*init_params)(struct txgbe_hw *hw);
 	s32  (*read)(struct txgbe_hw *hw, u32 *msg, u16 size, u16 vf_number);
@@ -591,6 +600,11 @@ struct txgbe_mbx_info {
 	s32  (*check_for_msg)(struct txgbe_hw *hw, u16 mbx_id);
 	s32  (*check_for_ack)(struct txgbe_hw *hw, u16 mbx_id);
 	s32  (*check_for_rst)(struct txgbe_hw *hw, u16 mbx_id);
+
+	struct txgbe_mbx_stats stats;
+	u32 timeout;
+	u32 usec_delay;
+	u16 size;
 };
 
 enum txgbe_isb_idx {
diff --git a/drivers/net/txgbe/meson.build b/drivers/net/txgbe/meson.build
index ea028d208..62f650504 100644
--- a/drivers/net/txgbe/meson.build
+++ b/drivers/net/txgbe/meson.build
@@ -7,6 +7,7 @@ objs = [base_objs]
 sources = files(
 	'txgbe_ethdev.c',
 	'txgbe_ptypes.c',
+	'txgbe_pf.c',
 	'txgbe_rxtx.c',
 )
 
diff --git a/drivers/net/txgbe/txgbe_ethdev.c b/drivers/net/txgbe/txgbe_ethdev.c
index f6154cd04..f98677f7a 100644
--- a/drivers/net/txgbe/txgbe_ethdev.c
+++ b/drivers/net/txgbe/txgbe_ethdev.c
@@ -368,6 +368,7 @@ eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
 	struct txgbe_hwstrip *hwstrip = TXGBE_DEV_HWSTRIP(eth_dev);
 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 	const struct rte_memzone *mz;
+	uint32_t ctrl_ext;
 	uint16_t csum;
 	int err;
 
@@ -515,6 +516,17 @@ eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
 	/* initialize the hw strip bitmap*/
 	memset(hwstrip, 0, sizeof(*hwstrip));
 
+	/* initialize PF if max_vfs not zero */
+	txgbe_pf_host_init(eth_dev);
+
+	ctrl_ext = rd32(hw, TXGBE_PORTCTL);
+	/* let hardware know driver is loaded */
+	ctrl_ext |= TXGBE_PORTCTL_DRVLOAD;
+	/* Set PF Reset Done bit so PF/VF Mail Ops can work */
+	ctrl_ext |= TXGBE_PORTCTL_RSTDONE;
+	wr32(hw, TXGBE_PORTCTL, ctrl_ext);
+	txgbe_flush(hw);
+
 	if (txgbe_is_sfp(hw) && hw->phy.sfp_type != txgbe_sfp_type_not_present)
 		PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
 			     (int)hw->mac.type, (int)hw->phy.type,
@@ -1502,6 +1514,9 @@ txgbe_dev_close(struct rte_eth_dev *dev)
 	/* cancel the delay handler before remove dev */
 	rte_eal_alarm_cancel(txgbe_dev_interrupt_delayed_handler, dev);
 
+	/* uninitialize PF if max_vfs not zero */
+	txgbe_pf_host_uninit(dev);
+
 	rte_free(dev->data->mac_addrs);
 	dev->data->mac_addrs = NULL;
 
diff --git a/drivers/net/txgbe/txgbe_ethdev.h b/drivers/net/txgbe/txgbe_ethdev.h
index ad57e0b61..1f64b2e1d 100644
--- a/drivers/net/txgbe/txgbe_ethdev.h
+++ b/drivers/net/txgbe/txgbe_ethdev.h
@@ -80,6 +80,18 @@ struct txgbe_uta_info {
 	uint32_t uta_shadow[TXGBE_MAX_UTA];
 };
 
+#define TXGBE_MAX_MIRROR_RULES 4  /* Maximum nb. of mirror rules. */
+
+struct txgbe_mirror_info {
+	struct rte_eth_mirror_conf mr_conf[TXGBE_MAX_MIRROR_RULES];
+	/* store PF mirror rules configuration */
+};
+
+struct txgbe_vf_info {
+	uint8_t vf_mac_addresses[RTE_ETHER_ADDR_LEN];
+	uint16_t switch_domain_id;
+};
+
 /*
  * Structure to store private data for each driver instance (for each port).
  */
@@ -90,6 +102,8 @@ struct txgbe_adapter {
 	struct txgbe_stat_mappings  stat_mappings;
 	struct txgbe_vfta           shadow_vfta;
 	struct txgbe_hwstrip        hwstrip;
+	struct txgbe_mirror_info    mr_data;
+	struct txgbe_vf_info        *vfdata;
 	struct txgbe_uta_info       uta_info;
 	bool rx_bulk_alloc_allowed;
 };
@@ -118,6 +132,9 @@ struct txgbe_adapter {
 #define TXGBE_DEV_VFDATA(dev) \
 	(&((struct txgbe_adapter *)(dev)->data->dev_private)->vfdata)
 
+#define TXGBE_DEV_MR_INFO(dev) \
+	(&((struct txgbe_adapter *)(dev)->data->dev_private)->mr_data)
+
 #define TXGBE_DEV_UTA_INFO(dev) \
 	(&((struct txgbe_adapter *)(dev)->data->dev_private)->uta_info)
 
@@ -192,6 +209,10 @@ void txgbe_set_ivar_map(struct txgbe_hw *hw, int8_t direction,
 int
 txgbe_dev_link_update_share(struct rte_eth_dev *dev,
 		int wait_to_complete);
+void txgbe_pf_host_init(struct rte_eth_dev *eth_dev);
+
+void txgbe_pf_host_uninit(struct rte_eth_dev *eth_dev);
+
 
 #define TXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
 #define TXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
diff --git a/drivers/net/txgbe/txgbe_pf.c b/drivers/net/txgbe/txgbe_pf.c
new file mode 100644
index 000000000..7e48b2fd0
--- /dev/null
+++ b/drivers/net/txgbe/txgbe_pf.c
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2020
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <inttypes.h>
+
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_eal.h>
+#include <rte_ether.h>
+#include <rte_ethdev_driver.h>
+#include <rte_memcpy.h>
+#include <rte_malloc.h>
+#include <rte_random.h>
+#include <rte_bus_pci.h>
+
+#include "base/txgbe.h"
+#include "txgbe_ethdev.h"
+
+static inline uint16_t
+dev_num_vf(struct rte_eth_dev *eth_dev)
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+
+	return pci_dev->max_vfs;
+}
+
+static inline
+int txgbe_vf_perm_addr_gen(struct rte_eth_dev *dev, uint16_t vf_num)
+{
+	unsigned char vf_mac_addr[RTE_ETHER_ADDR_LEN];
+	struct txgbe_vf_info *vfinfo = *TXGBE_DEV_VFDATA(dev);
+	uint16_t vfn;
+
+	for (vfn = 0; vfn < vf_num; vfn++) {
+		rte_eth_random_addr(vf_mac_addr);
+		/* keep the random address as default */
+		memcpy(vfinfo[vfn].vf_mac_addresses, vf_mac_addr,
+			   RTE_ETHER_ADDR_LEN);
+	}
+
+	return 0;
+}
+
+static inline int
+txgbe_mb_intr_setup(struct rte_eth_dev *dev)
+{
+	struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
+
+	intr->mask_misc |= TXGBE_ICRMISC_VFMBX;
+
+	return 0;
+}
+
+void txgbe_pf_host_init(struct rte_eth_dev *eth_dev)
+{
+	struct txgbe_vf_info **vfinfo = TXGBE_DEV_VFDATA(eth_dev);
+	struct txgbe_mirror_info *mirror_info = TXGBE_DEV_MR_INFO(eth_dev);
+	struct txgbe_uta_info *uta_info = TXGBE_DEV_UTA_INFO(eth_dev);
+	struct txgbe_hw *hw = TXGBE_DEV_HW(eth_dev);
+	uint16_t vf_num;
+	uint8_t nb_queue;
+
+	PMD_INIT_FUNC_TRACE();
+
+	RTE_ETH_DEV_SRIOV(eth_dev).active = 0;
+	vf_num = dev_num_vf(eth_dev);
+	if (vf_num == 0)
+		return;
+
+	*vfinfo = rte_zmalloc("vf_info",
+			sizeof(struct txgbe_vf_info) * vf_num, 0);
+	if (*vfinfo == NULL)
+		rte_panic("Cannot allocate memory for private VF data\n");
+
+	rte_eth_switch_domain_alloc(&(*vfinfo)->switch_domain_id);
+
+	memset(mirror_info, 0, sizeof(struct txgbe_mirror_info));
+	memset(uta_info, 0, sizeof(struct txgbe_uta_info));
+	hw->mac.mc_filter_type = 0;
+
+	if (vf_num >= ETH_32_POOLS) {
+		nb_queue = 2;
+		RTE_ETH_DEV_SRIOV(eth_dev).active = ETH_64_POOLS;
+	} else if (vf_num >= ETH_16_POOLS) {
+		nb_queue = 4;
+		RTE_ETH_DEV_SRIOV(eth_dev).active = ETH_32_POOLS;
+	} else {
+		nb_queue = 8;
+		RTE_ETH_DEV_SRIOV(eth_dev).active = ETH_16_POOLS;
+	}
+
+	RTE_ETH_DEV_SRIOV(eth_dev).nb_q_per_pool = nb_queue;
+	RTE_ETH_DEV_SRIOV(eth_dev).def_vmdq_idx = vf_num;
+	RTE_ETH_DEV_SRIOV(eth_dev).def_pool_q_idx =
+			(uint16_t)(vf_num * nb_queue);
+
+	txgbe_vf_perm_addr_gen(eth_dev, vf_num);
+
+	/* init_mailbox_params */
+	hw->mbx.init_params(hw);
+
+	/* set mb interrupt mask */
+	txgbe_mb_intr_setup(eth_dev);
+}
+
+void txgbe_pf_host_uninit(struct rte_eth_dev *eth_dev)
+{
+	struct txgbe_vf_info **vfinfo;
+	uint16_t vf_num;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+
+	RTE_ETH_DEV_SRIOV(eth_dev).active = 0;
+	RTE_ETH_DEV_SRIOV(eth_dev).nb_q_per_pool = 0;
+	RTE_ETH_DEV_SRIOV(eth_dev).def_vmdq_idx = 0;
+	RTE_ETH_DEV_SRIOV(eth_dev).def_pool_q_idx = 0;
+
+	vf_num = dev_num_vf(eth_dev);
+	if (vf_num == 0)
+		return;
+
+	vfinfo = TXGBE_DEV_VFDATA(eth_dev);
+	if (*vfinfo == NULL)
+		return;
+
+	ret = rte_eth_switch_domain_free((*vfinfo)->switch_domain_id);
+	if (ret)
+		PMD_INIT_LOG(WARNING, "failed to free switch domain: %d", ret);
+
+	rte_free(*vfinfo);
+	*vfinfo = NULL;
+}
+
-- 
2.18.4