[PATCH 0/8] bnxt patchset

[PATCH 0/8] bnxt patchset

[Ajit Khaparde]

This patchset adds:
  - support for xstats get by id
  - support for rx_queue_count
  - support for rx_descriptor_status
  - support for rx_descriptor_done
  - support for tx_descriptor_status
  - support for flow filter ops
  - new HWRM structures which are used by the flow filtering functions
It also fixes the HWRM_*() macros and locking to ensure that there is
only one outstanding command with the firmware.

Please consider including them for the upcoming release.

-- 
2.10.1 (Apple Git-78)

[PATCH 1/8] net/bnxt: add support for xstats get by id

[Ajit Khaparde]

This patch adds support for xstats_get_by_id/xstats_get_names_by_id.
Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/bnxt_ethdev.c |  2 ++
 drivers/net/bnxt/bnxt_stats.c  | 44 ++++++++++++++++++++++++++++++++++++++++++
 drivers/net/bnxt/bnxt_stats.h  |  5 +++++
 3 files changed, 51 insertions(+)

diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index c9d1122..1302710 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -1564,6 +1564,8 @@ static const struct eth_dev_ops bnxt_dev_ops = {
 	.txq_info_get = bnxt_txq_info_get_op,
 	.dev_led_on = bnxt_dev_led_on_op,
 	.dev_led_off = bnxt_dev_led_off_op,
+	.xstats_get_by_id = bnxt_dev_xstats_get_by_id_op,
+	.xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op,
 };
 
 static bool bnxt_vf_pciid(uint16_t id)
diff --git a/drivers/net/bnxt/bnxt_stats.c b/drivers/net/bnxt/bnxt_stats.c
index d7d0e35..46fc3c4 100644
--- a/drivers/net/bnxt/bnxt_stats.c
+++ b/drivers/net/bnxt/bnxt_stats.c
@@ -358,3 +358,47 @@ void bnxt_dev_xstats_reset_op(struct rte_eth_dev *eth_dev)
 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
 		RTE_LOG(ERR, PMD, "Operation not supported\n");
 }
+
+int bnxt_dev_xstats_get_by_id_op(struct rte_eth_dev *dev, const uint64_t *ids,
+		uint64_t *values, unsigned int limit)
+{
+	/* Account for the Tx drop pkts aka the Anti spoof counter */
+	const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) +
+				RTE_DIM(bnxt_tx_stats_strings) + 1;
+	struct rte_eth_xstat xstats[stat_cnt];
+	uint16_t i;
+
+	bnxt_dev_xstats_get_op(dev, xstats, limit);
+
+	for (i = 0; i < limit; i++) {
+		if (ids[i] >= stat_cnt) {
+			RTE_LOG(ERR, PMD, "id value isn't valid");
+			return -1;
+		}
+		values[i] = xstats[ids[i]].value;
+	}
+	return limit;
+}
+
+int bnxt_dev_xstats_get_names_by_id_op(struct rte_eth_dev *dev,
+				struct rte_eth_xstat_name *xstats_names,
+				const uint64_t *ids, unsigned int limit)
+{
+	/* Account for the Tx drop pkts aka the Anti spoof counter */
+	const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) +
+				RTE_DIM(bnxt_tx_stats_strings) + 1;
+	struct rte_eth_xstat_name xstats_names_copy[stat_cnt];
+	uint16_t i;
+
+	bnxt_dev_xstats_get_names_op(dev, xstats_names_copy, limit);
+
+	for (i = 0; i < limit; i++) {
+		if (ids[i] >= stat_cnt) {
+			RTE_LOG(ERR, PMD, "id value isn't valid");
+			return -1;
+		}
+		strcpy(xstats_names[i].name,
+				xstats_names_copy[ids[i]].name);
+	}
+	return limit;
+}
diff --git a/drivers/net/bnxt/bnxt_stats.h b/drivers/net/bnxt/bnxt_stats.h
index b6d133e..daeb3d9 100644
--- a/drivers/net/bnxt/bnxt_stats.h
+++ b/drivers/net/bnxt/bnxt_stats.h
@@ -46,6 +46,11 @@ int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev,
 int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev,
 			   struct rte_eth_xstat *xstats, unsigned int n);
 void bnxt_dev_xstats_reset_op(struct rte_eth_dev *eth_dev);
+int bnxt_dev_xstats_get_by_id_op(struct rte_eth_dev *dev, const uint64_t *ids,
+				uint64_t *values, unsigned int limit);
+int bnxt_dev_xstats_get_names_by_id_op(struct rte_eth_dev *dev,
+				struct rte_eth_xstat_name *xstats_names,
+				const uint64_t *ids, unsigned int limit);
 
 struct bnxt_xstats_name_off {
 	char name[RTE_ETH_XSTATS_NAME_SIZE];
-- 
2.10.1 (Apple Git-78)

Re: [PATCH 1/8] net/bnxt: add support for xstats get by id

[Ferruh Yigit]

On 8/24/2017 5:29 PM, Ajit Khaparde wrote:
> This patch adds support for xstats_get_by_id/xstats_get_names_by_id.
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

<...>

> +
> +int bnxt_dev_xstats_get_by_id_op(struct rte_eth_dev *dev, const uint64_t *ids,
> +		uint64_t *values, unsigned int limit)
> +{
> +	/* Account for the Tx drop pkts aka the Anti spoof counter */
> +	const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) +
> +				RTE_DIM(bnxt_tx_stats_strings) + 1;
> +	struct rte_eth_xstat xstats[stat_cnt];
> +	uint16_t i;
> +
> +	bnxt_dev_xstats_get_op(dev, xstats, limit);

I guess intention here is to read all xstats, so "stat_cnt" should be
used instead of "limit"

> +
> +	for (i = 0; i < limit; i++) {
> +		if (ids[i] >= stat_cnt) {

Current xstats_get_by_id API, which is an interesting one, can call this
with ids==NULL.

> +			RTE_LOG(ERR, PMD, "id value isn't valid");
> +			return -1;
> +		}
> +		values[i] = xstats[ids[i]].value;
> +	}
> +	return limit;

if limit is zero, API expects to get number of available xstats, so
can't just return limit here.

> +}

<...>

[PATCH 2/8] net/bnxt: add support for rx_queue_count

[Ajit Khaparde]

add support for rx_queue_count dev_op
Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/bnxt_ethdev.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index 1302710..316a167 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -1517,6 +1517,28 @@ bnxt_dev_led_off_op(struct rte_eth_dev *dev)
 	return bnxt_hwrm_port_led_cfg(bp, false);
 }
 
+static uint32_t
+bnxt_rx_queue_count_op(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	uint32_t desc = 0, raw_cons = 0, cons;
+	struct bnxt_cp_ring_info *cpr;
+	struct bnxt_rx_queue *rxq;
+	struct rx_pkt_cmpl *rxcmp;
+
+	rxq = dev->data->rx_queues[rx_queue_id];
+	cpr = rxq->cp_ring;
+
+	while (raw_cons < rxq->nb_rx_desc) {
+		cons = RING_CMP(cpr->cp_ring_struct, raw_cons++);
+		rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
+
+		if (CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct))
+			desc++;
+	}
+
+	return desc;
+}
+
 /*
  * Initialization
  */
@@ -1566,6 +1588,7 @@ static const struct eth_dev_ops bnxt_dev_ops = {
 	.dev_led_off = bnxt_dev_led_off_op,
 	.xstats_get_by_id = bnxt_dev_xstats_get_by_id_op,
 	.xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op,
+	.rx_queue_count = bnxt_rx_queue_count_op,
 };
 
 static bool bnxt_vf_pciid(uint16_t id)
-- 
2.10.1 (Apple Git-78)

Re: [PATCH 2/8] net/bnxt: add support for rx_queue_count

[Ferruh Yigit]

On 8/24/2017 5:29 PM, Ajit Khaparde wrote:
> add support for rx_queue_count dev_op
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

<...>

> +static uint32_t
> +bnxt_rx_queue_count_op(struct rte_eth_dev *dev, uint16_t rx_queue_id)
> +{
> +	uint32_t desc = 0, raw_cons = 0, cons;
> +	struct bnxt_cp_ring_info *cpr;
> +	struct bnxt_rx_queue *rxq;
> +	struct rx_pkt_cmpl *rxcmp;
> +
> +	rxq = dev->data->rx_queues[rx_queue_id];
> +	cpr = rxq->cp_ring;
> +
> +	while (raw_cons < rxq->nb_rx_desc) {
> +		cons = RING_CMP(cpr->cp_ring_struct, raw_cons++);
> +		rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
> +
> +		if (CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct))

"raw_cons" increased above, the "raw_cons" here and the one used to get
"cons" are different. Is this an issue for cmp_valid ?

> +			desc++;
> +	}
> +
> +	return desc;
> +}

<...>

[PATCH 3/8] net/bnxt: add support for rx_descriptor_status

[Ajit Khaparde]

add support for rx_descriptor_status dev_op
Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/bnxt_ethdev.c | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index 316a167..1d2fd78 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -1539,6 +1539,31 @@ bnxt_rx_queue_count_op(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	return desc;
 }
 
+static int
+bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset)
+{
+	struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue;
+	struct bnxt_cp_ring_info *cpr;
+	struct rx_pkt_cmpl *rxcmp;
+	uint32_t cons;
+
+	if (!rxq)
+		return -EINVAL;
+
+	cpr = rxq->cp_ring;
+
+	if (offset >= rxq->nb_rx_desc)
+		return -EINVAL;
+
+	cons = RING_CMP(cpr->cp_ring_struct, offset);
+	rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
+
+	if (CMP_VALID(rxcmp, offset, cpr->cp_ring_struct))
+		return RTE_ETH_RX_DESC_DONE;
+
+	return RTE_ETH_RX_DESC_AVAIL;
+}
+
 /*
  * Initialization
  */
@@ -1589,6 +1614,7 @@ static const struct eth_dev_ops bnxt_dev_ops = {
 	.xstats_get_by_id = bnxt_dev_xstats_get_by_id_op,
 	.xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op,
 	.rx_queue_count = bnxt_rx_queue_count_op,
+	.rx_descriptor_status = bnxt_rx_descriptor_status_op,
 };
 
 static bool bnxt_vf_pciid(uint16_t id)
-- 
2.10.1 (Apple Git-78)

Re: [PATCH 3/8] net/bnxt: add support for rx_descriptor_status

[Ferruh Yigit]

On 8/24/2017 5:29 PM, Ajit Khaparde wrote:
> add support for rx_descriptor_status dev_op
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

<...>

> +static int
> +bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset)
> +{
> +	struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue;
> +	struct bnxt_cp_ring_info *cpr;
> +	struct rx_pkt_cmpl *rxcmp;
> +	uint32_t cons;
> +
> +	if (!rxq)
> +		return -EINVAL;
> +
> +	cpr = rxq->cp_ring;
> +
> +	if (offset >= rxq->nb_rx_desc)
> +		return -EINVAL;
> +
> +	cons = RING_CMP(cpr->cp_ring_struct, offset);
> +	rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
> +
> +	if (CMP_VALID(rxcmp, offset, cpr->cp_ring_struct))
> +		return RTE_ETH_RX_DESC_DONE;
> +
> +	return RTE_ETH_RX_DESC_AVAIL;

Do we need a check to be sure if descriptor is available. Is PMD using
rx_free_thresh?

If so descriptor can be in a state that packet parsed by software but
descriptor not freed yet: RTE_ETH_RX_DESC_UNAVAIL

> +}

<...>

[PATCH 4/8] net/bnxt: add support for rx_descriptor_done

[Ajit Khaparde]

add support for rx_descriptor_done dev_op
Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/bnxt_ethdev.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index 1d2fd78..1b8e4a9 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -1564,6 +1564,28 @@ bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset)
 	return RTE_ETH_RX_DESC_AVAIL;
 }
 
+static int
+bnxt_rx_descriptor_done_op(void *rx_queue, uint16_t offset)
+{
+	struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue;
+	struct bnxt_cp_ring_info *cpr;
+	struct rx_pkt_cmpl *rxcmp;
+	uint32_t cons;
+
+	if (!rxq)
+		return -EINVAL;
+
+	cpr = rxq->cp_ring;
+
+	if (offset >= rxq->nb_rx_desc)
+		return -EINVAL;
+
+	cons = RING_CMP(cpr->cp_ring_struct, offset);
+	rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
+
+	return !!(CMP_VALID(rxcmp, offset, cpr->cp_ring_struct));
+}
+
 /*
  * Initialization
  */
@@ -1615,6 +1637,7 @@ static const struct eth_dev_ops bnxt_dev_ops = {
 	.xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op,
 	.rx_queue_count = bnxt_rx_queue_count_op,
 	.rx_descriptor_status = bnxt_rx_descriptor_status_op,
+	.rx_descriptor_done   = bnxt_rx_descriptor_done_op,
 };
 
 static bool bnxt_vf_pciid(uint16_t id)
-- 
2.10.1 (Apple Git-78)

Re: [PATCH 4/8] net/bnxt: add support for rx_descriptor_done

[Ferruh Yigit]

On 8/24/2017 5:29 PM, Ajit Khaparde wrote:
> add support for rx_descriptor_done dev_op
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

<...>

> @@ -1615,6 +1637,7 @@ static const struct eth_dev_ops bnxt_dev_ops = {
>  	.xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op,
>  	.rx_queue_count = bnxt_rx_queue_count_op,
>  	.rx_descriptor_status = bnxt_rx_descriptor_status_op,
> +	.rx_descriptor_done   = bnxt_rx_descriptor_done_op,

"rx_descriptor_status" is updated version of the "rx_descriptor_done".
Because "rx_descriptor_status" already implemented, this is not required.

>  };
>  
>  static bool bnxt_vf_pciid(uint16_t id)
> 

[PATCH 5/8] net/bnxt: add support for tx_descriptor_status

[Ajit Khaparde]

add support for tx_descriptor_status dev_op
Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/bnxt_ethdev.c | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index 1b8e4a9..088cf6a 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -1586,6 +1586,31 @@ bnxt_rx_descriptor_done_op(void *rx_queue, uint16_t offset)
 	return !!(CMP_VALID(rxcmp, offset, cpr->cp_ring_struct));
 }
 
+static int
+bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset)
+{
+	struct bnxt_tx_queue *txq = (struct bnxt_tx_queue *)tx_queue;
+	struct bnxt_cp_ring_info *cpr;
+	struct tx_pkt_cmpl *txcmp;
+	uint32_t cons;
+
+	if (!txq)
+		return -EINVAL;
+
+	cpr = txq->cp_ring;
+
+	if (offset >= txq->nb_tx_desc)
+		return -EINVAL;
+
+	cons = RING_CMP(cpr->cp_ring_struct, offset);
+	txcmp = (struct tx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
+
+	if (CMP_VALID(txcmp, offset, cpr->cp_ring_struct))
+		return RTE_ETH_TX_DESC_DONE;
+
+	return RTE_ETH_TX_DESC_FULL;
+}
+
 /*
  * Initialization
  */
@@ -1638,6 +1663,7 @@ static const struct eth_dev_ops bnxt_dev_ops = {
 	.rx_queue_count = bnxt_rx_queue_count_op,
 	.rx_descriptor_status = bnxt_rx_descriptor_status_op,
 	.rx_descriptor_done   = bnxt_rx_descriptor_done_op,
+	.tx_descriptor_status = bnxt_tx_descriptor_status_op,
 };
 
 static bool bnxt_vf_pciid(uint16_t id)
-- 
2.10.1 (Apple Git-78)

Re: [PATCH 5/8] net/bnxt: add support for tx_descriptor_status

[Ferruh Yigit]

On 8/24/2017 5:29 PM, Ajit Khaparde wrote:
> add support for tx_descriptor_status dev_op
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

<...>

> +static int
> +bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset)
> +{
> +	struct bnxt_tx_queue *txq = (struct bnxt_tx_queue *)tx_queue;
> +	struct bnxt_cp_ring_info *cpr;
> +	struct tx_pkt_cmpl *txcmp;
> +	uint32_t cons;
> +
> +	if (!txq)
> +		return -EINVAL;
> +
> +	cpr = txq->cp_ring;
> +
> +	if (offset >= txq->nb_tx_desc)
> +		return -EINVAL;
> +
> +	cons = RING_CMP(cpr->cp_ring_struct, offset);
> +	txcmp = (struct tx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
> +
> +	if (CMP_VALID(txcmp, offset, cpr->cp_ring_struct))
> +		return RTE_ETH_TX_DESC_DONE;
> +
> +	return RTE_ETH_TX_DESC_FULL;

Same comment with Rx one, a check may be required to distinguish between
status FULL and UNAVAIL.

> +}
<...>

[PATCH 6/8] net/bnxt: add new HWRM structures

[Ajit Khaparde]

Add new HWRM structures for exact match and ntuple filtering.
These will be used in the subsequent patches to add support for
filter_ctrl dev_op

Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/hsi_struct_def_dpdk.h | 984 +++++++++++++++++++++++++++++++++
 1 file changed, 984 insertions(+)

diff --git a/drivers/net/bnxt/hsi_struct_def_dpdk.h b/drivers/net/bnxt/hsi_struct_def_dpdk.h
index cb8660a..a5f871b 100644
--- a/drivers/net/bnxt/hsi_struct_def_dpdk.h
+++ b/drivers/net/bnxt/hsi_struct_def_dpdk.h
@@ -129,6 +129,9 @@
 #define HWRM_CFA_NTUPLE_FILTER_ALLOC	(UINT32_C(0x99))
 #define HWRM_CFA_NTUPLE_FILTER_FREE	(UINT32_C(0x9a))
 #define HWRM_CFA_NTUPLE_FILTER_CFG	(UINT32_C(0x9b))
+#define HWRM_CFA_EM_FLOW_ALLOC		(UINT32_C(0x9c))
+#define HWRM_CFA_EM_FLOW_FREE		(UINT32_C(0x9d))
+#define HWRM_CFA_EM_FLOW_CFG		(UINT32_C(0x9e))
 #define HWRM_TUNNEL_DST_PORT_QUERY	(UINT32_C(0xa0))
 #define HWRM_TUNNEL_DST_PORT_ALLOC	(UINT32_C(0xa1))
 #define HWRM_TUNNEL_DST_PORT_FREE	(UINT32_C(0xa2))
@@ -9471,6 +9474,987 @@ struct hwrm_cfa_l2_set_rx_mask_output {
 	 */
 } __attribute__((packed));
 
+/* hwrm_cfa_ntuple_filter_alloc */
+/*
+ * Description: This is a ntuple filter that uses fields from L4/L3 header and
+ * optionally fields from L2. The ntuple filters apply to receive traffic only.
+ * All L2/L3/L4 header fields are specified in network byte order. These filters
+ * can be used for Receive Flow Steering (RFS). # For ethertype value, only
+ * 0x0800 (IPv4) and 0x86dd (IPv6) shall be supported for ntuple filters. # If a
+ * field specified in this command is not enabled as a valid field, then that
+ * field shall not be used in matching packet header fields against this filter.
+ */
+/* Input	(128 bytes) */
+struct hwrm_cfa_ntuple_filter_alloc_input {
+	uint16_t req_type;
+	/*
+	 * This value indicates what type of request this is. The format
+	 * for the rest of the command is determined by this field.
+	 */
+	uint16_t cmpl_ring;
+	/*
+	 * This value indicates the what completion ring the request
+	 * will be optionally completed on. If the value is -1, then no
+	 * CR completion will be generated. Any other value must be a
+	 * valid CR ring_id value for this function.
+	 */
+	uint16_t seq_id;
+	/* This value indicates the command sequence number. */
+	uint16_t target_id;
+	/*
+	 * Target ID of this command. 0x0 - 0xFFF8 - Used for function
+	 * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF
+	 * - HWRM
+	 */
+	uint64_t resp_addr;
+	/*
+	 * This is the host address where the response will be written
+	 * when the request is complete. This area must be 16B aligned
+	 * and must be cleared to zero before the request is made.
+	 */
+	uint32_t flags;
+	/*
+	 * Setting of this flag indicates the applicability to the
+	 * loopback path.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_LOOPBACK	\
+		UINT32_C(0x1)
+	/*
+	 * Setting of this flag indicates drop action. If this flag is
+	 * not set, then it should be considered accept action.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP	UINT32_C(0x2)
+	/*
+	 * Setting of this flag indicates that a meter is expected to be
+	 * attached to this flow. This hint can be used when choosing
+	 * the action record format required for the flow.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_METER UINT32_C(0x4)
+	uint32_t enables;
+	/* This bit must be '1' for the l2_filter_id field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID   \
+		UINT32_C(0x1)
+	/* This bit must be '1' for the ethertype field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_ETHERTYPE	 \
+		UINT32_C(0x2)
+	/* This bit must be '1' for the tunnel_type field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE	\
+		UINT32_C(0x4)
+	/* This bit must be '1' for the src_macaddr field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR	\
+		UINT32_C(0x8)
+	/* This bit must be '1' for the ipaddr_type field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_IPADDR_TYPE	\
+		UINT32_C(0x10)
+	/* This bit must be '1' for the src_ipaddr field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR	\
+		UINT32_C(0x20)
+	/*
+	 * This bit must be '1' for the src_ipaddr_mask field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR_MASK \
+		UINT32_C(0x40)
+	/* This bit must be '1' for the dst_ipaddr field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR	\
+		UINT32_C(0x80)
+	/*
+	 * This bit must be '1' for the dst_ipaddr_mask field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR_MASK \
+		UINT32_C(0x100)
+	/* This bit must be '1' for the ip_protocol field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_IP_PROTOCOL	\
+		UINT32_C(0x200)
+	/* This bit must be '1' for the src_port field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT	\
+		UINT32_C(0x400)
+	/*
+	 * This bit must be '1' for the src_port_mask field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT_MASK  \
+		UINT32_C(0x800)
+	/* This bit must be '1' for the dst_port field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT	\
+		UINT32_C(0x1000)
+	/*
+	 * This bit must be '1' for the dst_port_mask field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT_MASK  \
+		UINT32_C(0x2000)
+	/* This bit must be '1' for the pri_hint field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_PRI_HINT	\
+		UINT32_C(0x4000)
+	/*
+	 * This bit must be '1' for the ntuple_filter_id field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_NTUPLE_FILTER_ID \
+		UINT32_C(0x8000)
+	/* This bit must be '1' for the dst_id field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_ID	\
+		UINT32_C(0x10000)
+	/*
+	 * This bit must be '1' for the mirror_vnic_id field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID \
+		UINT32_C(0x20000)
+	/* This bit must be '1' for the dst_macaddr field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_MACADDR	\
+		UINT32_C(0x40000)
+	uint64_t l2_filter_id;
+	/*
+	 * This value identifies a set of CFA data structures used for
+	 * an L2 context.
+	 */
+	uint8_t src_macaddr[6];
+	/*
+	 * This value indicates the source MAC address in the Ethernet
+	 * header.
+	 */
+	uint16_t ethertype;
+	/* This value indicates the ethertype in the Ethernet header. */
+	uint8_t ip_addr_type;
+	/*
+	 * This value indicates the type of IP address. 4 - IPv4 6 -
+	 * IPv6 All others are invalid.
+	 */
+	/* invalid */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_UNKNOWN \
+		UINT32_C(0x0)
+	/* IPv4 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 \
+		UINT32_C(0x4)
+	/* IPv6 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 \
+		UINT32_C(0x6)
+	uint8_t ip_protocol;
+	/*
+	 * The value of protocol filed in IP header. Applies to UDP and
+	 * TCP traffic. 6 - UDP 17 - TCP
+	 */
+	/* invalid */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN \
+		UINT32_C(0x0)
+	/* UDP */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UDP \
+		UINT32_C(0x6)
+	/* TCP */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_TCP \
+		UINT32_C(0x11)
+	uint16_t dst_id;
+	/*
+	 * If set, this value shall represent the Logical VNIC ID of the
+	 * destination VNIC for the RX path and network port id of the
+	 * destination port for the TX path.
+	 */
+	uint16_t mirror_vnic_id;
+	/* Logical VNIC ID of the VNIC where traffic is mirrored. */
+	uint8_t tunnel_type;
+	/*
+	 * This value indicates the tunnel type for this filter. If this
+	 * field is not specified, then the filter shall apply to both
+	 * non-tunneled and tunneled packets. If this field conflicts
+	 * with the tunnel_type specified in the l2_filter_id, then the
+	 * HWRM shall return an error for this command.
+	 */
+	/* Non-tunnel */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL \
+		UINT32_C(0x0)
+	/* Virtual eXtensible Local Area Network	(VXLAN) */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN \
+		UINT32_C(0x1)
+	/*
+	 * Network Virtualization Generic Routing
+	 * Encapsulation	(NVGRE)
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE \
+		UINT32_C(0x2)
+	/*
+	 * Generic Routing Encapsulation	(GRE) inside
+	 * Ethernet payload
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_L2GRE \
+		UINT32_C(0x3)
+	/* IP in IP */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPIP \
+		UINT32_C(0x4)
+	/* Generic Network Virtualization Encapsulation	(Geneve) */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_GENEVE \
+		UINT32_C(0x5)
+	/* Multi-Protocol Lable Switching	(MPLS) */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_MPLS \
+		UINT32_C(0x6)
+	/* Stateless Transport Tunnel	(STT) */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_STT UINT32_C(0x7)
+	/*
+	 * Generic Routing Encapsulation	(GRE) inside IP
+	 * datagram payload
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPGRE \
+		UINT32_C(0x8)
+	/* Any tunneled traffic */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL \
+		UINT32_C(0xff)
+	uint8_t pri_hint;
+	/*
+	 * This hint is provided to help in placing the filter in the
+	 * filter table.
+	 */
+	/* No preference */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_NO_PREFER \
+		UINT32_C(0x0)
+	/* Above the given filter */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_ABOVE UINT32_C(0x1)
+	/* Below the given filter */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_BELOW UINT32_C(0x2)
+	/* As high as possible */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_HIGHEST \
+		UINT32_C(0x3)
+	/* As low as possible */
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_LOWEST UINT32_C(0x4)
+	uint32_t src_ipaddr[4];
+	/*
+	 * The value of source IP address to be used in filtering. For
+	 * IPv4, first four bytes represent the IP address.
+	 */
+	uint32_t src_ipaddr_mask[4];
+	/*
+	 * The value of source IP address mask to be used in filtering.
+	 * For IPv4, first four bytes represent the IP address mask.
+	 */
+	uint32_t dst_ipaddr[4];
+	/*
+	 * The value of destination IP address to be used in filtering.
+	 * For IPv4, first four bytes represent the IP address.
+	 */
+	uint32_t dst_ipaddr_mask[4];
+	/*
+	 * The value of destination IP address mask to be used in
+	 * filtering. For IPv4, first four bytes represent the IP
+	 * address mask.
+	 */
+	uint16_t src_port;
+	/*
+	 * The value of source port to be used in filtering. Applies to
+	 * UDP and TCP traffic.
+	 */
+	uint16_t src_port_mask;
+	/*
+	 * The value of source port mask to be used in filtering.
+	 * Applies to UDP and TCP traffic.
+	 */
+	uint16_t dst_port;
+	/*
+	 * The value of destination port to be used in filtering.
+	 * Applies to UDP and TCP traffic.
+	 */
+	uint16_t dst_port_mask;
+	/*
+	 * The value of destination port mask to be used in filtering.
+	 * Applies to UDP and TCP traffic.
+	 */
+	uint64_t ntuple_filter_id_hint;
+	/* This is the ID of the filter that goes along with the pri_hint. */
+} __attribute__((packed));
+
+/* Output	(24 bytes) */
+struct hwrm_cfa_ntuple_filter_alloc_output {
+	uint16_t error_code;
+	/*
+	 * Pass/Fail or error type Note: receiver to verify the in
+	 * parameters, and fail the call with an error when appropriate
+	 */
+	uint16_t req_type;
+	/* This field returns the type of original request. */
+	uint16_t seq_id;
+	/* This field provides original sequence number of the command. */
+	uint16_t resp_len;
+	/*
+	 * This field is the length of the response in bytes. The last
+	 * byte of the response is a valid flag that will read as '1'
+	 * when the command has been completely written to memory.
+	 */
+	uint64_t ntuple_filter_id;
+	/* This value is an opaque id into CFA data structures. */
+	uint32_t flow_id;
+	/*
+	 * This is the ID of the flow associated with this filter. This
+	 * value shall be used to match and associate the flow
+	 * identifier returned in completion records. A value of
+	 * 0xFFFFFFFF shall indicate no flow id.
+	 */
+	uint8_t unused_0;
+	uint8_t unused_1;
+	uint8_t unused_2;
+	uint8_t valid;
+	/*
+	 * This field is used in Output records to indicate that the
+	 * output is completely written to RAM. This field should be
+	 * read as '1' to indicate that the output has been completely
+	 * written. When writing a command completion or response to an
+	 * internal processor, the order of writes has to be such that
+	 * this field is written last.
+	 */
+} __attribute__((packed));
+
+/* hwrm_cfa_ntuple_filter_free */
+/* Description: Free an ntuple filter */
+/* Input	(24 bytes) */
+struct hwrm_cfa_ntuple_filter_free_input {
+	uint16_t req_type;
+	/*
+	 * This value indicates what type of request this is. The format
+	 * for the rest of the command is determined by this field.
+	 */
+	uint16_t cmpl_ring;
+	/*
+	 * This value indicates the what completion ring the request
+	 * will be optionally completed on. If the value is -1, then no
+	 * CR completion will be generated. Any other value must be a
+	 * valid CR ring_id value for this function.
+	 */
+	uint16_t seq_id;
+	/* This value indicates the command sequence number. */
+	uint16_t target_id;
+	/*
+	 * Target ID of this command. 0x0 - 0xFFF8 - Used for function
+	 * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF
+	 * - HWRM
+	 */
+	uint64_t resp_addr;
+	/*
+	 * This is the host address where the response will be written
+	 * when the request is complete. This area must be 16B aligned
+	 * and must be cleared to zero before the request is made.
+	 */
+	uint64_t ntuple_filter_id;
+	/* This value is an opaque id into CFA data structures. */
+} __attribute__((packed));
+
+/* Output	(16 bytes) */
+struct hwrm_cfa_ntuple_filter_free_output {
+	uint16_t error_code;
+	/*
+	 * Pass/Fail or error type Note: receiver to verify the in
+	 * parameters, and fail the call with an error when appropriate
+	 */
+	uint16_t req_type;
+	/* This field returns the type of original request. */
+	uint16_t seq_id;
+	/* This field provides original sequence number of the command. */
+	uint16_t resp_len;
+	/*
+	 * This field is the length of the response in bytes. The last
+	 * byte of the response is a valid flag that will read as '1'
+	 * when the command has been completely written to memory.
+	 */
+	uint32_t unused_0;
+	uint8_t unused_1;
+	uint8_t unused_2;
+	uint8_t unused_3;
+	uint8_t valid;
+	/*
+	 * This field is used in Output records to indicate that the
+	 * output is completely written to RAM. This field should be
+	 * read as '1' to indicate that the output has been completely
+	 * written. When writing a command completion or response to an
+	 * internal processor, the order of writes has to be such that
+	 * this field is written last.
+	 */
+} __attribute__((packed));
+
+/* hwrm_cfa_ntuple_filter_cfg */
+/*
+ * Description: Configure an ntuple filter with a new destination VNIC and/or
+ * meter.
+ */
+/* Input	(48 bytes) */
+struct hwrm_cfa_ntuple_filter_cfg_input {
+	uint16_t req_type;
+	/*
+	 * This value indicates what type of request this is. The format
+	 * for the rest of the command is determined by this field.
+	 */
+	uint16_t cmpl_ring;
+	/*
+	 * This value indicates the what completion ring the request
+	 * will be optionally completed on. If the value is -1, then no
+	 * CR completion will be generated. Any other value must be a
+	 * valid CR ring_id value for this function.
+	 */
+	uint16_t seq_id;
+	/* This value indicates the command sequence number. */
+	uint16_t target_id;
+	/*
+	 * Target ID of this command. 0x0 - 0xFFF8 - Used for function
+	 * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF
+	 * - HWRM
+	 */
+	uint64_t resp_addr;
+	/*
+	 * This is the host address where the response will be written
+	 * when the request is complete. This area must be 16B aligned
+	 * and must be cleared to zero before the request is made.
+	 */
+	uint32_t enables;
+	/* This bit must be '1' for the new_dst_id field to be configured. */
+	#define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_DST_ID	\
+		UINT32_C(0x1)
+	/*
+	 * This bit must be '1' for the new_mirror_vnic_id field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID \
+		UINT32_C(0x2)
+	/*
+	 * This bit must be '1' for the new_meter_instance_id field to
+	 * be configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_METER_INSTANCE_ID \
+		UINT32_C(0x4)
+	uint32_t unused_0;
+	uint64_t ntuple_filter_id;
+	/* This value is an opaque id into CFA data structures. */
+	uint32_t new_dst_id;
+	/*
+	 * If set, this value shall represent the new Logical VNIC ID of
+	 * the destination VNIC for the RX path and new network port id
+	 * of the destination port for the TX path.
+	 */
+	uint32_t new_mirror_vnic_id;
+	/* New Logical VNIC ID of the VNIC where traffic is mirrored. */
+	uint16_t new_meter_instance_id;
+	/*
+	 * New meter to attach to the flow. Specifying the invalid
+	 * instance ID is used to remove any existing meter from the
+	 * flow.
+	 */
+	/*
+	 * A value of 0xfff is considered invalid and
+	 * implies the instance is not configured.
+	 */
+	#define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID \
+		UINT32_C(0xffff)
+	uint16_t unused_1[3];
+} __attribute__((packed));
+
+/* Output	(16 bytes) */
+struct hwrm_cfa_ntuple_filter_cfg_output {
+	uint16_t error_code;
+	/*
+	 * Pass/Fail or error type Note: receiver to verify the in
+	 * parameters, and fail the call with an error when appropriate
+	 */
+	uint16_t req_type;
+	/* This field returns the type of original request. */
+	uint16_t seq_id;
+	/* This field provides original sequence number of the command. */
+	uint16_t resp_len;
+	/*
+	 * This field is the length of the response in bytes. The last
+	 * byte of the response is a valid flag that will read as '1'
+	 * when the command has been completely written to memory.
+	 */
+	uint32_t unused_0;
+	uint8_t unused_1;
+	uint8_t unused_2;
+	uint8_t unused_3;
+	uint8_t valid;
+	/*
+	 * This field is used in Output records to indicate that the
+	 * output is completely written to RAM. This field should be
+	 * read as '1' to indicate that the output has been completely
+	 * written. When writing a command completion or response to an
+	 * internal processor, the order of writes has to be such that
+	 * this field is written last.
+	 */
+} __attribute__((packed));
+
+/* hwrm_cfa_em_flow_alloc */
+/*
+ * Description: This is a generic Exact Match	(EM) flow that uses fields from
+ * L4/L3/L2 headers. The EM flows apply to transmit and receive traffic. All
+ * L2/L3/L4 header fields are specified in network byte order. For each EM flow,
+ * there is an associated set of actions specified. For tunneled packets, all
+ * L2/L3/L4 fields specified are fields of inner headers unless otherwise
+ * specified. # If a field specified in this command is not enabled as a valid
+ * field, then that field shall not be used in matching packet header fields
+ * against this EM flow entry.
+ */
+/* Input	(112 bytes) */
+struct hwrm_cfa_em_flow_alloc_input {
+	uint16_t req_type;
+	/*
+	 * This value indicates what type of request this is. The format
+	 * for the rest of the command is determined by this field.
+	 */
+	uint16_t cmpl_ring;
+	/*
+	 * This value indicates the what completion ring the request
+	 * will be optionally completed on. If the value is -1, then no
+	 * CR completion will be generated. Any other value must be a
+	 * valid CR ring_id value for this function.
+	 */
+	uint16_t seq_id;
+	/* This value indicates the command sequence number. */
+	uint16_t target_id;
+	/*
+	 * Target ID of this command. 0x0 - 0xFFF8 - Used for function
+	 * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF
+	 * - HWRM
+	 */
+	uint64_t resp_addr;
+	/*
+	 * This is the host address where the response will be written
+	 * when the request is complete. This area must be 16B aligned
+	 * and must be cleared to zero before the request is made.
+	 */
+	uint32_t flags;
+	/*
+	 * Enumeration denoting the RX, TX type of the resource. This
+	 * enumeration is used for resources that are similar for both
+	 * TX and RX paths of the chip.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH	UINT32_C(0x1)
+	/* tx path */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_TX	\
+		(UINT32_C(0x0) << 0)
+	/* rx path */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX	\
+		(UINT32_C(0x1) << 0)
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_LAST \
+		CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX
+	/*
+	 * Setting of this flag indicates enabling of a byte counter for
+	 * a given flow.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_BYTE_CTR	UINT32_C(0x2)
+	/*
+	 * Setting of this flag indicates enabling of a packet counter
+	 * for a given flow.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PKT_CTR	UINT32_C(0x4)
+	/*
+	 * Setting of this flag indicates de-capsulation action for the
+	 * given flow.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DECAP	UINT32_C(0x8)
+	/*
+	 * Setting of this flag indicates encapsulation action for the
+	 * given flow.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_ENCAP	UINT32_C(0x10)
+	/*
+	 * Setting of this flag indicates drop action. If this flag is
+	 * not set, then it should be considered accept action.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP	UINT32_C(0x20)
+	/*
+	 * Setting of this flag indicates that a meter is expected to be
+	 * attached to this flow. This hint can be used when choosing
+	 * the action record format required for the flow.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_METER	UINT32_C(0x40)
+	uint32_t enables;
+	/* This bit must be '1' for the l2_filter_id field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID UINT32_C(0x1)
+	/* This bit must be '1' for the tunnel_type field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_TUNNEL_TYPE UINT32_C(0x2)
+	/* This bit must be '1' for the tunnel_id field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_TUNNEL_ID UINT32_C(0x4)
+	/* This bit must be '1' for the src_macaddr field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_MACADDR UINT32_C(0x8)
+	/* This bit must be '1' for the dst_macaddr field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_MACADDR UINT32_C(0x10)
+	/* This bit must be '1' for the ovlan_vid field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_OVLAN_VID UINT32_C(0x20)
+	/* This bit must be '1' for the ivlan_vid field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IVLAN_VID UINT32_C(0x40)
+	/* This bit must be '1' for the ethertype field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ETHERTYPE UINT32_C(0x80)
+	/* This bit must be '1' for the src_ipaddr field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_IPADDR	UINT32_C(0x100)
+	/* This bit must be '1' for the dst_ipaddr field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_IPADDR	UINT32_C(0x200)
+	/* This bit must be '1' for the ipaddr_type field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IPADDR_TYPE UINT32_C(0x400)
+	/* This bit must be '1' for the ip_protocol field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IP_PROTOCOL UINT32_C(0x800)
+	/* This bit must be '1' for the src_port field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_PORT UINT32_C(0x1000)
+	/* This bit must be '1' for the dst_port field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_PORT UINT32_C(0x2000)
+	/* This bit must be '1' for the dst_id field to be configured. */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_ID	UINT32_C(0x4000)
+	/*
+	 * This bit must be '1' for the mirror_vnic_id field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID	\
+		UINT32_C(0x8000)
+	/*
+	 * This bit must be '1' for the encap_record_id field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ENCAP_RECORD_ID	 \
+		UINT32_C(0x10000)
+	/*
+	 * This bit must be '1' for the meter_instance_id field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_METER_INSTANCE_ID	\
+		UINT32_C(0x20000)
+	uint64_t l2_filter_id;
+	/*
+	 * This value identifies a set of CFA data structures used for
+	 * an L2 context.
+	 */
+	uint8_t tunnel_type;
+	/* Tunnel Type. */
+	/* Non-tunnel */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL \
+		UINT32_C(0x0)
+	/* Virtual eXtensible Local Area Network	(VXLAN) */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_VXLAN	UINT32_C(0x1)
+	/*
+	 * Network Virtualization Generic Routing
+	 * Encapsulation	(NVGRE)
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_NVGRE	UINT32_C(0x2)
+	/*
+	 * Generic Routing Encapsulation	(GRE) inside
+	 * Ethernet payload
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_L2GRE	UINT32_C(0x3)
+	/* IP in IP */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_IPIP	UINT32_C(0x4)
+	/* Generic Network Virtualization Encapsulation	(Geneve) */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_GENEVE	UINT32_C(0x5)
+	/* Multi-Protocol Lable Switching	(MPLS) */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_MPLS	UINT32_C(0x6)
+	/* Stateless Transport Tunnel	(STT) */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_STT	UINT32_C(0x7)
+	/*
+	 * Generic Routing Encapsulation	(GRE) inside IP
+	 * datagram payload
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_IPGRE	UINT32_C(0x8)
+	/* Any tunneled traffic */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL \
+		UINT32_C(0xff)
+	uint8_t unused_0;
+	uint16_t unused_1;
+	uint32_t tunnel_id;
+	/*
+	 * Tunnel identifier. Virtual Network Identifier	(VNI). Only
+	 * valid with tunnel_types VXLAN, NVGRE, and Geneve. Only lower
+	 * 24-bits of VNI field are used in setting up the filter.
+	 */
+	uint8_t src_macaddr[6];
+	/*
+	 * This value indicates the source MAC address in the Ethernet
+	 * header.
+	 */
+	uint16_t meter_instance_id;
+	/* The meter instance to attach to the flow. */
+	/*
+	 * A value of 0xfff is considered invalid and
+	 * implies the instance is not configured.
+	 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_METER_INSTANCE_ID_INVALID   \
+		UINT32_C(0xffff)
+	uint8_t dst_macaddr[6];
+	/*
+	 * This value indicates the destination MAC address in the
+	 * Ethernet header.
+	 */
+	uint16_t ovlan_vid;
+	/*
+	 * This value indicates the VLAN ID of the outer VLAN tag in the
+	 * Ethernet header.
+	 */
+	uint16_t ivlan_vid;
+	/*
+	 * This value indicates the VLAN ID of the inner VLAN tag in the
+	 * Ethernet header.
+	 */
+	uint16_t ethertype;
+	/* This value indicates the ethertype in the Ethernet header. */
+	uint8_t ip_addr_type;
+	/*
+	 * This value indicates the type of IP address. 4 - IPv4 6 -
+	 * IPv6 All others are invalid.
+	 */
+	/* invalid */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_UNKNOWN UINT32_C(0x0)
+	/* IPv4 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV4	UINT32_C(0x4)
+	/* IPv6 */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6	UINT32_C(0x6)
+	uint8_t ip_protocol;
+	/*
+	 * The value of protocol filed in IP header. Applies to UDP and
+	 * TCP traffic. 6 - UDP 17 - TCP
+	 */
+	/* invalid */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN UINT32_C(0x0)
+	/* UDP */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_UDP	UINT32_C(0x6)
+	/* TCP */
+	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_TCP	UINT32_C(0x11)
+	uint8_t unused_2;
+	uint8_t unused_3;
+	uint32_t src_ipaddr[4];
+	/*
+	 * The value of source IP address to be used in filtering. For
+	 * IPv4, first four bytes represent the IP address.
+	 */
+	uint32_t dst_ipaddr[4];
+	/*
+	 * big_endian = True The value of destination IP address to be
+	 * used in filtering. For IPv4, first four bytes represent the
+	 * IP address.
+	 */
+	uint16_t src_port;
+	/*
+	 * The value of source port to be used in filtering. Applies to
+	 * UDP and TCP traffic.
+	 */
+	uint16_t dst_port;
+	/*
+	 * The value of destination port to be used in filtering.
+	 * Applies to UDP and TCP traffic.
+	 */
+	uint16_t dst_id;
+	/*
+	 * If set, this value shall represent the Logical VNIC ID of the
+	 * destination VNIC for the RX path and network port id of the
+	 * destination port for the TX path.
+	 */
+	uint16_t mirror_vnic_id;
+	/* Logical VNIC ID of the VNIC where traffic is mirrored. */
+	uint32_t encap_record_id;
+	/* Logical ID of the encapsulation record. */
+	uint32_t unused_4;
+} __attribute__((packed));
+
+/* Output	(24 bytes) */
+struct hwrm_cfa_em_flow_alloc_output {
+	uint16_t error_code;
+	/*
+	 * Pass/Fail or error type Note: receiver to verify the in
+	 * parameters, and fail the call with an error when appropriate
+	 */
+	uint16_t req_type;
+	/* This field returns the type of original request. */
+	uint16_t seq_id;
+	/* This field provides original sequence number of the command. */
+	uint16_t resp_len;
+	/*
+	 * This field is the length of the response in bytes. The last
+	 * byte of the response is a valid flag that will read as '1'
+	 * when the command has been completely written to memory.
+	 */
+	uint64_t em_filter_id;
+	/* This value is an opaque id into CFA data structures. */
+	uint32_t flow_id;
+	/*
+	 * This is the ID of the flow associated with this filter. This
+	 * value shall be used to match and associate the flow
+	 * identifier returned in completion records. A value of
+	 * 0xFFFFFFFF shall indicate no flow id.
+	 */
+	uint8_t unused_0;
+	uint8_t unused_1;
+	uint8_t unused_2;
+	uint8_t valid;
+	/*
+	 * This field is used in Output records to indicate that the
+	 * output is completely written to RAM. This field should be
+	 * read as '1' to indicate that the output has been completely
+	 * written. When writing a command completion or response to an
+	 * internal processor, the order of writes has to be such that
+	 * this field is written last.
+	 */
+} __attribute__((packed));
+
+/* hwrm_cfa_em_flow_free */
+/* Description: Free an EM flow table entry */
+/* Input	(24 bytes) */
+struct hwrm_cfa_em_flow_free_input {
+	uint16_t req_type;
+	/*
+	 * This value indicates what type of request this is. The format
+	 * for the rest of the command is determined by this field.
+	 */
+	uint16_t cmpl_ring;
+	/*
+	 * This value indicates the what completion ring the request
+	 * will be optionally completed on. If the value is -1, then no
+	 * CR completion will be generated. Any other value must be a
+	 * valid CR ring_id value for this function.
+	 */
+	uint16_t seq_id;
+	/* This value indicates the command sequence number. */
+	uint16_t target_id;
+	/*
+	 * Target ID of this command. 0x0 - 0xFFF8 - Used for function
+	 * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF
+	 * - HWRM
+	 */
+	uint64_t resp_addr;
+	/*
+	 * This is the host address where the response will be written
+	 * when the request is complete. This area must be 16B aligned
+	 * and must be cleared to zero before the request is made.
+	 */
+	uint64_t em_filter_id;
+	/* This value is an opaque id into CFA data structures. */
+} __attribute__((packed));
+
+/* Output	(16 bytes) */
+struct hwrm_cfa_em_flow_free_output {
+	uint16_t error_code;
+	/*
+	 * Pass/Fail or error type Note: receiver to verify the in
+	 * parameters, and fail the call with an error when appropriate
+	 */
+	uint16_t req_type;
+	/* This field returns the type of original request. */
+	uint16_t seq_id;
+	/* This field provides original sequence number of the command. */
+	uint16_t resp_len;
+	/*
+	 * This field is the length of the response in bytes. The last
+	 * byte of the response is a valid flag that will read as '1'
+	 * when the command has been completely written to memory.
+	 */
+	uint32_t unused_0;
+	uint8_t unused_1;
+	uint8_t unused_2;
+	uint8_t unused_3;
+	uint8_t valid;
+	/*
+	 * This field is used in Output records to indicate that the
+	 * output is completely written to RAM. This field should be
+	 * read as '1' to indicate that the output has been completely
+	 * written. When writing a command completion or response to an
+	 * internal processor, the order of writes has to be such that
+	 * this field is written last.
+	 */
+} __attribute__((packed));
+
+/* hwrm_cfa_em_flow_cfg */
+/*
+ * Description: Configure an EM flow with a new destination VNIC and/or meter.
+ */
+/* Input	(48 bytes) */
+struct hwrm_cfa_em_flow_cfg_input {
+	uint16_t req_type;
+	/*
+	 * This value indicates what type of request this is. The format
+	 * for the rest of the command is determined by this field.
+	 */
+	uint16_t cmpl_ring;
+	/*
+	 * This value indicates the what completion ring the request
+	 * will be optionally completed on. If the value is -1, then no
+	 * CR completion will be generated. Any other value must be a
+	 * valid CR ring_id value for this function.
+	 */
+	uint16_t seq_id;
+	/* This value indicates the command sequence number. */
+	uint16_t target_id;
+	/*
+	 * Target ID of this command. 0x0 - 0xFFF8 - Used for function
+	 * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF
+	 * - HWRM
+	 */
+	uint64_t resp_addr;
+	/*
+	 * This is the host address where the response will be written
+	 * when the request is complete. This area must be 16B aligned
+	 * and must be cleared to zero before the request is made.
+	 */
+	uint32_t enables;
+	/* This bit must be '1' for the new_dst_id field to be configured. */
+	#define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_DST_ID	UINT32_C(0x1)
+	/*
+	 * This bit must be '1' for the new_mirror_vnic_id field to be
+	 * configured.
+	 */
+	#define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID	\
+		UINT32_C(0x2)
+	/*
+	 * This bit must be '1' for the new_meter_instance_id field to
+	 * be configured.
+	 */
+	#define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_METER_INSTANCE_ID  \
+		UINT32_C(0x4)
+	uint32_t unused_0;
+	uint64_t em_filter_id;
+	/* This value is an opaque id into CFA data structures. */
+	uint32_t new_dst_id;
+	/*
+	 * If set, this value shall represent the new Logical VNIC ID of
+	 * the destination VNIC for the RX path and network port id of
+	 * the destination port for the TX path.
+	 */
+	uint32_t new_mirror_vnic_id;
+	/* New Logical VNIC ID of the VNIC where traffic is mirrored. */
+	uint16_t new_meter_instance_id;
+	/*
+	 * New meter to attach to the flow. Specifying the invalid
+	 * instance ID is used to remove any existing meter from the
+	 * flow.
+	 */
+	/*
+	 * A value of 0xfff is considered invalid and
+	 * implies the instance is not configured.
+	 */
+	#define HWRM_CFA_EM_FLOW_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID \
+		UINT32_C(0xffff)
+	uint16_t unused_1[3];
+} __attribute__((packed));
+
+/* Output	(16 bytes) */
+struct hwrm_cfa_em_flow_cfg_output {
+	uint16_t error_code;
+	/*
+	 * Pass/Fail or error type Note: receiver to verify the in
+	 * parameters, and fail the call with an error when appropriate
+	 */
+	uint16_t req_type;
+	/* This field returns the type of original request. */
+	uint16_t seq_id;
+	/* This field provides original sequence number of the command. */
+	uint16_t resp_len;
+	/*
+	 * This field is the length of the response in bytes. The last
+	 * byte of the response is a valid flag that will read as '1'
+	 * when the command has been completely written to memory.
+	 */
+	uint32_t unused_0;
+	uint8_t unused_1;
+	uint8_t unused_2;
+	uint8_t unused_3;
+	uint8_t valid;
+	/*
+	 * This field is used in Output records to indicate that the
+	 * output is completely written to RAM. This field should be
+	 * read as '1' to indicate that the output has been completely
+	 * written. When writing a command completion or response to an
+	 * internal processor, the order of writes has to be such that
+	 * this field is written last.
+	 */
+} __attribute__((packed));
+
+
 /* hwrm_cfa_vlan_antispoof_cfg */
 /* Description: Configures vlan anti-spoof filters for VF. */
 /* Input (32 bytes) */
-- 
2.10.1 (Apple Git-78)

[PATCH 7/8] net/bnxt: fix HWRM_*() macros and locking

[Ajit Khaparde]

Obtain the spinlock in HWRM_PREP()
Eliminate two unnecessary arguments in HWRM_PREP().
Unlock the spinlock before returning in HWRM_ERROR_CHECK()
Add new HWRM_UNLOCK() macro
Update usage of the thre macros.

Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/bnxt_hwrm.c | 413 ++++++++++++++++++++++++++++---------------
 1 file changed, 269 insertions(+), 144 deletions(-)

diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c
index e710e63..0fc5244 100644
--- a/drivers/net/bnxt/bnxt_hwrm.c
+++ b/drivers/net/bnxt/bnxt_hwrm.c
@@ -54,7 +54,7 @@
 
 #include <rte_io.h>
 
-#define HWRM_CMD_TIMEOUT		2000
+#define HWRM_CMD_TIMEOUT		10000
 
 struct bnxt_plcmodes_cfg {
 	uint32_t	flags;
@@ -95,7 +95,7 @@ static int page_roundup(size_t size)
  * command was failed by the ChiMP.
  */
 
-static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg,
+static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg,
 					uint32_t msg_len)
 {
 	unsigned int i;
@@ -171,29 +171,34 @@ static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg,
 	return -1;
 }
 
-static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, uint32_t msg_len)
-{
-	int rc;
-
-	rte_spinlock_lock(&bp->hwrm_lock);
-	rc = bnxt_hwrm_send_message_locked(bp, msg, msg_len);
-	rte_spinlock_unlock(&bp->hwrm_lock);
-	return rc;
-}
-
-#define HWRM_PREP(req, type, cr, resp) \
+/*
+ * HWRM_PREP() should be used to prepare *ALL* HWRM commands.  It grabs the
+ * spinlock, and does initial processing.
+ *
+ * HWRM_CHECK_RESULT() returns errors on failure and may not be used.  It
+ * releases the spinlock only if it returns.  If the regular int return codes
+ * are not used by the function, HWRM_CHECK_RESULT() should not be used
+ * directly, rather it should be copied and modified to suit the function.
+ *
+ * HWRM_UNLOCK() must be called after all response processing is completed.
+ */
+#define HWRM_PREP(x, type) do { \
+	typeof(x) req = (x); \
+	rte_spinlock_lock(&bp->hwrm_lock); \
 	memset(bp->hwrm_cmd_resp_addr, 0, bp->max_resp_len); \
 	req.req_type = rte_cpu_to_le_16(HWRM_##type); \
-	req.cmpl_ring = rte_cpu_to_le_16(cr); \
+	req.cmpl_ring = rte_cpu_to_le_16(-1); \
 	req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); \
 	req.target_id = rte_cpu_to_le_16(0xffff); \
-	req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr)
+	req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr); \
+} while (0)
 
-#define HWRM_CHECK_RESULT \
-	{ \
+#define HWRM_CHECK_RESULT() \
+	do {\
 		if (rc) { \
 			RTE_LOG(ERR, PMD, "%s failed rc:%d\n", \
 				__func__, rc); \
+			rte_spinlock_unlock(&bp->hwrm_lock); \
 			return rc; \
 		} \
 		if (resp->error_code) { \
@@ -214,9 +219,12 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, uint32_t msg_len)
 				RTE_LOG(ERR, PMD, \
 					"%s error %d\n", __func__, rc); \
 			} \
+			rte_spinlock_unlock(&bp->hwrm_lock); \
 			return rc; \
 		} \
-	}
+	} while (0)
+
+#define HWRM_UNLOCK()		rte_spinlock_unlock(&bp->hwrm_lock)
 
 int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 {
@@ -224,13 +232,14 @@ int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	struct hwrm_cfa_l2_set_rx_mask_input req = {.req_type = 0 };
 	struct hwrm_cfa_l2_set_rx_mask_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, CFA_L2_SET_RX_MASK, -1, resp);
+	HWRM_PREP(req, CFA_L2_SET_RX_MASK);
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 	req.mask = 0;
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -245,7 +254,7 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp,
 	struct hwrm_cfa_l2_set_rx_mask_output *resp = bp->hwrm_cmd_resp_addr;
 	uint32_t mask = 0;
 
-	HWRM_PREP(req, CFA_L2_SET_RX_MASK, -1, resp);
+	HWRM_PREP(req, CFA_L2_SET_RX_MASK);
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 
 	/* FIXME add multicast flag, when multicast adding options is supported
@@ -278,7 +287,8 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -307,7 +317,7 @@ int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid,
 				return 0;
 		}
 	}
-	HWRM_PREP(req, CFA_VLAN_ANTISPOOF_CFG, -1, resp);
+	HWRM_PREP(req, CFA_VLAN_ANTISPOOF_CFG);
 	req.fid = rte_cpu_to_le_16(fid);
 
 	req.vlan_tag_mask_tbl_addr =
@@ -316,7 +326,8 @@ int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -331,13 +342,14 @@ int bnxt_hwrm_clear_filter(struct bnxt *bp,
 	if (filter->fw_l2_filter_id == UINT64_MAX)
 		return 0;
 
-	HWRM_PREP(req, CFA_L2_FILTER_FREE, -1, resp);
+	HWRM_PREP(req, CFA_L2_FILTER_FREE);
 
 	req.l2_filter_id = rte_cpu_to_le_64(filter->fw_l2_filter_id);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	filter->fw_l2_filter_id = -1;
 
@@ -356,7 +368,7 @@ int bnxt_hwrm_set_filter(struct bnxt *bp,
 	if (filter->fw_l2_filter_id != UINT64_MAX)
 		bnxt_hwrm_clear_filter(bp, filter);
 
-	HWRM_PREP(req, CFA_L2_FILTER_ALLOC, -1, resp);
+	HWRM_PREP(req, CFA_L2_FILTER_ALLOC);
 
 	req.flags = rte_cpu_to_le_32(filter->flags);
 
@@ -387,9 +399,10 @@ int bnxt_hwrm_set_filter(struct bnxt *bp,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	filter->fw_l2_filter_id = rte_le_to_cpu_64(resp->l2_filter_id);
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -402,13 +415,13 @@ int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 	uint16_t new_max_vfs;
 	int i;
 
-	HWRM_PREP(req, FUNC_QCAPS, -1, resp);
+	HWRM_PREP(req, FUNC_QCAPS);
 
 	req.fid = rte_cpu_to_le_16(0xffff);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	bp->max_ring_grps = rte_le_to_cpu_32(resp->max_hw_ring_grps);
 	if (BNXT_PF(bp)) {
@@ -469,6 +482,7 @@ int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 	bp->max_stat_ctx = rte_le_to_cpu_16(resp->max_stat_ctx);
 	if (BNXT_PF(bp))
 		bp->pf.total_vnics = rte_le_to_cpu_16(resp->max_vnics);
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -479,13 +493,14 @@ int bnxt_hwrm_func_reset(struct bnxt *bp)
 	struct hwrm_func_reset_input req = {.req_type = 0 };
 	struct hwrm_func_reset_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_RESET, -1, resp);
+	HWRM_PREP(req, FUNC_RESET);
 
 	req.enables = rte_cpu_to_le_32(0);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -499,7 +514,7 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp)
 	if (bp->flags & BNXT_FLAG_REGISTERED)
 		return 0;
 
-	HWRM_PREP(req, FUNC_DRV_RGTR, -1, resp);
+	HWRM_PREP(req, FUNC_DRV_RGTR);
 	req.enables = rte_cpu_to_le_32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER |
 			HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD);
 	req.ver_maj = RTE_VER_YEAR;
@@ -519,7 +534,8 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp)
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	bp->flags |= BNXT_FLAG_REGISTERED;
 
@@ -538,19 +554,15 @@ int bnxt_hwrm_ver_get(struct bnxt *bp)
 	uint32_t dev_caps_cfg;
 
 	bp->max_req_len = HWRM_MAX_REQ_LEN;
-	HWRM_PREP(req, VER_GET, -1, resp);
+	HWRM_PREP(req, VER_GET);
 
 	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
 	req.hwrm_intf_min = HWRM_VERSION_MINOR;
 	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
 
-	/*
-	 * Hold the lock since we may be adjusting the response pointers.
-	 */
-	rte_spinlock_lock(&bp->hwrm_lock);
-	rc = bnxt_hwrm_send_message_locked(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	RTE_LOG(INFO, PMD, "%d.%d.%d:%d.%d.%d\n",
 		resp->hwrm_intf_maj, resp->hwrm_intf_min,
@@ -651,7 +663,7 @@ int bnxt_hwrm_ver_get(struct bnxt *bp)
 	}
 
 error:
-	rte_spinlock_unlock(&bp->hwrm_lock);
+	HWRM_UNLOCK();
 	return rc;
 }
 
@@ -664,12 +676,13 @@ int bnxt_hwrm_func_driver_unregister(struct bnxt *bp, uint32_t flags)
 	if (!(bp->flags & BNXT_FLAG_REGISTERED))
 		return 0;
 
-	HWRM_PREP(req, FUNC_DRV_UNRGTR, -1, resp);
+	HWRM_PREP(req, FUNC_DRV_UNRGTR);
 	req.flags = flags;
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	bp->flags &= ~BNXT_FLAG_REGISTERED;
 
@@ -685,7 +698,7 @@ static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf)
 	uint32_t link_speed_mask =
 		HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK;
 
-	HWRM_PREP(req, PORT_PHY_CFG, -1, resp);
+	HWRM_PREP(req, PORT_PHY_CFG);
 
 	if (conf->link_up) {
 		req.flags = rte_cpu_to_le_32(conf->phy_flags);
@@ -729,7 +742,8 @@ static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf)
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -741,11 +755,11 @@ static int bnxt_hwrm_port_phy_qcfg(struct bnxt *bp,
 	struct hwrm_port_phy_qcfg_input req = {0};
 	struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, PORT_PHY_QCFG, -1, resp);
+	HWRM_PREP(req, PORT_PHY_QCFG);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	link_info->phy_link_status = resp->link;
 	link_info->link_up =
@@ -765,6 +779,8 @@ static int bnxt_hwrm_port_phy_qcfg(struct bnxt *bp,
 	link_info->phy_ver[1] = resp->phy_min;
 	link_info->phy_ver[2] = resp->phy_bld;
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -774,11 +790,11 @@ int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
 	struct hwrm_queue_qportcfg_input req = {.req_type = 0 };
 	struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, QUEUE_QPORTCFG, -1, resp);
+	HWRM_PREP(req, QUEUE_QPORTCFG);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 #define GET_QUEUE_INFO(x) \
 	bp->cos_queue[x].id = resp->queue_id##x; \
@@ -793,6 +809,8 @@ int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
 	GET_QUEUE_INFO(6);
 	GET_QUEUE_INFO(7);
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -806,7 +824,7 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp,
 	struct hwrm_ring_alloc_input req = {.req_type = 0 };
 	struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_ALLOC, -1, resp);
+	HWRM_PREP(req, RING_ALLOC);
 
 	req.page_tbl_addr = rte_cpu_to_le_64(ring->bd_dma);
 	req.fbo = rte_cpu_to_le_32(0);
@@ -837,6 +855,7 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp,
 	default:
 		RTE_LOG(ERR, PMD, "hwrm alloc invalid ring type %d\n",
 			ring_type);
+		HWRM_UNLOCK();
 		return -1;
 	}
 	req.enables = rte_cpu_to_le_32(enables);
@@ -850,22 +869,27 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp,
 		case HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL:
 			RTE_LOG(ERR, PMD,
 				"hwrm_ring_alloc cp failed. rc:%d\n", rc);
+			HWRM_UNLOCK();
 			return rc;
 		case HWRM_RING_FREE_INPUT_RING_TYPE_RX:
 			RTE_LOG(ERR, PMD,
 				"hwrm_ring_alloc rx failed. rc:%d\n", rc);
+			HWRM_UNLOCK();
 			return rc;
 		case HWRM_RING_FREE_INPUT_RING_TYPE_TX:
 			RTE_LOG(ERR, PMD,
 				"hwrm_ring_alloc tx failed. rc:%d\n", rc);
+			HWRM_UNLOCK();
 			return rc;
 		default:
 			RTE_LOG(ERR, PMD, "Invalid ring. rc:%d\n", rc);
+			HWRM_UNLOCK();
 			return rc;
 		}
 	}
 
 	ring->fw_ring_id = rte_le_to_cpu_16(resp->ring_id);
+	HWRM_UNLOCK();
 	return rc;
 }
 
@@ -876,7 +900,7 @@ int bnxt_hwrm_ring_free(struct bnxt *bp,
 	struct hwrm_ring_free_input req = {.req_type = 0 };
 	struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_FREE, -1, resp);
+	HWRM_PREP(req, RING_FREE);
 
 	req.ring_type = ring_type;
 	req.ring_id = rte_cpu_to_le_16(ring->fw_ring_id);
@@ -886,6 +910,7 @@ int bnxt_hwrm_ring_free(struct bnxt *bp,
 	if (rc || resp->error_code) {
 		if (rc == 0 && resp->error_code)
 			rc = rte_le_to_cpu_16(resp->error_code);
+		HWRM_UNLOCK();
 
 		switch (ring_type) {
 		case HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL:
@@ -905,6 +930,7 @@ int bnxt_hwrm_ring_free(struct bnxt *bp,
 			return rc;
 		}
 	}
+	HWRM_UNLOCK();
 	return 0;
 }
 
@@ -914,7 +940,7 @@ int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp, unsigned int idx)
 	struct hwrm_ring_grp_alloc_input req = {.req_type = 0 };
 	struct hwrm_ring_grp_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_GRP_ALLOC, -1, resp);
+	HWRM_PREP(req, RING_GRP_ALLOC);
 
 	req.cr = rte_cpu_to_le_16(bp->grp_info[idx].cp_fw_ring_id);
 	req.rr = rte_cpu_to_le_16(bp->grp_info[idx].rx_fw_ring_id);
@@ -923,11 +949,13 @@ int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp, unsigned int idx)
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	bp->grp_info[idx].fw_grp_id =
 	    rte_le_to_cpu_16(resp->ring_group_id);
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -937,13 +965,14 @@ int bnxt_hwrm_ring_grp_free(struct bnxt *bp, unsigned int idx)
 	struct hwrm_ring_grp_free_input req = {.req_type = 0 };
 	struct hwrm_ring_grp_free_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_GRP_FREE, -1, resp);
+	HWRM_PREP(req, RING_GRP_FREE);
 
 	req.ring_group_id = rte_cpu_to_le_16(bp->grp_info[idx].fw_grp_id);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	bp->grp_info[idx].fw_grp_id = INVALID_HW_RING_ID;
 	return rc;
@@ -958,13 +987,14 @@ int bnxt_hwrm_stat_clear(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
 	if (cpr->hw_stats_ctx_id == (uint32_t)HWRM_NA_SIGNATURE)
 		return rc;
 
-	HWRM_PREP(req, STAT_CTX_CLR_STATS, -1, resp);
+	HWRM_PREP(req, STAT_CTX_CLR_STATS);
 
 	req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -976,7 +1006,7 @@ int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 	struct hwrm_stat_ctx_alloc_input req = {.req_type = 0 };
 	struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, STAT_CTX_ALLOC, -1, resp);
+	HWRM_PREP(req, STAT_CTX_ALLOC);
 
 	req.update_period_ms = rte_cpu_to_le_32(0);
 
@@ -985,10 +1015,12 @@ int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	cpr->hw_stats_ctx_id = rte_le_to_cpu_16(resp->stat_ctx_id);
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -999,13 +1031,14 @@ int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 	struct hwrm_stat_ctx_free_input req = {.req_type = 0 };
 	struct hwrm_stat_ctx_free_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, STAT_CTX_FREE, -1, resp);
+	HWRM_PREP(req, STAT_CTX_FREE);
 
 	req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -1027,15 +1060,16 @@ int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	vnic->lb_rule = (uint16_t)HWRM_NA_SIGNATURE;
 	vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN +
 				ETHER_CRC_LEN + VLAN_TAG_SIZE;
-	HWRM_PREP(req, VNIC_ALLOC, -1, resp);
+	HWRM_PREP(req, VNIC_ALLOC);
 
 	if (vnic->func_default)
 		req.flags = HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT;
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	vnic->fw_vnic_id = rte_le_to_cpu_16(resp->vnic_id);
+	HWRM_UNLOCK();
 	RTE_LOG(DEBUG, PMD, "VNIC ID %x\n", vnic->fw_vnic_id);
 	return rc;
 }
@@ -1048,13 +1082,13 @@ static int bnxt_hwrm_vnic_plcmodes_qcfg(struct bnxt *bp,
 	struct hwrm_vnic_plcmodes_qcfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_plcmodes_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_PLCMODES_QCFG, -1, resp);
+	HWRM_PREP(req, VNIC_PLCMODES_QCFG);
 
 	req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	pmode->flags = rte_le_to_cpu_32(resp->flags);
 	/* dflt_vnic bit doesn't exist in the _cfg command */
@@ -1063,6 +1097,8 @@ static int bnxt_hwrm_vnic_plcmodes_qcfg(struct bnxt *bp,
 	pmode->hds_offset = rte_le_to_cpu_16(resp->hds_offset);
 	pmode->hds_threshold = rte_le_to_cpu_16(resp->hds_threshold);
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -1074,7 +1110,7 @@ static int bnxt_hwrm_vnic_plcmodes_cfg(struct bnxt *bp,
 	struct hwrm_vnic_plcmodes_cfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_PLCMODES_CFG, -1, resp);
+	HWRM_PREP(req, VNIC_PLCMODES_CFG);
 
 	req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id);
 	req.flags = rte_cpu_to_le_32(pmode->flags);
@@ -1089,7 +1125,8 @@ static int bnxt_hwrm_vnic_plcmodes_cfg(struct bnxt *bp,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -1111,7 +1148,7 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	if (rc)
 		return rc;
 
-	HWRM_PREP(req, VNIC_CFG, -1, resp);
+	HWRM_PREP(req, VNIC_CFG);
 
 	/* Only RSS support for now TBD: COS & LB */
 	req.enables =
@@ -1151,7 +1188,8 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	rc = bnxt_hwrm_vnic_plcmodes_cfg(bp, vnic, &pmodes);
 
@@ -1169,7 +1207,7 @@ int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic,
 		RTE_LOG(DEBUG, PMD, "VNIC QCFG ID %d\n", vnic->fw_vnic_id);
 		return rc;
 	}
-	HWRM_PREP(req, VNIC_QCFG, -1, resp);
+	HWRM_PREP(req, VNIC_QCFG);
 
 	req.enables =
 		rte_cpu_to_le_32(HWRM_VNIC_QCFG_INPUT_ENABLES_VF_ID_VALID);
@@ -1178,7 +1216,7 @@ int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	vnic->dflt_ring_grp = rte_le_to_cpu_16(resp->dflt_ring_grp);
 	vnic->rss_rule = rte_le_to_cpu_16(resp->rss_rule);
@@ -1198,6 +1236,8 @@ int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic,
 	vnic->rss_dflt_cr = rte_le_to_cpu_32(resp->flags) &
 			HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE;
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -1208,13 +1248,14 @@ int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
 						bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_ALLOC, -1, resp);
+	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_ALLOC);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	vnic->rss_rule = rte_le_to_cpu_16(resp->rss_cos_lb_ctx_id);
+	HWRM_UNLOCK();
 	RTE_LOG(DEBUG, PMD, "VNIC RSS Rule %x\n", vnic->rss_rule);
 
 	return rc;
@@ -1231,13 +1272,14 @@ int bnxt_hwrm_vnic_ctx_free(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 		RTE_LOG(DEBUG, PMD, "VNIC RSS Rule %x\n", vnic->rss_rule);
 		return rc;
 	}
-	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_FREE, -1, resp);
+	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_FREE);
 
 	req.rss_cos_lb_ctx_id = rte_cpu_to_le_16(vnic->rss_rule);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	vnic->rss_rule = INVALID_HW_RING_ID;
 
@@ -1255,13 +1297,14 @@ int bnxt_hwrm_vnic_free(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 		return rc;
 	}
 
-	HWRM_PREP(req, VNIC_FREE, -1, resp);
+	HWRM_PREP(req, VNIC_FREE);
 
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	vnic->fw_vnic_id = INVALID_HW_RING_ID;
 	return rc;
@@ -1274,7 +1317,7 @@ int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp,
 	struct hwrm_vnic_rss_cfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_rss_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_RSS_CFG, -1, resp);
+	HWRM_PREP(req, VNIC_RSS_CFG);
 
 	req.hash_type = rte_cpu_to_le_32(vnic->hash_type);
 
@@ -1286,7 +1329,8 @@ int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -1299,7 +1343,7 @@ int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp,
 	struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	uint16_t size;
 
-	HWRM_PREP(req, VNIC_PLCMODES_CFG, -1, resp);
+	HWRM_PREP(req, VNIC_PLCMODES_CFG);
 
 	req.flags = rte_cpu_to_le_32(
 			HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_JUMBO_PLACEMENT);
@@ -1315,7 +1359,8 @@ int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -1327,7 +1372,7 @@ int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp,
 	struct hwrm_vnic_tpa_cfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_tpa_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_TPA_CFG, -1, resp);
+	HWRM_PREP(req, VNIC_TPA_CFG);
 
 	if (enable) {
 		req.enables = rte_cpu_to_le_32(
@@ -1350,7 +1395,8 @@ int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp,
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -1367,10 +1413,11 @@ int bnxt_hwrm_func_vf_mac(struct bnxt *bp, uint16_t vf, const uint8_t *mac_addr)
 	memcpy(req.dflt_mac_addr, mac_addr, sizeof(req.dflt_mac_addr));
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	bp->pf.vf_info[vf].random_mac = false;
 
@@ -1384,17 +1431,19 @@ int bnxt_hwrm_func_qstats_tx_drop(struct bnxt *bp, uint16_t fid,
 	struct hwrm_func_qstats_input req = {.req_type = 0};
 	struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_QSTATS, -1, resp);
+	HWRM_PREP(req, FUNC_QSTATS);
 
 	req.fid = rte_cpu_to_le_16(fid);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	if (dropped)
 		*dropped = rte_le_to_cpu_64(resp->tx_drop_pkts);
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -1405,13 +1454,13 @@ int bnxt_hwrm_func_qstats(struct bnxt *bp, uint16_t fid,
 	struct hwrm_func_qstats_input req = {.req_type = 0};
 	struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_QSTATS, -1, resp);
+	HWRM_PREP(req, FUNC_QSTATS);
 
 	req.fid = rte_cpu_to_le_16(fid);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	stats->ipackets = rte_le_to_cpu_64(resp->rx_ucast_pkts);
 	stats->ipackets += rte_le_to_cpu_64(resp->rx_mcast_pkts);
@@ -1432,6 +1481,8 @@ int bnxt_hwrm_func_qstats(struct bnxt *bp, uint16_t fid,
 
 	stats->imissed = rte_le_to_cpu_64(resp->rx_drop_pkts);
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -1441,13 +1492,14 @@ int bnxt_hwrm_func_clr_stats(struct bnxt *bp, uint16_t fid)
 	struct hwrm_func_clr_stats_input req = {.req_type = 0};
 	struct hwrm_func_clr_stats_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_CLR_STATS, -1, resp);
+	HWRM_PREP(req, FUNC_CLR_STATS);
 
 	req.fid = rte_cpu_to_le_16(fid);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2038,12 +2090,12 @@ int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc = 0;
 
-	HWRM_PREP(req, FUNC_QCFG, -1, resp);
+	HWRM_PREP(req, FUNC_QCFG);
 	req.fid = rte_cpu_to_le_16(0xffff);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	/* Hard Coded.. 0xfff VLAN ID mask */
 	bp->vlan = rte_le_to_cpu_16(resp->vlan) & 0xfff;
@@ -2059,6 +2111,8 @@ int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 		break;
 	}
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2118,10 +2172,12 @@ static int bnxt_hwrm_pf_func_cfg(struct bnxt *bp, int tx_rings)
 	req.num_hw_ring_grps = rte_cpu_to_le_16(bp->max_ring_grps);
 	req.fid = rte_cpu_to_le_16(0xffff);
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2187,7 +2243,7 @@ static void reserve_resources_from_vf(struct bnxt *bp,
 	int rc;
 
 	/* Get the actual allocated values now */
-	HWRM_PREP(req, FUNC_QCAPS, -1, resp);
+	HWRM_PREP(req, FUNC_QCAPS);
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
@@ -2212,6 +2268,8 @@ static void reserve_resources_from_vf(struct bnxt *bp,
 	 */
 	//bp->max_vnics -= rte_le_to_cpu_16(esp->max_vnics);
 	bp->max_ring_grps -= rte_le_to_cpu_16(resp->max_hw_ring_grps);
+
+	HWRM_UNLOCK();
 }
 
 int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf)
@@ -2221,7 +2279,7 @@ int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf)
 	int rc;
 
 	/* Check for zero MAC address */
-	HWRM_PREP(req, FUNC_QCFG, -1, resp);
+	HWRM_PREP(req, FUNC_QCFG);
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 	if (rc) {
@@ -2232,7 +2290,11 @@ int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf)
 		RTE_LOG(ERR, PMD, "hwrm_func_qcfg error %d\n", rc);
 		return -1;
 	}
-	return rte_le_to_cpu_16(resp->vlan);
+	rc = rte_le_to_cpu_16(resp->vlan);
+
+	HWRM_UNLOCK();
+
+	return rc;
 }
 
 static int update_pf_resource_max(struct bnxt *bp)
@@ -2242,15 +2304,17 @@ static int update_pf_resource_max(struct bnxt *bp)
 	int rc;
 
 	/* And copy the allocated numbers into the pf struct */
-	HWRM_PREP(req, FUNC_QCFG, -1, resp);
+	HWRM_PREP(req, FUNC_QCFG);
 	req.fid = rte_cpu_to_le_16(0xffff);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	/* Only TX ring value reflects actual allocation? TODO */
 	bp->max_tx_rings = rte_le_to_cpu_16(resp->alloc_tx_rings);
 	bp->pf.evb_mode = resp->evb_mode;
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2342,7 +2406,7 @@ int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs)
 	for (i = 0; i < num_vfs; i++) {
 		add_random_mac_if_needed(bp, &req, i);
 
-		HWRM_PREP(req, FUNC_CFG, -1, resp);
+		HWRM_PREP(req, FUNC_CFG);
 		req.flags = rte_cpu_to_le_32(bp->pf.vf_info[i].func_cfg_flags);
 		req.fid = rte_cpu_to_le_16(bp->pf.vf_info[i].fid);
 		rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
@@ -2357,9 +2421,12 @@ int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs)
 			RTE_LOG(ERR, PMD,
 				"Not all VFs available. (%d, %d)\n",
 				rc, resp->error_code);
+			HWRM_UNLOCK();
 			break;
 		}
 
+		HWRM_UNLOCK();
+
 		reserve_resources_from_vf(bp, &req, i);
 		bp->pf.active_vfs++;
 		bnxt_hwrm_func_clr_stats(bp, bp->pf.vf_info[i].fid);
@@ -2392,14 +2459,15 @@ int bnxt_hwrm_pf_evb_mode(struct bnxt *bp)
 	struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
 
 	req.fid = rte_cpu_to_le_16(0xffff);
 	req.enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_EVB_MODE);
 	req.evb_mode = bp->pf.evb_mode;
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2411,11 +2479,11 @@ int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, uint16_t port,
 	struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc = 0;
 
-	HWRM_PREP(req, TUNNEL_DST_PORT_ALLOC, -1, resp);
+	HWRM_PREP(req, TUNNEL_DST_PORT_ALLOC);
 	req.tunnel_type = tunnel_type;
 	req.tunnel_dst_port_val = port;
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	switch (tunnel_type) {
 	case HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN:
@@ -2429,6 +2497,9 @@ int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, uint16_t port,
 	default:
 		break;
 	}
+
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2439,11 +2510,14 @@ int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, uint16_t port,
 	struct hwrm_tunnel_dst_port_free_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc = 0;
 
-	HWRM_PREP(req, TUNNEL_DST_PORT_FREE, -1, resp);
+	HWRM_PREP(req, TUNNEL_DST_PORT_FREE);
+
 	req.tunnel_type = tunnel_type;
 	req.tunnel_dst_port_id = rte_cpu_to_be_16(port);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2455,11 +2529,14 @@ int bnxt_hwrm_func_cfg_vf_set_flags(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
+
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.flags = rte_cpu_to_le_32(flags);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2482,7 +2559,7 @@ int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp)
 	struct hwrm_func_buf_rgtr_input req = {.req_type = 0 };
 	struct hwrm_func_buf_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_BUF_RGTR, -1, resp);
+	HWRM_PREP(req, FUNC_BUF_RGTR);
 
 	req.req_buf_num_pages = rte_cpu_to_le_16(1);
 	req.req_buf_page_size = rte_cpu_to_le_16(
@@ -2498,7 +2575,8 @@ int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp)
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2509,11 +2587,12 @@ int bnxt_hwrm_func_buf_unrgtr(struct bnxt *bp)
 	struct hwrm_func_buf_unrgtr_input req = {.req_type = 0 };
 	struct hwrm_func_buf_unrgtr_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_BUF_UNRGTR, -1, resp);
+	HWRM_PREP(req, FUNC_BUF_UNRGTR);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2524,7 +2603,8 @@ int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp)
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
+
 	req.fid = rte_cpu_to_le_16(0xffff);
 	req.flags = rte_cpu_to_le_32(bp->pf.func_cfg_flags);
 	req.enables = rte_cpu_to_le_32(
@@ -2532,7 +2612,9 @@ int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp)
 	req.async_event_cr = rte_cpu_to_le_16(
 			bp->def_cp_ring->cp_ring_struct->fw_ring_id);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2543,13 +2625,16 @@ int bnxt_hwrm_vf_func_cfg_def_cp(struct bnxt *bp)
 	struct hwrm_func_vf_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_VF_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_VF_CFG);
+
 	req.enables = rte_cpu_to_le_32(
 			HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
 	req.async_event_cr = rte_cpu_to_le_16(
 			bp->def_cp_ring->cp_ring_struct->fw_ring_id);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2562,7 +2647,7 @@ int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf)
 	uint32_t func_cfg_flags;
 	int rc = 0;
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
 
 	if (is_vf) {
 		dflt_vlan = bp->pf.vf_info[vf].dflt_vlan;
@@ -2580,7 +2665,9 @@ int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf)
 	req.dflt_vlan = rte_cpu_to_le_16(dflt_vlan);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2592,13 +2679,16 @@ int bnxt_hwrm_func_bw_cfg(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
+
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.enables |= rte_cpu_to_le_32(enables);
 	req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags);
 	req.max_bw = rte_cpu_to_le_32(max_bw);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2609,14 +2699,17 @@ int bnxt_hwrm_set_vf_vlan(struct bnxt *bp, int vf)
 	struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc = 0;
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
+
 	req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags);
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.enables |= rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN);
 	req.dflt_vlan = rte_cpu_to_le_16(bp->pf.vf_info[vf].dflt_vlan);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2631,14 +2724,15 @@ int bnxt_hwrm_reject_fwd_resp(struct bnxt *bp, uint16_t target_id,
 	if (ec_size > sizeof(req.encap_request))
 		return -1;
 
-	HWRM_PREP(req, REJECT_FWD_RESP, -1, resp);
+	HWRM_PREP(req, REJECT_FWD_RESP);
 
 	req.encap_resp_target_id = rte_cpu_to_le_16(target_id);
 	memcpy(req.encap_request, encaped, ec_size);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2650,13 +2744,17 @@ int bnxt_hwrm_func_qcfg_vf_default_mac(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc;
 
-	HWRM_PREP(req, FUNC_QCFG, -1, resp);
+	HWRM_PREP(req, FUNC_QCFG);
+
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	memcpy(mac->addr_bytes, resp->mac_address, ETHER_ADDR_LEN);
+
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2670,14 +2768,15 @@ int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, uint16_t target_id,
 	if (ec_size > sizeof(req.encap_request))
 		return -1;
 
-	HWRM_PREP(req, EXEC_FWD_RESP, -1, resp);
+	HWRM_PREP(req, EXEC_FWD_RESP);
 
 	req.encap_resp_target_id = rte_cpu_to_le_16(target_id);
 	memcpy(req.encap_request, encaped, ec_size);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2689,13 +2788,13 @@ int bnxt_hwrm_ctx_qstats(struct bnxt *bp, uint32_t cid, int idx,
 	struct hwrm_stat_ctx_query_input req = {.req_type = 0};
 	struct hwrm_stat_ctx_query_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, STAT_CTX_QUERY, -1, resp);
+	HWRM_PREP(req, STAT_CTX_QUERY);
 
 	req.stat_ctx_id = rte_cpu_to_le_32(cid);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT;
+	HWRM_CHECK_RESULT();
 
 	stats->q_ipackets[idx] = rte_le_to_cpu_64(resp->rx_ucast_pkts);
 	stats->q_ipackets[idx] += rte_le_to_cpu_64(resp->rx_mcast_pkts);
@@ -2715,6 +2814,8 @@ int bnxt_hwrm_ctx_qstats(struct bnxt *bp, uint32_t cid, int idx,
 	stats->q_errors[idx] += rte_le_to_cpu_64(resp->tx_err_pkts);
 	stats->q_errors[idx] += rte_le_to_cpu_64(resp->rx_drop_pkts);
 
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2728,12 +2829,16 @@ int bnxt_hwrm_port_qstats(struct bnxt *bp)
 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
 		return 0;
 
-	HWRM_PREP(req, PORT_QSTATS, -1, resp);
+	HWRM_PREP(req, PORT_QSTATS);
+
 	req.port_id = rte_cpu_to_le_16(pf->port_id);
 	req.tx_stat_host_addr = rte_cpu_to_le_64(bp->hw_tx_port_stats_map);
 	req.rx_stat_host_addr = rte_cpu_to_le_64(bp->hw_rx_port_stats_map);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2747,10 +2852,14 @@ int bnxt_hwrm_port_clr_stats(struct bnxt *bp)
 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
 		return 0;
 
-	HWRM_PREP(req, PORT_CLR_STATS, -1, resp);
+	HWRM_PREP(req, PORT_CLR_STATS);
+
 	req.port_id = rte_cpu_to_le_16(pf->port_id);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2763,10 +2872,11 @@ int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
 	if (BNXT_VF(bp))
 		return 0;
 
-	HWRM_PREP(req, PORT_LED_QCAPS, -1, resp);
+	HWRM_PREP(req, PORT_LED_QCAPS);
 	req.port_id = bp->pf.port_id;
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
 
 	if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
 		unsigned int i;
@@ -2786,6 +2896,9 @@ int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
 			}
 		}
 	}
+
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -2801,7 +2914,8 @@ int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on)
 	if (!bp->num_leds || BNXT_VF(bp))
 		return -EOPNOTSUPP;
 
-	HWRM_PREP(req, PORT_LED_CFG, -1, resp);
+	HWRM_PREP(req, PORT_LED_CFG);
+
 	if (led_on) {
 		led_state = HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINKALT;
 		duration = rte_cpu_to_le_16(500);
@@ -2819,7 +2933,9 @@ int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on)
 	}
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -2857,28 +2973,34 @@ static int bnxt_hwrm_func_vf_vnic_query(struct bnxt *bp, uint16_t vf,
 	int rc;
 
 	/* First query all VNIC ids */
-	HWRM_PREP(req, FUNC_VF_VNIC_IDS_QUERY, -1, resp_vf_vnic_ids);
+	HWRM_PREP(req, FUNC_VF_VNIC_IDS_QUERY);
 
 	req.vf_id = rte_cpu_to_le_16(bp->pf.first_vf_id + vf);
 	req.max_vnic_id_cnt = rte_cpu_to_le_32(bp->pf.total_vnics);
 	req.vnic_id_tbl_addr = rte_cpu_to_le_64(rte_mem_virt2phy(vnic_ids));
 
 	if (req.vnic_id_tbl_addr == 0) {
+		HWRM_UNLOCK();
 		RTE_LOG(ERR, PMD,
 		"unable to map VNIC ID table address to physical memory\n");
 		return -ENOMEM;
 	}
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 	if (rc) {
+		HWRM_UNLOCK();
 		RTE_LOG(ERR, PMD, "hwrm_func_vf_vnic_query failed rc:%d\n", rc);
 		return -1;
 	} else if (resp->error_code) {
 		rc = rte_le_to_cpu_16(resp->error_code);
+		HWRM_UNLOCK();
 		RTE_LOG(ERR, PMD, "hwrm_func_vf_vnic_query error %d\n", rc);
 		return -1;
 	}
+	rc = rte_le_to_cpu_32(resp->vnic_id_cnt);
 
-	return rte_le_to_cpu_32(resp->vnic_id_cnt);
+	HWRM_UNLOCK();
+
+	return rc;
 }
 
 /*
@@ -2943,7 +3065,8 @@ int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG, -1, resp);
+	HWRM_PREP(req, FUNC_CFG);
+
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.enables |= rte_cpu_to_le_32(
 			HWRM_FUNC_CFG_INPUT_ENABLES_VLAN_ANTISPOOF_MODE);
@@ -2951,7 +3074,9 @@ int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf,
 		HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN :
 		HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_NOCHECK;
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT;
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
-- 
2.10.1 (Apple Git-78)

Re: [PATCH 7/8] net/bnxt: fix HWRM_*() macros and locking

[Ferruh Yigit]

On 8/24/2017 5:29 PM, Ajit Khaparde wrote:
> Obtain the spinlock in HWRM_PREP()
> Eliminate two unnecessary arguments in HWRM_PREP().
> Unlock the spinlock before returning in HWRM_ERROR_CHECK()
> Add new HWRM_UNLOCK() macro
> Update usage of the thre macros.
> 
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

<...>

> -#define HWRM_PREP(req, type, cr, resp) \
> +/*
> + * HWRM_PREP() should be used to prepare *ALL* HWRM commands.  It grabs the
> + * spinlock, and does initial processing.
> + *
> + * HWRM_CHECK_RESULT() returns errors on failure and may not be used.  It
> + * releases the spinlock only if it returns.  If the regular int return codes
> + * are not used by the function, HWRM_CHECK_RESULT() should not be used
> + * directly, rather it should be copied and modified to suit the function.
> + *
> + * HWRM_UNLOCK() must be called after all response processing is completed.
> + */
> +#define HWRM_PREP(x, type) do { \
> +	typeof(x) req = (x); \
> +	rte_spinlock_lock(&bp->hwrm_lock); \
>  	memset(bp->hwrm_cmd_resp_addr, 0, bp->max_resp_len); \
>  	req.req_type = rte_cpu_to_le_16(HWRM_##type); \
> -	req.cmpl_ring = rte_cpu_to_le_16(cr); \
> +	req.cmpl_ring = rte_cpu_to_le_16(-1); \
>  	req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); \
>  	req.target_id = rte_cpu_to_le_16(0xffff); \
> -	req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr)
> +	req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr); \
> +} while (0)
>  

<...>

> -	HWRM_PREP(req, CFA_L2_SET_RX_MASK, -1, resp);
> +	HWRM_PREP(req, CFA_L2_SET_RX_MASK);

Getting following build error [1] many times with clang (and icc).
I guess this is because using variable name in macro itself.

[1]
.../drivers/net/bnxt/bnxt_hwrm.c:235:12: error: variable 'req' is
uninitialized when used within its own initialization
[-Werror,-Wuninitialized]

        HWRM_PREP(req, CFA_L2_SET_RX_MASK);
        ~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~
.../drivers/net/bnxt/bnxt_hwrm.c:186:19: note: expanded from macro
'HWRM_PREP'
        typeof(x) req = (x); \
                  ~~~    ^

[PATCH 8/8] net/bnxt: add support for flow filter ops

[Ajit Khaparde]

This patch adds support for flow validate/create/destroy/flush ops.
Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/bnxt.h         |   9 +
 drivers/net/bnxt/bnxt_ethdev.c  |  50 ++-
 drivers/net/bnxt/bnxt_filter.c  | 661 +++++++++++++++++++++++++++++++++++++++-
 drivers/net/bnxt/bnxt_filter.h  |  50 +++
 drivers/net/bnxt/bnxt_hwrm.c    | 206 ++++++++++++-
 drivers/net/bnxt/bnxt_hwrm.h    |  12 +-
 drivers/net/bnxt/bnxt_rxq.c     |  45 ++-
 drivers/net/bnxt/rte_pmd_bnxt.c |   4 +-
 8 files changed, 1011 insertions(+), 26 deletions(-)

diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h
index 405d94d..bf0ab9b 100644
--- a/drivers/net/bnxt/bnxt.h
+++ b/drivers/net/bnxt/bnxt.h
@@ -171,6 +171,13 @@ struct bnxt_cos_queue_info {
 	uint8_t	profile;
 };
 
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) node;
+	struct bnxt_filter_info *filter;
+};
+
+TAILQ_HEAD(bnxt_flow_list, rte_flow);
+
 #define BNXT_HWRM_SHORT_REQ_LEN		sizeof(struct hwrm_short_input)
 struct bnxt {
 	void				*bar0;
@@ -261,6 +268,7 @@ struct bnxt {
 
 	struct bnxt_led_info	leds[BNXT_MAX_LED];
 	uint8_t			num_leds;
+	struct bnxt_flow_list   flow_list;
 };
 
 int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete);
@@ -269,4 +277,5 @@ int bnxt_rcv_msg_from_vf(struct bnxt *bp, uint16_t vf_id, void *msg);
 #define RX_PROD_AGG_BD_TYPE_RX_PROD_AGG		0x6
 
 bool is_bnxt_supported(struct rte_eth_dev *dev);
+extern const struct rte_flow_ops bnxt_flow_ops;
 #endif
diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index 088cf6a..d64be4a 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -610,7 +610,7 @@ static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
 				if (filter->mac_index == index) {
 					STAILQ_REMOVE(&vnic->filter, filter,
 						      bnxt_filter_info, next);
-					bnxt_hwrm_clear_filter(bp, filter);
+					bnxt_hwrm_clear_l2_filter(bp, filter);
 					filter->mac_index = INVALID_MAC_INDEX;
 					memset(&filter->l2_addr, 0,
 					       ETHER_ADDR_LEN);
@@ -657,7 +657,7 @@ static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev,
 	STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
 	filter->mac_index = index;
 	memcpy(filter->l2_addr, mac_addr, ETHER_ADDR_LEN);
-	return bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id, filter);
+	return bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
 }
 
 int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete)
@@ -1147,7 +1147,7 @@ static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 					/* Must delete the filter */
 					STAILQ_REMOVE(&vnic->filter, filter,
 						      bnxt_filter_info, next);
-					bnxt_hwrm_clear_filter(bp, filter);
+					bnxt_hwrm_clear_l2_filter(bp, filter);
 					STAILQ_INSERT_TAIL(
 							&bp->free_filter_list,
 							filter, next);
@@ -1173,7 +1173,7 @@ static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 					memcpy(new_filter->l2_addr,
 					       filter->l2_addr, ETHER_ADDR_LEN);
 					/* MAC only filter */
-					rc = bnxt_hwrm_set_filter(bp,
+					rc = bnxt_hwrm_set_l2_filter(bp,
 							vnic->fw_vnic_id,
 							new_filter);
 					if (rc)
@@ -1225,7 +1225,7 @@ static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 					/* Must delete the MAC filter */
 					STAILQ_REMOVE(&vnic->filter, filter,
 						      bnxt_filter_info, next);
-					bnxt_hwrm_clear_filter(bp, filter);
+					bnxt_hwrm_clear_l2_filter(bp, filter);
 					filter->l2_ovlan = 0;
 					STAILQ_INSERT_TAIL(
 							&bp->free_filter_list,
@@ -1248,8 +1248,9 @@ static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 				new_filter->l2_ovlan = vlan_id;
 				new_filter->l2_ovlan_mask = 0xF000;
 				new_filter->enables |= en;
-				rc = bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id,
-							  new_filter);
+				rc = bnxt_hwrm_set_l2_filter(bp,
+							     vnic->fw_vnic_id,
+							     new_filter);
 				if (rc)
 					goto exit;
 				RTE_LOG(INFO, PMD,
@@ -1328,7 +1329,7 @@ bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev, struct ether_addr *addr)
 		/* Default Filter is at Index 0 */
 		if (filter->mac_index != 0)
 			continue;
-		rc = bnxt_hwrm_clear_filter(bp, filter);
+		rc = bnxt_hwrm_clear_l2_filter(bp, filter);
 		if (rc)
 			break;
 		memcpy(filter->l2_addr, bp->mac_addr, ETHER_ADDR_LEN);
@@ -1337,7 +1338,7 @@ bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev, struct ether_addr *addr)
 		filter->enables |=
 			HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR |
 			HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK;
-		rc = bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id, filter);
+		rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
 		if (rc)
 			break;
 		filter->mac_index = 0;
@@ -1611,6 +1612,36 @@ bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset)
 	return RTE_ETH_TX_DESC_FULL;
 }
 
+static int
+bnxt_filter_ctrl_op(struct rte_eth_dev *dev __rte_unused,
+		    enum rte_filter_type filter_type,
+		    enum rte_filter_op filter_op, void *arg)
+{
+	int ret = 0;
+
+	switch (filter_type) {
+	case RTE_ETH_FILTER_NTUPLE:
+	case RTE_ETH_FILTER_ETHERTYPE:
+	case RTE_ETH_FILTER_FDIR:
+	case RTE_ETH_FILTER_TUNNEL:
+		/* FALLTHROUGH */
+		RTE_LOG(ERR, PMD,
+			"filter type: %d: To be implemented\n", filter_type);
+		break;
+	case RTE_ETH_FILTER_GENERIC:
+		if (filter_op != RTE_ETH_FILTER_GET)
+			return -EINVAL;
+		*(const void **)arg = &bnxt_flow_ops;
+		break;
+	default:
+		RTE_LOG(ERR, PMD,
+			"Filter type (%d) not supported", filter_type);
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
 /*
  * Initialization
  */
@@ -1664,6 +1695,7 @@ static const struct eth_dev_ops bnxt_dev_ops = {
 	.rx_descriptor_status = bnxt_rx_descriptor_status_op,
 	.rx_descriptor_done   = bnxt_rx_descriptor_done_op,
 	.tx_descriptor_status = bnxt_tx_descriptor_status_op,
+	.filter_ctrl = bnxt_filter_ctrl_op,
 };
 
 static bool bnxt_vf_pciid(uint16_t id)
diff --git a/drivers/net/bnxt/bnxt_filter.c b/drivers/net/bnxt/bnxt_filter.c
index e9aac27..d0e11eb 100644
--- a/drivers/net/bnxt/bnxt_filter.c
+++ b/drivers/net/bnxt/bnxt_filter.c
@@ -35,6 +35,9 @@
 
 #include <rte_log.h>
 #include <rte_malloc.h>
+#include <rte_flow.h>
+#include <rte_flow_driver.h>
+#include <rte_tailq.h>
 
 #include "bnxt.h"
 #include "bnxt_filter.h"
@@ -121,7 +124,7 @@ void bnxt_free_all_filters(struct bnxt *bp)
 
 	for (i = 0; i < bp->pf.max_vfs; i++) {
 		STAILQ_FOREACH(filter, &bp->pf.vf_info[i].filter, next) {
-			bnxt_hwrm_clear_filter(bp, filter);
+			bnxt_hwrm_clear_l2_filter(bp, filter);
 		}
 	}
 }
@@ -142,7 +145,7 @@ void bnxt_free_filter_mem(struct bnxt *bp)
 		if (filter->fw_l2_filter_id != ((uint64_t)-1)) {
 			RTE_LOG(ERR, PMD, "HWRM filter is not freed??\n");
 			/* Call HWRM to try to free filter again */
-			rc = bnxt_hwrm_clear_filter(bp, filter);
+			rc = bnxt_hwrm_clear_l2_filter(bp, filter);
 			if (rc)
 				RTE_LOG(ERR, PMD,
 				       "HWRM filter cannot be freed rc = %d\n",
@@ -174,3 +177,657 @@ int bnxt_alloc_filter_mem(struct bnxt *bp)
 	bp->filter_info = filter_mem;
 	return 0;
 }
+
+static struct bnxt_filter_info *bnxt_get_unused_filter(struct bnxt *bp)
+{
+	struct bnxt_filter_info *filter;
+
+	/* Find the 1st unused filter from the free_filter_list pool*/
+	filter = STAILQ_FIRST(&bp->free_filter_list);
+	if (!filter) {
+		RTE_LOG(ERR, PMD, "No more free filter resources\n");
+		return NULL;
+	}
+	STAILQ_REMOVE_HEAD(&bp->free_filter_list, next);
+
+	return filter;
+}
+
+static void bnxt_free_filter(struct bnxt *bp, struct bnxt_filter_info *filter)
+{
+	STAILQ_INSERT_TAIL(&bp->free_filter_list, filter, next);
+}
+
+static int
+bnxt_flow_agrs_validate(const struct rte_flow_attr *attr,
+			const struct rte_flow_item pattern[],
+			const struct rte_flow_action actions[],
+			struct rte_flow_error *error)
+{
+	if (!pattern) {
+		rte_flow_error_set(error, EINVAL,
+			RTE_FLOW_ERROR_TYPE_ITEM_NUM,
+			NULL, "NULL pattern.");
+		return -rte_errno;
+	}
+
+	if (!actions) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ACTION_NUM,
+				   NULL, "NULL action.");
+		return -rte_errno;
+	}
+
+	if (!attr) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR,
+				   NULL, "NULL attribute.");
+		return -rte_errno;
+	}
+
+	return 0;
+}
+
+static const struct rte_flow_item *
+nxt_non_void_pattern(const struct rte_flow_item *cur)
+{
+	while (1) {
+		if (cur->type != RTE_FLOW_ITEM_TYPE_VOID)
+			return cur;
+		cur++;
+	}
+}
+
+static const struct rte_flow_action *
+nxt_non_void_action(const struct rte_flow_action *cur)
+{
+	while (1) {
+		if (cur->type != RTE_FLOW_ACTION_TYPE_VOID)
+			return cur;
+		cur++;
+	}
+}
+
+static inline int check_zero_bytes(const uint8_t *bytes, int len)
+{
+	int i;
+	for (i = 0; i < len; i++)
+		if (bytes[i] != 0x00)
+			return 0;
+	return 1;
+}
+
+static int
+bnxt_filter_type_check(const struct rte_flow_item pattern[],
+		       struct rte_flow_error *error)
+{
+	const struct rte_flow_item *item = nxt_non_void_pattern(pattern);
+	int use_ntuple = 1;
+
+	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
+		switch (item->type) {
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			use_ntuple = 1;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			use_ntuple = 0;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+		case RTE_FLOW_ITEM_TYPE_TCP:
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			/* FALLTHROUGH */
+			/* need ntuple match, reset exact match */
+			if (!use_ntuple) {
+				RTE_LOG(ERR, PMD,
+					"VLAN flow cannot use NTUPLE filter\n");
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Cannot use VLAN with NTUPLE");
+				return -rte_errno;
+			}
+			use_ntuple |= 1;
+			break;
+		default:
+			RTE_LOG(ERR, PMD, "Unknown Flow type");
+			use_ntuple |= 1;
+			//return -1;
+		}
+		item++;
+	}
+	return use_ntuple;
+}
+
+static int
+bnxt_validate_and_parse_flow_type(const struct rte_flow_item pattern[],
+				  struct rte_flow_error *error,
+				  struct bnxt_filter_info *filter)
+{
+	const struct rte_flow_item *item = nxt_non_void_pattern(pattern);
+	const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
+	const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask;
+	const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask;
+	const struct rte_flow_item_tcp *tcp_spec, *tcp_mask;
+	const struct rte_flow_item_udp *udp_spec, *udp_mask;
+	const struct rte_flow_item_eth *eth_spec, *eth_mask;
+	int use_ntuple;
+	uint32_t en = 0;
+
+	use_ntuple = bnxt_filter_type_check(pattern, error);
+	RTE_LOG(ERR, PMD, "Use NTUPLE %d\n", use_ntuple);
+	if (use_ntuple < 0)
+		return use_ntuple;
+
+	filter->filter_type = use_ntuple ?
+		HWRM_CFA_NTUPLE_FILTER : HWRM_CFA_EM_FILTER;
+
+	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
+		if (item->last) {
+			/* last or range is NOT supported as match criteria */
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "No support for range");
+			return -rte_errno;
+		}
+		if (!item->spec || !item->mask) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "spec/mask is NULL");
+			return -rte_errno;
+		}
+		switch (item->type) {
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			//filter_TYPE = EM
+			eth_spec = (const struct rte_flow_item_eth *)item->spec;
+			eth_mask = (const struct rte_flow_item_eth *)item->mask;
+
+			/* Source MAC address mask cannot be partially set.
+			 * Should be All 0's or all 1's.
+			 * Destination MAC address mask must not be partially
+			 * set. Should be all 1's or all 0's.
+			 */
+			if ((!is_zero_ether_addr(&eth_mask->src) &&
+			     !is_broadcast_ether_addr(&eth_mask->src)) ||
+			    (!is_zero_ether_addr(&eth_mask->dst) &&
+			     !is_broadcast_ether_addr(&eth_mask->dst))) {
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "MAC_addr mask not valid");
+				return -rte_errno;
+			}
+
+			/* Mask is not allowed. Only exact matches are */
+			if ((eth_mask->type & UINT16_MAX) != UINT16_MAX) {
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "ethertype mask not valid");
+				return -rte_errno;
+			}
+
+			if (is_broadcast_ether_addr(&eth_mask->dst)) {
+				rte_memcpy(filter->dst_macaddr,
+					   &eth_spec->dst, 6);
+				en |= use_ntuple ?
+					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR :
+					EM_FLOW_ALLOC_INPUT_EN_DST_MACADDR;
+				RTE_LOG(ERR, PMD, "DST MAC\n");
+			}
+			if (is_broadcast_ether_addr(&eth_mask->src)) {
+				rte_memcpy(filter->src_macaddr,
+					   &eth_spec->src, 6);
+				en |= use_ntuple ?
+					NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR :
+					EM_FLOW_ALLOC_INPUT_EN_SRC_MACADDR;
+				RTE_LOG(ERR, PMD, "SRC MAC\n");
+			} /*
+			   * else {
+			   *  RTE_LOG(ERR, PMD, "Handle this condition\n");
+			   * }
+			   */
+			if (eth_spec->type) {
+				filter->ethertype =
+					rte_be_to_cpu_16(eth_spec->type);
+				en |= use_ntuple ?
+					NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE :
+					EM_FLOW_ALLOC_INPUT_EN_ETHERTYPE;
+			}
+
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			//filter_TYPE = EM
+			vlan_spec =
+				(const struct rte_flow_item_vlan *)item->spec;
+			vlan_mask =
+				(const struct rte_flow_item_vlan *)item->mask;
+			if (vlan_mask->tci & 0xFFFF && !vlan_mask->tpid) {
+				//Only the VLAN ID can be matched.
+				filter->l2_ovlan =
+					rte_be_to_cpu_16(vlan_spec->tci &
+							 0xFFF);
+				en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID;
+			} else {
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "VLAN mask is invalid");
+				return -rte_errno;
+			}
+
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			//If mask is not involved, we could use EM filters.
+			ipv4_spec =
+				(const struct rte_flow_item_ipv4 *)item->spec;
+			ipv4_mask =
+				(const struct rte_flow_item_ipv4 *)item->mask;
+			/* Only IP DST and SRC fields are maskable. */
+			if (ipv4_mask->hdr.version_ihl ||
+			    ipv4_mask->hdr.type_of_service ||
+			    ipv4_mask->hdr.total_length ||
+			    ipv4_mask->hdr.packet_id ||
+			    ipv4_mask->hdr.fragment_offset ||
+			    ipv4_mask->hdr.time_to_live ||
+			    ipv4_mask->hdr.next_proto_id ||
+			    ipv4_mask->hdr.hdr_checksum) {
+				rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "Invalid IPv4 mask.");
+				return -rte_errno;
+			}
+			filter->dst_ipaddr[0] = ipv4_spec->hdr.dst_addr;
+			filter->src_ipaddr[0] = ipv4_spec->hdr.src_addr;
+			en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
+				NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
+			if (ipv4_mask->hdr.src_addr) {
+				filter->src_ipaddr_mask[0] =
+					ipv4_mask->hdr.src_addr;
+				en |=
+				     NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
+			}
+			if (ipv4_mask->hdr.dst_addr) {
+				filter->dst_ipaddr_mask[0] =
+					ipv4_mask->hdr.dst_addr;
+				en |=
+				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ipv6_spec =
+				(const struct rte_flow_item_ipv6 *)item->spec;
+			ipv6_mask =
+				(const struct rte_flow_item_ipv6 *)item->mask;
+
+			/* Only IP DST and SRC fields are maskable. */
+			if (ipv6_mask->hdr.vtc_flow ||
+			    ipv6_mask->hdr.payload_len ||
+			    ipv6_mask->hdr.proto ||
+			    ipv6_mask->hdr.hop_limits) {
+				rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "Invalid IPv6 mask.");
+				return -rte_errno;
+			}
+
+			en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
+				NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
+			rte_memcpy(filter->src_ipaddr,
+				   ipv6_spec->hdr.src_addr, 16);
+			rte_memcpy(filter->dst_ipaddr,
+				   ipv6_spec->hdr.dst_addr, 16);
+			if (!check_zero_bytes(ipv6_mask->hdr.src_addr, 16)) {
+				rte_memcpy(filter->src_ipaddr_mask,
+					   ipv6_mask->hdr.src_addr, 16);
+				en |=
+				     NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
+			}
+			if (!check_zero_bytes(ipv6_mask->hdr.dst_addr, 16)) {
+				rte_memcpy(filter->dst_ipaddr_mask,
+					   ipv6_mask->hdr.dst_addr, 16);
+				en |=
+				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			tcp_spec = (const struct rte_flow_item_tcp *)item->spec;
+			tcp_mask = (const struct rte_flow_item_tcp *)item->mask;
+
+			/* Check TCP mask. Only DST & SRC ports are maskable */
+			if (tcp_mask->hdr.sent_seq ||
+			    tcp_mask->hdr.recv_ack ||
+			    tcp_mask->hdr.data_off ||
+			    tcp_mask->hdr.tcp_flags ||
+			    tcp_mask->hdr.rx_win ||
+			    tcp_mask->hdr.cksum ||
+			    tcp_mask->hdr.tcp_urp) {
+				rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "Invalid TCP mask");
+				return -rte_errno;
+			}
+			filter->src_port = tcp_spec->hdr.src_port;
+			filter->dst_port = tcp_spec->hdr.dst_port;
+			en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
+				NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
+			if (tcp_mask->hdr.dst_port) {
+				filter->dst_port_mask = tcp_mask->hdr.dst_port;
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
+			}
+			if (tcp_mask->hdr.src_port) {
+				filter->src_port_mask = tcp_mask->hdr.src_port;
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			udp_spec = (const struct rte_flow_item_udp *)item->spec;
+			udp_mask = (const struct rte_flow_item_udp *)item->mask;
+
+			if (udp_mask->hdr.dgram_len ||
+			    udp_mask->hdr.dgram_cksum) {
+				rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "Invalid UDP mask");
+				return -rte_errno;
+			}
+
+			filter->src_port = udp_spec->hdr.src_port;
+			filter->dst_port = udp_spec->hdr.dst_port;
+			en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
+				NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
+
+			if (udp_mask->hdr.dst_port) {
+				filter->dst_port_mask = udp_mask->hdr.dst_port;
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
+			}
+			if (udp_mask->hdr.src_port) {
+				filter->src_port_mask = udp_mask->hdr.src_port;
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
+			}
+			break;
+		default:
+			break;
+		}
+		item++;
+	}
+	filter->enables = en;
+
+	return 0;
+}
+
+/* Parse attributes */
+static int
+bnxt_flow_parse_attr(const struct rte_flow_attr *attr,
+		     struct rte_flow_error *error)
+{
+	/* Must be input direction */
+	if (!attr->ingress) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				   attr, "Only support ingress.");
+		return -rte_errno;
+	}
+
+	/* Not supported */
+	if (attr->egress) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+				   attr, "No support for egress.");
+		return -rte_errno;
+	}
+
+	/* Not supported */
+	if (attr->priority) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+				   attr, "No support for priority.");
+		return -rte_errno;
+	}
+
+	/* Not supported */
+	if (attr->group) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+				   attr, "No support for group.");
+		return -rte_errno;
+	}
+
+	return 0;
+}
+
+static int
+bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
+			     const struct rte_flow_item pattern[],
+			     const struct rte_flow_action actions[],
+			     const struct rte_flow_attr *attr,
+			     struct rte_flow_error *error,
+			     struct bnxt_filter_info *filter)
+{
+	const struct rte_flow_action *act = nxt_non_void_action(actions);
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	const struct rte_flow_action_queue *act_q;
+	struct bnxt_vnic_info *vnic;
+	int rc;
+
+	rc = bnxt_validate_and_parse_flow_type(pattern, error, filter);
+	if (rc != 0)
+		goto ret;
+
+	rc = bnxt_flow_parse_attr(attr, error);
+	if (rc != 0)
+		goto ret;
+	//Since we support ingress attribute only - right now.
+	filter->flags = HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX;
+
+	switch (act->type) {
+	case RTE_FLOW_ACTION_TYPE_QUEUE:
+		//Allow this flow. Redirect to a VNIC.
+		act_q = (const struct rte_flow_action_queue *)act->conf;
+		if (act_q->index >= bp->rx_nr_rings) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION, act,
+					   "Invalid queue ID.");
+			rc = -rte_errno;
+			goto ret;
+		}
+		RTE_LOG(ERR, PMD, "Queue index %d\n", act_q->index);
+
+		vnic = STAILQ_FIRST(&bp->ff_pool[act_q->index]);
+		if (vnic == NULL) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION, act,
+					   "No matching VNIC for queue ID.");
+			rc = -rte_errno;
+			goto ret;
+		}
+		filter->dst_id = vnic->fw_vnic_id;
+		RTE_LOG(ERR, PMD, "VNIC found\n");
+		break;
+	case RTE_FLOW_ACTION_TYPE_DROP:
+		if (filter->filter_type == HWRM_CFA_EM_FILTER)
+			filter->flags |=
+				HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP;
+		else
+			filter->flags |=
+				HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
+		//HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_DROP;
+		//HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_DROP;
+		//HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP
+		//HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP
+		//HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_DROP
+		break;
+	default:
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ACTION, act,
+				   "Invalid action.");
+		rc = -rte_errno;
+		goto ret;
+	}
+
+	act = nxt_non_void_action(++act);
+	if (act->type != RTE_FLOW_ACTION_TYPE_END) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ACTION,
+				   act, "Invalid action.");
+		rc = -rte_errno;
+		goto ret;
+	}
+ret:
+	return rc;
+}
+
+static int
+bnxt_flow_validate(struct rte_eth_dev *dev,
+		const struct rte_flow_attr *attr,
+		const struct rte_flow_item pattern[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct bnxt_filter_info *filter;
+	int ret = 0;
+
+	ret = bnxt_flow_agrs_validate(attr, pattern, actions, error);
+	if (ret != 0)
+		return ret;
+
+	filter = bnxt_get_unused_filter(bp);
+	if (filter == NULL) {
+		RTE_LOG(ERR, PMD, "Not enough resources for a new flow.\n");
+		return -ENOMEM;
+	}
+
+	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
+					   error, filter);
+	/* No need to hold on to this filter if we are just validating flow */
+	bnxt_free_filter(bp, filter);
+
+	return ret;
+}
+
+static struct rte_flow *
+bnxt_flow_create(struct rte_eth_dev *dev,
+		  const struct rte_flow_attr *attr,
+		  const struct rte_flow_item pattern[],
+		  const struct rte_flow_action actions[],
+		  struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct bnxt_filter_info *filter;
+	struct rte_flow *flow;
+	int ret = 0;
+
+	flow = rte_zmalloc("bnxt_flow", sizeof(struct rte_flow), 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Failed to allocate memory");
+		return flow;
+	}
+
+	ret = bnxt_flow_agrs_validate(attr, pattern, actions, error);
+	if (ret != 0) {
+		RTE_LOG(ERR, PMD, "Not a validate flow.\n");
+		goto free_flow;
+	}
+
+	filter = bnxt_get_unused_filter(bp);
+	if (filter == NULL) {
+		RTE_LOG(ERR, PMD, "Not enough resources for a new flow.\n");
+		goto free_flow;
+	}
+
+	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
+					   error, filter);
+	if (ret != 0)
+		goto free_flow;
+
+	if (filter->filter_type == HWRM_CFA_EM_FILTER)
+		ret = bnxt_hwrm_set_em_filter(bp, filter->dst_id, filter);
+	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
+		ret = bnxt_hwrm_set_ntuple_filter(bp, filter->dst_id, filter);
+
+	if (!ret) {
+		RTE_LOG(ERR, PMD, "Successfully created flow.\n");
+		TAILQ_INSERT_TAIL(&bp->flow_list, flow, node);
+		return flow;
+	}
+free_flow:
+	RTE_LOG(ERR, PMD, "Failed to create flow.\n");
+	rte_flow_error_set(error, -ret,
+			   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+			   "Failed to create flow.");
+	rte_free(flow);
+	flow = NULL;
+	return flow;
+}
+
+static int
+bnxt_flow_destroy(struct rte_eth_dev *dev,
+		  struct rte_flow *flow,
+		  struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct bnxt_filter_info *filter = flow->filter;
+	int ret = 0;
+
+	if (filter->filter_type == HWRM_CFA_EM_FILTER)
+		ret = bnxt_hwrm_clear_em_filter(bp, filter);
+	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
+		ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
+
+	if (!ret) {
+		TAILQ_REMOVE(&bp->flow_list, flow, node);
+		rte_free(flow);
+	} else {
+		rte_flow_error_set(error, -ret,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Failed to destroy flow.");
+	}
+
+	return ret;
+}
+
+static int
+bnxt_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct rte_flow *flow;
+	int ret = 0;
+	void *temp;
+
+	TAILQ_FOREACH_SAFE(flow, &bp->flow_list, node, temp) {
+		struct bnxt_filter_info *filter = flow->filter;
+
+		if (filter->filter_type == HWRM_CFA_EM_FILTER)
+			ret = bnxt_hwrm_clear_em_filter(bp, filter);
+		if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
+			ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
+
+		if (ret) {
+			rte_flow_error_set(error, -ret,
+					   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+					   "Failed to flush flow in HW.");
+			return -rte_errno;
+		}
+
+		TAILQ_REMOVE(&bp->flow_list, flow, node);
+		rte_free(flow);
+	}
+
+	return ret;
+}
+
+const struct rte_flow_ops bnxt_flow_ops = {
+	.validate = bnxt_flow_validate,
+	.create = bnxt_flow_create,
+	.destroy = bnxt_flow_destroy,
+	.flush = bnxt_flow_flush,
+};
diff --git a/drivers/net/bnxt/bnxt_filter.h b/drivers/net/bnxt/bnxt_filter.h
index 613b2ee..357dcb1 100644
--- a/drivers/net/bnxt/bnxt_filter.h
+++ b/drivers/net/bnxt/bnxt_filter.h
@@ -40,8 +40,15 @@ struct bnxt;
 struct bnxt_filter_info {
 	STAILQ_ENTRY(bnxt_filter_info)	next;
 	uint64_t		fw_l2_filter_id;
+	uint64_t		fw_em_filter_id;
+	uint64_t		fw_ntuple_filter_id;
 #define INVALID_MAC_INDEX	((uint16_t)-1)
 	uint16_t		mac_index;
+#define HWRM_CFA_L2_FILTER     0
+#define HWRM_CFA_EM_FILTER     1
+#define HWRM_CFA_NTUPLE_FILTER 2
+	uint8_t                 filter_type;    //L2 or EM or NTUPLE filter
+	uint16_t                dst_id;
 
 	/* Filter Characteristics */
 	uint32_t		flags;
@@ -65,6 +72,19 @@ struct bnxt_filter_info {
 	uint64_t		l2_filter_id_hint;
 	uint32_t		src_id;
 	uint8_t			src_type;
+	uint8_t                 src_macaddr[6];
+	uint8_t                 dst_macaddr[6];
+	uint32_t                dst_ipaddr[4];
+	uint32_t                dst_ipaddr_mask[4];
+	uint32_t                src_ipaddr[4];
+	uint32_t                src_ipaddr_mask[4];
+	uint16_t                dst_port;
+	uint16_t                dst_port_mask;
+	uint16_t                src_port;
+	uint16_t                src_port_mask;
+	uint16_t                ip_protocol;
+	uint16_t                ip_addr_type;
+	uint16_t                ethertype;
 };
 
 struct bnxt_filter_info *bnxt_alloc_filter(struct bnxt *bp);
@@ -74,4 +94,34 @@ void bnxt_free_all_filters(struct bnxt *bp);
 void bnxt_free_filter_mem(struct bnxt *bp);
 int bnxt_alloc_filter_mem(struct bnxt *bp);
 
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR	\
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR
+#define EM_FLOW_ALLOC_INPUT_EN_SRC_MACADDR	\
+	HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_MACADDR
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR	\
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_MACADDR
+#define EM_FLOW_ALLOC_INPUT_EN_DST_MACADDR	\
+	HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_MACADDR
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE   \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_ETHERTYPE
+#define EM_FLOW_ALLOC_INPUT_EN_ETHERTYPE       \
+	HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ETHERTYPE
+#define EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID       \
+	HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_OVLAN_VID
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR  \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK     \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR_MASK
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR  \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK     \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR_MASK
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT    \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK       \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT_MASK
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT    \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT
+#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK       \
+	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT_MASK
 #endif
diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c
index 0fc5244..74d5c91 100644
--- a/drivers/net/bnxt/bnxt_hwrm.c
+++ b/drivers/net/bnxt/bnxt_hwrm.c
@@ -332,7 +332,7 @@ int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid,
 	return rc;
 }
 
-int bnxt_hwrm_clear_filter(struct bnxt *bp,
+int bnxt_hwrm_clear_l2_filter(struct bnxt *bp,
 			   struct bnxt_filter_info *filter)
 {
 	int rc = 0;
@@ -356,7 +356,7 @@ int bnxt_hwrm_clear_filter(struct bnxt *bp,
 	return 0;
 }
 
-int bnxt_hwrm_set_filter(struct bnxt *bp,
+int bnxt_hwrm_set_l2_filter(struct bnxt *bp,
 			 uint16_t dst_id,
 			 struct bnxt_filter_info *filter)
 {
@@ -366,7 +366,7 @@ int bnxt_hwrm_set_filter(struct bnxt *bp,
 	uint32_t enables = 0;
 
 	if (filter->fw_l2_filter_id != UINT64_MAX)
-		bnxt_hwrm_clear_filter(bp, filter);
+		bnxt_hwrm_clear_l2_filter(bp, filter);
 
 	HWRM_PREP(req, CFA_L2_FILTER_ALLOC);
 
@@ -1752,7 +1752,12 @@ int bnxt_clear_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	int rc = 0;
 
 	STAILQ_FOREACH(filter, &vnic->filter, next) {
-		rc = bnxt_hwrm_clear_filter(bp, filter);
+		if (filter->filter_type == HWRM_CFA_EM_FILTER)
+			rc = bnxt_hwrm_clear_em_filter(bp, filter);
+		else if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
+			rc = bnxt_hwrm_clear_ntuple_filter(bp, filter);
+		else
+			rc = bnxt_hwrm_clear_l2_filter(bp, filter);
 		if (rc)
 			break;
 	}
@@ -1765,7 +1770,7 @@ int bnxt_set_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	int rc = 0;
 
 	STAILQ_FOREACH(filter, &vnic->filter, next) {
-		rc = bnxt_hwrm_set_filter(bp, vnic->fw_vnic_id, filter);
+		rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
 		if (rc)
 			break;
 	}
@@ -3129,3 +3134,194 @@ int bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(struct bnxt *bp, int vf)
 	rte_free(vnic_ids);
 	return -1;
 }
+
+int bnxt_hwrm_set_em_filter(struct bnxt *bp,
+			 uint16_t dst_id,
+			 struct bnxt_filter_info *filter)
+{
+	int rc = 0;
+	struct hwrm_cfa_em_flow_alloc_input req = {.req_type = 0 };
+	struct hwrm_cfa_em_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr;
+	uint32_t enables = 0;
+
+	if (filter->fw_em_filter_id != UINT64_MAX)
+		bnxt_hwrm_clear_em_filter(bp, filter);
+
+	HWRM_PREP(req, CFA_EM_FLOW_ALLOC);
+
+	req.flags = rte_cpu_to_le_32(filter->flags);
+
+	enables = filter->enables |
+	      HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_ID;
+	req.dst_id = rte_cpu_to_le_16(dst_id);
+
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID)
+		req.l2_filter_id = filter->fw_l2_filter_id;
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_MACADDR)
+		memcpy(req.src_macaddr, filter->src_macaddr,
+		       ETHER_ADDR_LEN);
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_MACADDR)
+		memcpy(req.dst_macaddr, filter->dst_macaddr,
+		       ETHER_ADDR_LEN);
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_OVLAN_VID)
+		req.ovlan_vid = filter->l2_ovlan;
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IVLAN_VID)
+		req.ivlan_vid = filter->l2_ivlan;
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ETHERTYPE)
+		req.ethertype = filter->ethertype;
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_IPADDR)
+		req.src_ipaddr[0] = filter->src_ipaddr[0];
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_IPADDR)
+		req.dst_ipaddr[0] = filter->dst_ipaddr[0];
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_PORT)
+		req.src_port = filter->src_port;
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_PORT)
+		req.dst_port = filter->dst_port;
+	if (enables &
+	    HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID)
+		req.mirror_vnic_id = filter->mirror_vnic_id;
+
+	req.enables = rte_cpu_to_le_32(enables);
+
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+
+	HWRM_CHECK_RESULT();
+
+	filter->fw_em_filter_id = rte_le_to_cpu_64(resp->em_filter_id);
+	HWRM_UNLOCK();
+
+	return rc;
+}
+
+int bnxt_hwrm_clear_em_filter(struct bnxt *bp, struct bnxt_filter_info *filter)
+{
+	int rc = 0;
+	struct hwrm_cfa_em_flow_free_input req = {.req_type = 0 };
+	struct hwrm_cfa_em_flow_free_output *resp = bp->hwrm_cmd_resp_addr;
+
+	if (filter->fw_em_filter_id == UINT64_MAX)
+		return 0;
+
+	HWRM_PREP(req, CFA_EM_FLOW_FREE);
+
+	req.em_filter_id = rte_cpu_to_le_64(filter->fw_em_filter_id);
+
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
+
+	filter->fw_l2_filter_id = -1;
+
+	return 0;
+}
+
+int bnxt_hwrm_set_ntuple_filter(struct bnxt *bp,
+			 uint16_t dst_id,
+			 struct bnxt_filter_info *filter)
+{
+	int rc = 0;
+	struct hwrm_cfa_ntuple_filter_alloc_input req = {.req_type = 0 };
+	struct hwrm_cfa_ntuple_filter_alloc_output *resp =
+						bp->hwrm_cmd_resp_addr;
+	uint32_t enables = 0;
+
+	if (filter->fw_ntuple_filter_id != UINT64_MAX)
+		bnxt_hwrm_clear_ntuple_filter(bp, filter);
+
+	HWRM_PREP(req, CFA_NTUPLE_FILTER_ALLOC);
+
+	req.flags = rte_cpu_to_le_32(filter->flags);
+
+	enables = filter->enables |
+	      HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_ID;
+	req.dst_id = rte_cpu_to_le_16(dst_id);
+
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID)
+		req.l2_filter_id = filter->fw_l2_filter_id;
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR)
+		memcpy(req.src_macaddr, filter->src_macaddr,
+		       ETHER_ADDR_LEN);
+	//if (enables &
+	    //HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_MACADDR)
+		//memcpy(req.dst_macaddr, filter->dst_macaddr,
+		       //ETHER_ADDR_LEN);
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_ETHERTYPE)
+		req.ethertype = filter->ethertype;
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR)
+		req.src_ipaddr[0] = filter->src_ipaddr[0];
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR_MASK)
+		req.src_ipaddr_mask[0] = filter->src_ipaddr_mask[0];
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR)
+		req.dst_ipaddr[0] = filter->dst_ipaddr[0];
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR_MASK)
+		req.dst_ipaddr_mask[0] = filter->dst_ipaddr_mask[0];
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT)
+		req.src_port = filter->src_port;
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT_MASK)
+		req.src_port_mask = filter->src_port_mask;
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT)
+		req.dst_port = filter->dst_port;
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT_MASK)
+		req.dst_port_mask = filter->dst_port_mask;
+	if (enables &
+	    HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID)
+		req.mirror_vnic_id = filter->mirror_vnic_id;
+
+	req.enables = rte_cpu_to_le_32(enables);
+
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+
+	HWRM_CHECK_RESULT();
+
+	filter->fw_ntuple_filter_id = rte_le_to_cpu_64(resp->ntuple_filter_id);
+	HWRM_UNLOCK();
+
+	return rc;
+}
+
+int bnxt_hwrm_clear_ntuple_filter(struct bnxt *bp,
+				struct bnxt_filter_info *filter)
+{
+	int rc = 0;
+	struct hwrm_cfa_ntuple_filter_free_input req = {.req_type = 0 };
+	struct hwrm_cfa_ntuple_filter_free_output *resp =
+						bp->hwrm_cmd_resp_addr;
+
+	if (filter->fw_ntuple_filter_id == UINT64_MAX)
+		return 0;
+
+	HWRM_PREP(req, CFA_NTUPLE_FILTER_FREE);
+
+	req.ntuple_filter_id = rte_cpu_to_le_64(filter->fw_ntuple_filter_id);
+
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
+
+	filter->fw_l2_filter_id = -1;
+
+	return 0;
+}
diff --git a/drivers/net/bnxt/bnxt_hwrm.h b/drivers/net/bnxt/bnxt_hwrm.h
index 51cd0dd..bd9017f 100644
--- a/drivers/net/bnxt/bnxt_hwrm.h
+++ b/drivers/net/bnxt/bnxt_hwrm.h
@@ -51,9 +51,9 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic,
 int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid,
 			uint16_t vlan_count,
 			struct bnxt_vlan_antispoof_table_entry *vlan_table);
-int bnxt_hwrm_clear_filter(struct bnxt *bp,
+int bnxt_hwrm_clear_l2_filter(struct bnxt *bp,
 			   struct bnxt_filter_info *filter);
-int bnxt_hwrm_set_filter(struct bnxt *bp,
+int bnxt_hwrm_set_l2_filter(struct bnxt *bp,
 			 uint16_t dst_id,
 			 struct bnxt_filter_info *filter);
 int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, uint16_t target_id,
@@ -156,4 +156,12 @@ int bnxt_hwrm_func_vf_vnic_query_and_config(struct bnxt *bp, uint16_t vf,
 int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf,
 					      bool on);
 int bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(struct bnxt *bp, int vf);
+int bnxt_hwrm_set_em_filter(struct bnxt *bp, uint16_t dst_id,
+			struct bnxt_filter_info *filter);
+int bnxt_hwrm_clear_em_filter(struct bnxt *bp, struct bnxt_filter_info *filter);
+
+int bnxt_hwrm_set_ntuple_filter(struct bnxt *bp, uint16_t dst_id,
+			 struct bnxt_filter_info *filter);
+int bnxt_hwrm_clear_ntuple_filter(struct bnxt *bp,
+				struct bnxt_filter_info *filter);
 #endif
diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c
index 0793820..da53e99 100644
--- a/drivers/net/bnxt/bnxt_rxq.c
+++ b/drivers/net/bnxt/bnxt_rxq.c
@@ -98,7 +98,7 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
 	}
 
 	/* Multi-queue mode */
-	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
+	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB_RSS) {
 		/* VMDq ONLY, VMDq+RSS, VMDq+DCB, VMDq+DCB+RSS */
 		enum rte_eth_nb_pools pools;
 
@@ -113,6 +113,9 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
 				pools = conf->nb_queue_pools;
 				break;
 			}
+		case ETH_MQ_RX_RSS:
+			pools = 1;	//bp->rx_cp_nr_rings;
+			break;
 		default:
 			RTE_LOG(ERR, PMD, "Unsupported mq_mod %d\n",
 				dev_conf->rxmode.mq_mode);
@@ -203,12 +206,42 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
 	}
 	STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
 
-	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
-		vnic->hash_type =
-			HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4 |
-			HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
-
 out:
+	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
+		struct rte_eth_rss_conf *rss = &dev_conf->rx_adv_conf.rss_conf;
+		uint16_t hash_type = 0;
+
+		if (rss->rss_hf & ETH_RSS_IPV4)
+			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4;
+		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
+			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4;
+		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
+			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4;
+		if (rss->rss_hf & ETH_RSS_IPV6)
+			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
+		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
+			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6;
+		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
+			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
+
+		for (i = 0; i < bp->nr_vnics; i++) {
+			STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
+			vnic->hash_type |= hash_type;
+
+			/*
+			 * Use the supplied key if the key length is
+			 * acceptable and the rss_key is not NULL
+			 */
+			if (rss->rss_key &&
+			    rss->rss_key_len <= HW_HASH_KEY_SIZE)
+				memcpy(vnic->rss_hash_key,
+				       rss->rss_key, rss->rss_key_len);
+			}
+		}
+		RTE_LOG(ERR, PMD,
+			"VNIC rss hash key_len %d\n", rss->rss_key_len);
+	}
+
 	return rc;
 
 err_out:
diff --git a/drivers/net/bnxt/rte_pmd_bnxt.c b/drivers/net/bnxt/rte_pmd_bnxt.c
index c343d90..3148a7c 100644
--- a/drivers/net/bnxt/rte_pmd_bnxt.c
+++ b/drivers/net/bnxt/rte_pmd_bnxt.c
@@ -731,7 +731,7 @@ int rte_pmd_bnxt_mac_addr_add(uint8_t port, struct ether_addr *addr,
 		    (HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR |
 		     HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK) &&
 		    memcmp(addr, filter->l2_addr, ETHER_ADDR_LEN) == 0) {
-			bnxt_hwrm_clear_filter(bp, filter);
+			bnxt_hwrm_clear_l2_filter(bp, filter);
 			break;
 		}
 	}
@@ -749,7 +749,7 @@ int rte_pmd_bnxt_mac_addr_add(uint8_t port, struct ether_addr *addr,
 	/* Do not add a filter for the default MAC */
 	if (bnxt_hwrm_func_qcfg_vf_default_mac(bp, vf_id, &dflt_mac) ||
 	    memcmp(filter->l2_addr, dflt_mac.addr_bytes, ETHER_ADDR_LEN))
-		rc = bnxt_hwrm_set_filter(bp, vnic.fw_vnic_id, filter);
+		rc = bnxt_hwrm_set_l2_filter(bp, vnic.fw_vnic_id, filter);
 
 exit:
 	return rc;
-- 
2.10.1 (Apple Git-78)

Re: [PATCH 8/8] net/bnxt: add support for flow filter ops

[Ferruh Yigit]

On 8/24/2017 5:29 PM, Ajit Khaparde wrote:
> This patch adds support for flow validate/create/destroy/flush ops.

Can you please update feature file [1] to document flow API support and
for supported filters.

Also I believe this worth mentioning in release notes [2].

[1]
doc/guides/nics/features/bnxt.ini

[2]
doc/guides/rel_notes/release_17_11.rst

> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

<...>

> --- a/drivers/net/bnxt/bnxt_ethdev.c
> +++ b/drivers/net/bnxt/bnxt_ethdev.c
> @@ -610,7 +610,7 @@ static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
>  				if (filter->mac_index == index) {
>  					STAILQ_REMOVE(&vnic->filter, filter,
>  						      bnxt_filter_info, next);
> -					bnxt_hwrm_clear_filter(bp, filter);
> +					bnxt_hwrm_clear_l2_filter(bp, filter);

Is it possible to extract these function name changes into different patch?

<...>

> +		default:
> +			RTE_LOG(ERR, PMD, "Unknown Flow type");
> +			use_ntuple |= 1;
> +			//return -1;

Please avoid dead code. "break" ?

> +		}
> +		item++;
> +	}
> +	return use_ntuple;
> +}
> +

<...>

> +		switch (item->type) {
> +		case RTE_FLOW_ITEM_TYPE_ETH:
> +			//filter_TYPE = EM

can be removed.

<...>

> +			} /*
> +			   * else {
> +			   *  RTE_LOG(ERR, PMD, "Handle this condition\n");
> +			   * }
> +			   */

Please remove unsued code.

<...>

> +	rc = bnxt_flow_parse_attr(attr, error);
> +	if (rc != 0)
> +		goto ret;
> +	//Since we support ingress attribute only - right now.

Please prefer /* */ comments.

<...>

> +			filter->flags |=
> +				HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
> +		//HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_DROP;
> +		//HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_DROP;
> +		//HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP
> +		//HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP
> +		//HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_DROP

These can go.

<...>

> +free_flow:
> +	RTE_LOG(ERR, PMD, "Failed to create flow.\n");
> +	rte_flow_error_set(error, -ret,
> +			   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
> +			   "Failed to create flow.");
> +	rte_free(flow);
> +	flow = NULL;
> +	return flow;

return NULL; ?

> +static int
> +bnxt_flow_destroy(struct rte_eth_dev *dev,
> +		  struct rte_flow *flow,
> +		  struct rte_flow_error *error)
> +{
> +	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
> +	struct bnxt_filter_info *filter = flow->filter;
> +	int ret = 0;
> +
> +	if (filter->filter_type == HWRM_CFA_EM_FILTER)
> +		ret = bnxt_hwrm_clear_em_filter(bp, filter);
> +	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
> +		ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
> +
> +	if (!ret) {
> +		TAILQ_REMOVE(&bp->flow_list, flow, node);
> +		rte_free(flow);
> +	} else {
> +		rte_flow_error_set(error, -ret,
> +				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
> +				   "Failed to destroy flow.");
> +	}
> +
> +	return ret;
> +}
> +
> +static int
> +bnxt_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
> +{
> +	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
> +	struct rte_flow *flow;
> +	int ret = 0;
> +	void *temp;
> +
> +	TAILQ_FOREACH_SAFE(flow, &bp->flow_list, node, temp) {
> +		struct bnxt_filter_info *filter = flow->filter;
> +
> +		if (filter->filter_type == HWRM_CFA_EM_FILTER)
> +			ret = bnxt_hwrm_clear_em_filter(bp, filter);
> +		if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
> +			ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
> +
> +		if (ret) {
> +			rte_flow_error_set(error, -ret,
> +					   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
> +					   "Failed to flush flow in HW.");
> +			return -rte_errno;
> +		}
> +
> +		TAILQ_REMOVE(&bp->flow_list, flow, node);
> +		rte_free(flow);

This part looks like duplication of bnxt_flow_destroy()

> +	}
> +
> +	return ret;
> +}

<...>

> diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c
> index 0793820..da53e99 100644
> --- a/drivers/net/bnxt/bnxt_rxq.c
> +++ b/drivers/net/bnxt/bnxt_rxq.c
> @@ -98,7 +98,7 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
>  	}
>  
>  	/* Multi-queue mode */
> -	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
> +	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB_RSS) {

Is this change related?

>  		/* VMDq ONLY, VMDq+RSS, VMDq+DCB, VMDq+DCB+RSS */
>  		enum rte_eth_nb_pools pools;
>  
> @@ -113,6 +113,9 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
>  				pools = conf->nb_queue_pools;
>  				break;
>  			}
> +		case ETH_MQ_RX_RSS:
> +			pools = 1;	//bp->rx_cp_nr_rings;
> +			break;
>  		default:
>  			RTE_LOG(ERR, PMD, "Unsupported mq_mod %d\n",
>  				dev_conf->rxmode.mq_mode);
> @@ -203,12 +206,42 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
>  	}
>  	STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
>  
> -	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
> -		vnic->hash_type =
> -			HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4 |
> -			HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
> -
>  out:
> +	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
> +		struct rte_eth_rss_conf *rss = &dev_conf->rx_adv_conf.rss_conf;
> +		uint16_t hash_type = 0;
> +
> +		if (rss->rss_hf & ETH_RSS_IPV4)
> +			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4;
> +		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
> +			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4;
> +		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
> +			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4;
> +		if (rss->rss_hf & ETH_RSS_IPV6)
> +			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
> +		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
> +			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6;
> +		if (rss->rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
> +			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
> +
> +		for (i = 0; i < bp->nr_vnics; i++) {
> +			STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
> +			vnic->hash_type |= hash_type;
> +
> +			/*
> +			 * Use the supplied key if the key length is
> +			 * acceptable and the rss_key is not NULL
> +			 */
> +			if (rss->rss_key &&
> +			    rss->rss_key_len <= HW_HASH_KEY_SIZE)
> +				memcpy(vnic->rss_hash_key,
> +				       rss->rss_key, rss->rss_key_len);
> +			}
> +		}
> +		RTE_LOG(ERR, PMD,
> +			"VNIC rss hash key_len %d\n", rss->rss_key_len);
> +	}
> +

Same for above, these looks like logically unrelated, can be separated
into another patch?

<...>