[dpdk-dev] [PATCH v3 05/13] net/bnxt: add flow templates support for Thor

[Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>]

From: Kishore Padmanabha <kishore.padmanabha@broadcom.com>

Template adds non-VFR based support for testpmd with:
matches to include
- DMAC, SIP, DIP, Proto, Sport, Dport
- SIP, DIP, Proto, Sport, Dport
actions:
- count, drop

Signed-off-by: Kishore Padmanabha <kishore.padmanabha@broadcom.com>
Signed-off-by: Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>
Reviewed-by: Mike Baucom <michael.baucom@broadcom.com>
---
 drivers/net/bnxt/tf_ulp/bnxt_tf_common.h      |   6 +
 drivers/net/bnxt/tf_ulp/bnxt_ulp.c            |  36 +++---
 drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c       |  12 ++
 .../bnxt/tf_ulp/generic_templates/meson.build |  17 ++-
 .../ulp_template_db_thor_class.c              |   1 -
 drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c          |   2 +-
 drivers/net/bnxt/tf_ulp/ulp_flow_db.c         | 122 +++++++++++++++++-
 drivers/net/bnxt/tf_ulp/ulp_flow_db.h         |  26 +++-
 drivers/net/bnxt/tf_ulp/ulp_gen_tbl.c         |   5 +
 drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c          |   2 +-
 drivers/net/bnxt/tf_ulp/ulp_mapper.c          | 111 +++++++++++++++-
 drivers/net/bnxt/tf_ulp/ulp_matcher.c         |  13 ++
 drivers/net/bnxt/tf_ulp/ulp_port_db.c         |  15 ++-
 drivers/net/bnxt/tf_ulp/ulp_rte_parser.c      |   9 +-
 drivers/net/bnxt/tf_ulp/ulp_tun.c             |  20 +++
 drivers/net/bnxt/tf_ulp/ulp_utils.c           |   8 +-
 16 files changed, 356 insertions(+), 49 deletions(-)

diff --git a/drivers/net/bnxt/tf_ulp/bnxt_tf_common.h b/drivers/net/bnxt/tf_ulp/bnxt_tf_common.h
index f59da41e54..e0ebed3fed 100644
--- a/drivers/net/bnxt/tf_ulp/bnxt_tf_common.h
+++ b/drivers/net/bnxt/tf_ulp/bnxt_tf_common.h
@@ -13,6 +13,12 @@
 
 #define BNXT_TF_DBG(lvl, fmt, args...)	PMD_DRV_LOG(lvl, fmt, ## args)
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#define BNXT_TF_INF(fmt, args...)	PMD_DRV_LOG(INFO, fmt, ## args)
+#else
+#define BNXT_TF_INF(fmt, args...)
+#endif
+
 #define BNXT_ULP_EM_FLOWS			8192
 #define BNXT_ULP_1M_FLOWS			1000000
 #define BNXT_EEM_RX_GLOBAL_ID_MASK		(BNXT_ULP_1M_FLOWS - 1)
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
index 183bae66c5..475c7a6cdf 100644
--- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
@@ -698,6 +698,11 @@ ulp_eem_tbl_scope_init(struct bnxt *bp)
 			    rc);
 		return rc;
 	}
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+	BNXT_TF_DBG(DEBUG, "TableScope=0x%0x %d\n",
+		    params.tbl_scope_id,
+		    params.tbl_scope_id);
+#endif
 	rc = bnxt_ulp_cntxt_tbl_scope_id_set(bp->ulp_ctx, params.tbl_scope_id);
 	if (rc) {
 		BNXT_TF_DBG(ERR, "Unable to set table scope id\n");
@@ -825,6 +830,8 @@ ulp_ctx_init(struct bnxt *bp,
 		goto error_deinit;
 	}
 
+	/* TODO: For now we are overriding to APP:1 on this branch*/
+	bp->app_id = 1;
 	rc = bnxt_ulp_cntxt_app_id_set(bp->ulp_ctx, bp->app_id);
 	if (rc) {
 		BNXT_TF_DBG(ERR, "Unable to set app_id for ULP init.\n");
@@ -838,11 +845,6 @@ ulp_ctx_init(struct bnxt *bp,
 		goto error_deinit;
 	}
 
-	if (devid == BNXT_ULP_DEVICE_ID_THOR) {
-		ulp_data->ulp_flags &= ~BNXT_ULP_VF_REP_ENABLED;
-		BNXT_TF_DBG(ERR, "Enabled non-VFR mode\n");
-	}
-
 	/*
 	 * Shared session must be created before first regular session but after
 	 * the ulp_ctx is valid.
@@ -902,7 +904,7 @@ ulp_dparms_init(struct bnxt *bp, struct bnxt_ulp_context *ulp_ctx)
 	dparms->ext_flow_db_num_entries = bp->max_num_kflows * 1024;
 	/* GFID =  2 * num_flows */
 	dparms->mark_db_gfid_entries = dparms->ext_flow_db_num_entries * 2;
-	BNXT_TF_DBG(DEBUG, "Set the number of flows = %"PRIu64"\n",
+	BNXT_TF_DBG(DEBUG, "Set the number of flows = %" PRIu64 "\n",
 		    dparms->ext_flow_db_num_entries);
 
 	return 0;
@@ -1393,17 +1395,13 @@ bnxt_ulp_port_init(struct bnxt *bp)
 	uint32_t ulp_flags;
 	int32_t rc = 0;
 
+	if (!bp || !BNXT_TRUFLOW_EN(bp))
+		return rc;
+
 	if (!BNXT_PF(bp) && !BNXT_VF_IS_TRUSTED(bp)) {
 		BNXT_TF_DBG(ERR,
 			    "Skip ulp init for port: %d, not a TVF or PF\n",
-			    bp->eth_dev->data->port_id);
-		return rc;
-	}
-
-	if (!BNXT_TRUFLOW_EN(bp)) {
-		BNXT_TF_DBG(DEBUG,
-			    "Skip ulp init for port: %d, truflow is not enabled\n",
-			    bp->eth_dev->data->port_id);
+			bp->eth_dev->data->port_id);
 		return rc;
 	}
 
@@ -1524,6 +1522,9 @@ bnxt_ulp_port_deinit(struct bnxt *bp)
 	struct rte_pci_device *pci_dev;
 	struct rte_pci_addr *pci_addr;
 
+	if (!BNXT_TRUFLOW_EN(bp))
+		return;
+
 	if (!BNXT_PF(bp) && !BNXT_VF_IS_TRUSTED(bp)) {
 		BNXT_TF_DBG(ERR,
 			    "Skip ULP deinit port:%d, not a TVF or PF\n",
@@ -1531,13 +1532,6 @@ bnxt_ulp_port_deinit(struct bnxt *bp)
 		return;
 	}
 
-	if (!BNXT_TRUFLOW_EN(bp)) {
-		BNXT_TF_DBG(DEBUG,
-			    "Skip ULP deinit for port:%d, truflow is not enabled\n",
-			    bp->eth_dev->data->port_id);
-		return;
-	}
-
 	if (!bp->ulp_ctx) {
 		BNXT_TF_DBG(DEBUG, "ulp ctx already de-allocated\n");
 		return;
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c
index 19e9dba356..238b1d9657 100644
--- a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c
@@ -13,6 +13,9 @@
 #include "ulp_port_db.h"
 #include "ulp_ha_mgr.h"
 #include <rte_malloc.h>
+#ifdef	RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#include "ulp_template_debug_proto.h"
+#endif
 
 static int32_t
 bnxt_ulp_flow_validate_args(const struct rte_flow_attr *attr,
@@ -222,6 +225,15 @@ bnxt_ulp_flow_create(struct rte_eth_dev *dev,
 	else if (ret == BNXT_TF_RC_FID)
 		goto return_fid;
 
+#ifdef	RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef	RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_PARSER
+	/* Dump the rte flow pattern */
+	ulp_parser_hdr_info_dump(&params);
+	/* Dump the rte flow action */
+	ulp_parser_act_info_dump(&params);
+#endif
+#endif
+
 	ret = ulp_matcher_pattern_match(&params, &params.class_id);
 	if (ret != BNXT_TF_RC_SUCCESS)
 		goto free_fid;
diff --git a/drivers/net/bnxt/tf_ulp/generic_templates/meson.build b/drivers/net/bnxt/tf_ulp/generic_templates/meson.build
index b1e7b8cc32..16b27a2e24 100644
--- a/drivers/net/bnxt/tf_ulp/generic_templates/meson.build
+++ b/drivers/net/bnxt/tf_ulp/generic_templates/meson.build
@@ -2,12 +2,15 @@
 # Copyright(c) 2018 Intel Corporation
 # Copyright(c) 2020 Broadcom
 
+#Include the folder for headers
 includes += include_directories('.')
+
+#Add the source files
 sources += files(
-        'ulp_template_db_class.c',
-        'ulp_template_db_act.c',
-        'ulp_template_db_tbl.c',
-        'ulp_template_db_wh_plus_act.c',
-        'ulp_template_db_wh_plus_class.c',
-        'ulp_template_db_thor_act.c',
-        'ulp_template_db_thor_class.c')
+	'ulp_template_db_class.c',
+	'ulp_template_db_act.c',
+	'ulp_template_db_tbl.c',
+	'ulp_template_db_wh_plus_act.c',
+	'ulp_template_db_wh_plus_class.c',
+	'ulp_template_db_thor_act.c',
+	'ulp_template_db_thor_class.c')
diff --git a/drivers/net/bnxt/tf_ulp/generic_templates/ulp_template_db_thor_class.c b/drivers/net/bnxt/tf_ulp/generic_templates/ulp_template_db_thor_class.c
index 83f6152700..e342f340d9 100644
--- a/drivers/net/bnxt/tf_ulp/generic_templates/ulp_template_db_thor_class.c
+++ b/drivers/net/bnxt/tf_ulp/generic_templates/ulp_template_db_thor_class.c
@@ -6104,4 +6104,3 @@ struct bnxt_ulp_mapper_ident_info ulp_thor_class_ident_list[] = {
 	.ident_bit_pos = 29
 	}
 };
-
diff --git a/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c b/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c
index 13f71ed83b..22c51976ac 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c
@@ -35,7 +35,7 @@ ulp_fc_mgr_shadow_mem_alloc(struct hw_fc_mem_info *parms, int size)
 	rte_mem_lock_page(parms->mem_va);
 
 	parms->mem_pa = (void *)(uintptr_t)rte_mem_virt2phy(parms->mem_va);
-	if (parms->mem_pa == (void *)(uintptr_t)RTE_BAD_IOVA) {
+	if (parms->mem_pa == (void *)RTE_BAD_IOVA) {
 		BNXT_TF_DBG(ERR, "Allocate failed mem_pa\n");
 		return -ENOMEM;
 	}
diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.c b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
index ab6013f0e3..747a360aa0 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
@@ -116,7 +116,7 @@ ulp_flow_db_resource_func_get(struct ulp_fdb_resource_info *res_info)
 
 	func = (((res_info->nxt_resource_idx & ULP_FLOW_DB_RES_FUNC_MASK) >>
 		 ULP_FLOW_DB_RES_FUNC_BITS) << ULP_FLOW_DB_RES_FUNC_UPPER);
-	/* The resource func is split into upper and lower */
+	/* The reource func is split into upper and lower */
 	if (func & ULP_FLOW_DB_RES_FUNC_NEED_LOWER)
 		return (func | res_info->resource_func_lower);
 	return func;
@@ -654,6 +654,9 @@ ulp_flow_db_fid_alloc(struct bnxt_ulp_context *ulp_ctxt,
 	if (flow_type == BNXT_ULP_FDB_TYPE_REGULAR)
 		ulp_flow_db_func_id_set(flow_db, *fid, func_id);
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+	BNXT_TF_DBG(ERR, "flow_id = %u:%u allocated\n", flow_type, *fid);
+#endif
 	/* return success */
 	return 0;
 }
@@ -714,7 +717,7 @@ ulp_flow_db_resource_add(struct bnxt_ulp_context *ulp_ctxt,
 
 	if (params->critical_resource && fid_resource->resource_em_handle) {
 		BNXT_TF_DBG(DEBUG, "Ignore multiple critical resources\n");
-		/* Ignore the multiple critical resources */
+		/* Ignore the multiple criticial resources */
 		params->critical_resource = BNXT_ULP_CRITICAL_RESOURCE_NO;
 	}
 
@@ -766,7 +769,7 @@ ulp_flow_db_resource_add(struct bnxt_ulp_context *ulp_ctxt,
  * flow_type [in] Specify it is regular or default flow
  * fid [in] The index to the flow entry
  * params [in/out] The contents to be copied into params.
- * Only the critical_resource needs to be set by the caller.
+ * Onlythe critical_resource needs to be set by the caller.
  *
  * Returns 0 on success and negative on failure.
  */
@@ -937,6 +940,9 @@ ulp_flow_db_fid_free(struct bnxt_ulp_context *ulp_ctxt,
 
 	ulp_clear_tun_inner_entry(tun_tbl, fid);
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+	BNXT_TF_DBG(ERR, "flow_id = %u:%u freed\n", flow_type, fid);
+#endif
 	/* all good, return success */
 	return 0;
 }
@@ -1921,3 +1927,113 @@ void ulp_flow_db_shared_session_set(struct ulp_flow_db_res_params *res,
 	if (res && (shared & BNXT_ULP_SHARED_SESSION_YES))
 		res->fdb_flags |= ULP_FDB_FLAG_SHARED_SESSION;
 }
+
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+/*
+ * Dump the entry details
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ *
+ * returns none
+ */
+static void ulp_flow_db_res_dump(struct ulp_fdb_resource_info *r,
+				 uint32_t *nxt_res)
+{
+	uint8_t res_func = ulp_flow_db_resource_func_get(r);
+
+	BNXT_TF_DBG(DEBUG, "Resource func = %x, nxt_resource_idx = %x\n",
+		    res_func, (ULP_FLOW_DB_RES_NXT_MASK & r->nxt_resource_idx));
+	if (res_func == BNXT_ULP_RESOURCE_FUNC_EM_TABLE)
+		BNXT_TF_DBG(DEBUG, "EM Handle = 0x%016" PRIX64 "\n",
+			    r->resource_em_handle);
+	else
+		BNXT_TF_DBG(DEBUG, "Handle = 0x%08x\n", r->resource_hndl);
+
+	*nxt_res = 0;
+	ULP_FLOW_DB_RES_NXT_SET(*nxt_res,
+				r->nxt_resource_idx);
+}
+
+/*
+ * Dump the flow entry details
+ *
+ * flow_db [in] Ptr to flow db
+ * fid [in] flow id
+ *
+ * returns none
+ */
+void
+ulp_flow_db_debug_fid_dump(struct bnxt_ulp_flow_db *flow_db, uint32_t fid)
+{
+	struct ulp_fdb_resource_info *r;
+	struct bnxt_ulp_flow_tbl *flow_tbl;
+	uint32_t nxt_res = 0;
+	uint32_t def_flag = 0, reg_flag = 0;
+
+	flow_tbl = &flow_db->flow_tbl;
+	if (ulp_flow_db_active_flows_bit_is_set(flow_db,
+						BNXT_ULP_FDB_TYPE_REGULAR, fid))
+		reg_flag = 1;
+	if (ulp_flow_db_active_flows_bit_is_set(flow_db,
+						BNXT_ULP_FDB_TYPE_DEFAULT, fid))
+		def_flag = 1;
+
+	if (reg_flag && def_flag)
+		BNXT_TF_DBG(DEBUG, "RID = %u\n", fid);
+	else if (reg_flag)
+		BNXT_TF_DBG(DEBUG, "Regular fid = %u and func id = %u\n",
+			    fid, flow_db->func_id_tbl[fid]);
+	else if (def_flag)
+		BNXT_TF_DBG(DEBUG, "Default fid = %u\n", fid);
+	else
+		return;
+	/* iterate the resource */
+	nxt_res = fid;
+	do {
+		r = &flow_tbl->flow_resources[nxt_res];
+		ulp_flow_db_res_dump(r, &nxt_res);
+	} while (nxt_res);
+}
+
+/*
+ * Dump the flow database entry details
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * flow_id [in] if zero then all fids are dumped.
+ *
+ * returns none
+ */
+int32_t	ulp_flow_db_debug_dump(struct bnxt_ulp_context *ulp_ctxt,
+			       uint32_t flow_id)
+{
+	struct bnxt_ulp_flow_db *flow_db;
+	struct bnxt_ulp_flow_tbl *flow_tbl;
+	uint32_t fid;
+
+	if (!ulp_ctxt || !ulp_ctxt->cfg_data) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+
+	flow_tbl = &flow_db->flow_tbl;
+	if (flow_id) {
+		ulp_flow_db_debug_fid_dump(flow_db, flow_id);
+		return 0;
+	}
+
+	BNXT_TF_DBG(DEBUG, "Dump flows = %u:%u\n",
+		    flow_tbl->num_flows,
+		    flow_tbl->num_resources);
+	BNXT_TF_DBG(DEBUG, "Head_index = %u, Tail_index = %u\n",
+		    flow_tbl->head_index, flow_tbl->tail_index);
+	for (fid = 1; fid < flow_tbl->num_flows; fid++)
+		ulp_flow_db_debug_fid_dump(flow_db, fid);
+	BNXT_TF_DBG(DEBUG, "Done.\n");
+	return 0;
+}
+#endif
diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.h b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
index 67afca8872..0ddfa6f66d 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
+++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
@@ -18,7 +18,7 @@
 
 /*
  * Structure for the flow database resource information
- * The below structure is based on the below partitions
+ * The below structure is based on the below paritions
  * nxt_resource_idx = dir[31],resource_func_upper[30:28],nxt_resource_idx[27:0]
  * If resource_func is EM_TBL then use resource_em_handle.
  * Else the other part of the union is used and
@@ -417,4 +417,28 @@ ulp_flow_db_parent_flow_count_reset(struct bnxt_ulp_context *ulp_ctxt);
 void ulp_flow_db_shared_session_set(struct ulp_flow_db_res_params *res,
 				    enum bnxt_ulp_shared_session shared);
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+/*
+ * Dump the flow entry details
+ *
+ * flow_db [in] Ptr to flow db
+ * fid [in] flow id
+ *
+ * returns none
+ */
+void
+ulp_flow_db_debug_fid_dump(struct bnxt_ulp_flow_db *flow_db, uint32_t fid);
+
+/*
+ * Dump the flow database entry details
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * flow_id [in] if zero then all fids are dumped.
+ *
+ * returns none
+ */
+int32_t	ulp_flow_db_debug_dump(struct bnxt_ulp_context *ulp_ctxt,
+			       uint32_t flow_id);
+#endif
+
 #endif /* _ULP_FLOW_DB_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/ulp_gen_tbl.c b/drivers/net/bnxt/tf_ulp/ulp_gen_tbl.c
index 3c1af0b007..c6b2b1675d 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_gen_tbl.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_gen_tbl.c
@@ -10,6 +10,11 @@
 #include "ulp_mapper.h"
 #include "ulp_flow_db.h"
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#include "ulp_template_debug_proto.h"
+#include "ulp_tf_debug.h"
+#endif
+
 /* Retrieve the generic table  initialization parameters for the tbl_idx */
 static struct bnxt_ulp_generic_tbl_params*
 ulp_mapper_gen_tbl_params_get(uint32_t tbl_idx)
diff --git a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c
index 5f5b5d639e..bc5627ec5b 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c
@@ -185,7 +185,7 @@ ulp_ha_mgr_timer_cb(void *arg __rte_unused)
 	rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
 	if (rc) {
 		/*
-		 * This shouldn't happen, if it does, reset the timer
+		 * This shouldn't happen, if it does, resetart the timer
 		 * and try again next time.
 		 */
 		BNXT_TF_DBG(ERR, "Failed(%d) to get state.\n",
diff --git a/drivers/net/bnxt/tf_ulp/ulp_mapper.c b/drivers/net/bnxt/tf_ulp/ulp_mapper.c
index f3a60cc880..7fc3767b33 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_mapper.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_mapper.c
@@ -22,6 +22,11 @@
 #include "ulp_ha_mgr.h"
 #include "bnxt_tf_pmd_shim.h"
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#include "ulp_template_debug_proto.h"
+#include "ulp_tf_debug.h"
+#endif
+
 static uint8_t mapper_fld_zeros[16] = { 0 };
 
 static uint8_t mapper_fld_ones[16] = {
@@ -156,6 +161,13 @@ ulp_mapper_resource_ident_allocate(struct bnxt_ulp_context *ulp_ctx,
 		tf_free_identifier(tfp, &fparms);
 		return rc;
 	}
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	BNXT_TF_DBG(DEBUG, "Allocated Glb Res Ident [%s][%d][%d] = 0x%04x\n",
+		    tf_dir_2_str(iparms.dir),
+		    glb_res->glb_regfile_index, iparms.ident_type, iparms.id);
+#endif
+#endif
 	return rc;
 }
 
@@ -216,6 +228,13 @@ ulp_mapper_resource_index_tbl_alloc(struct bnxt_ulp_context *ulp_ctx,
 		tf_free_tbl_entry(tfp, &free_parms);
 		return rc;
 	}
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	BNXT_TF_DBG(DEBUG, "Allocated Glb Res Index [%s][%d][%d] = 0x%04x\n",
+		    tf_dir_2_str(aparms.dir),
+		    glb_res->glb_regfile_index, aparms.type, aparms.idx);
+#endif
+#endif
 	return rc;
 }
 
@@ -784,6 +803,9 @@ ulp_mapper_ident_process(struct bnxt_ulp_mapper_parms *parms,
 			    tf_ident_2_str(iparms.ident_type));
 		return rc;
 	}
+	BNXT_TF_INF("Alloc ident %s:%s.success.\n",
+		    tf_dir_2_str(iparms.dir),
+		    tf_ident_2_str(iparms.ident_type));
 
 	id = (uint64_t)tfp_cpu_to_be_64(iparms.id);
 	if (ulp_regfile_write(parms->regfile, idx, id)) {
@@ -813,6 +835,11 @@ ulp_mapper_ident_process(struct bnxt_ulp_mapper_parms *parms,
 	} else {
 		*val = iparms.id;
 	}
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	ulp_mapper_ident_field_dump("Ident", ident, tbl, iparms.id);
+#endif
+#endif
 	return 0;
 
 error:
@@ -877,6 +904,10 @@ ulp_mapper_ident_extract(struct bnxt_ulp_mapper_parms *parms,
 			    sparms.search_id);
 		return rc;
 	}
+	BNXT_TF_INF("Search ident %s:%s:%x.success.\n",
+		    tf_dir_2_str(sparms.dir),
+		    tf_tbl_type_2_str(sparms.ident_type),
+		    sparms.search_id);
 
 	/* Write it to the regfile */
 	id = (uint64_t)tfp_cpu_to_be_64(sparms.search_id);
@@ -904,6 +935,11 @@ ulp_mapper_ident_extract(struct bnxt_ulp_mapper_parms *parms,
 		goto error;
 	}
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	ulp_mapper_ident_field_dump("Ident", ident, tbl, sparms.search_id);
+#endif
+#endif
 	return 0;
 
 error:
@@ -996,7 +1032,7 @@ ulp_mapper_field_src_process(struct bnxt_ulp_mapper_parms *parms,
 			return -EINVAL;
 		}
 		idx = tfp_be_to_cpu_16(idx);
-		if (idx >= BNXT_ULP_CF_IDX_LAST || bytelen > sizeof(uint64_t)) {
+		if (idx >= BNXT_ULP_CF_IDX_LAST || bytelen > sizeof(uint32_t)) {
 			BNXT_TF_DBG(ERR, "comp field [%d] read oob %d\n", idx,
 				    bytelen);
 			return -EINVAL;
@@ -1448,7 +1484,16 @@ ulp_mapper_field_opc_process(struct bnxt_ulp_mapper_parms *parms,
 		break;
 	}
 
-	return rc;
+	if (!rc) {
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+		if (fld->field_src1 != BNXT_ULP_FIELD_SRC_ZERO)
+			ulp_mapper_field_dump(name, fld, blob, write_idx, val,
+					      val_len);
+#endif
+#endif
+		return rc;
+	}
 error:
 	BNXT_TF_DBG(ERR, "Error in %s:%s process %u:%u\n", name,
 		    fld->description, (val) ? write_idx : 0, val_len);
@@ -1500,8 +1545,15 @@ ulp_mapper_tbl_result_build(struct bnxt_ulp_mapper_parms *parms,
 	}
 
 	/* if encap bit swap is enabled perform the bit swap */
-	if (parms->device_params->encap_byte_swap && encap_flds)
+	if (parms->device_params->encap_byte_swap && encap_flds) {
 		ulp_blob_perform_encap_swap(data);
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+		BNXT_TF_DBG(INFO, "Dump after encap swap\n");
+		ulp_mapper_blob_dump(data);
+#endif
+#endif
+	}
 
 	return rc;
 }
@@ -1725,6 +1777,9 @@ ulp_mapper_tcam_tbl_entry_write(struct bnxt_ulp_mapper_parms *parms,
 			    tf_dir_2_str(sparms.dir), sparms.idx);
 		return -EIO;
 	}
+	BNXT_TF_INF("tcam[%s][%s][%x] write success.\n",
+		    tf_tcam_tbl_2_str(sparms.tcam_tbl_type),
+		    tf_dir_2_str(sparms.dir), sparms.idx);
 
 	/* Mark action */
 	rc = ulp_mapper_mark_act_ptr_process(parms, tbl);
@@ -1733,6 +1788,11 @@ ulp_mapper_tcam_tbl_entry_write(struct bnxt_ulp_mapper_parms *parms,
 		return rc;
 	}
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	ulp_mapper_tcam_entry_dump("TCAM", idx, tbl, key, mask, data);
+#endif
+#endif
 	return rc;
 }
 
@@ -1838,6 +1898,12 @@ static void ulp_mapper_wc_tcam_tbl_post_process(struct ulp_blob *blob)
 {
 	ulp_blob_perform_64B_word_swap(blob);
 	ulp_blob_perform_64B_byte_swap(blob);
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	BNXT_TF_DBG(INFO, "Dump after wc tcam post process\n");
+	ulp_mapper_blob_dump(blob);
+#endif
+#endif
 }
 
 static int32_t
@@ -2134,6 +2200,11 @@ ulp_mapper_em_tbl_process(struct bnxt_ulp_mapper_parms *parms,
 		BNXT_TF_DBG(ERR, "Failed to build the result blob\n");
 		return rc;
 	}
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	ulp_mapper_result_dump("EM Result", tbl, &data);
+#endif
+#endif
 	if (dparms->dynamic_pad_en) {
 		uint32_t abits = dparms->em_blk_align_bits;
 
@@ -2148,6 +2219,11 @@ ulp_mapper_em_tbl_process(struct bnxt_ulp_mapper_parms *parms,
 		ulp_blob_pad_align(&data, abits);
 
 		ulp_blob_perform_byte_reverse(&data, ULP_BITS_2_BYTE(abits));
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	ulp_mapper_result_dump("EM Merged Result", tbl, &data);
+#endif
+#endif
 	}
 
 	/* do the transpose for the internal EM keys */
@@ -2160,6 +2236,11 @@ ulp_mapper_em_tbl_process(struct bnxt_ulp_mapper_parms *parms,
 		}
 		tmplen = ulp_blob_data_len_get(&key);
 		ulp_blob_perform_byte_reverse(&key, ULP_BITS_2_BYTE(tmplen));
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	ulp_mapper_result_dump("EM Key Transpose", tbl, &key);
+#endif
+#endif
 	}
 
 	rc = bnxt_ulp_cntxt_tbl_scope_id_get(parms->ulp_ctx,
@@ -2190,6 +2271,12 @@ ulp_mapper_em_tbl_process(struct bnxt_ulp_mapper_parms *parms,
 		return rc;
 	}
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	ulp_mapper_em_dump("EM", &key, &data, &iparms);
+	/* tf_dump_tables(tfp, iparms.tbl_scope_id); */
+#endif
+#endif
 	/* Mark action process */
 	if (mtype == BNXT_ULP_FLOW_MEM_TYPE_EXT &&
 	    tbl->resource_type == TF_MEM_EXTERNAL)
@@ -2479,6 +2566,9 @@ ulp_mapper_index_tbl_process(struct bnxt_ulp_mapper_parms *parms,
 				    sparms.idx, rc);
 			goto error;
 		}
+		BNXT_TF_INF("Index table[%s][%s][%x] write successful.\n",
+			    tf_tbl_type_2_str(sparms.type),
+			    tf_dir_2_str(sparms.dir), sparms.idx);
 
 		/* Calculate action record size */
 		if (tbl->resource_type == TF_TBL_TYPE_EXT) {
@@ -2635,6 +2725,10 @@ ulp_mapper_if_tbl_process(struct bnxt_ulp_mapper_parms *parms,
 			    iftbl_params.idx, rc);
 		return rc;
 	}
+	BNXT_TF_INF("Set table[%s][%s][%x] success.\n",
+		    tf_if_tbl_2_str(iftbl_params.type),
+		    tf_dir_2_str(iftbl_params.dir),
+		    iftbl_params.idx);
 
 	/*
 	 * TBD: Need to look at the need to store idx in flow db for restore
@@ -2697,6 +2791,12 @@ ulp_mapper_gen_tbl_process(struct bnxt_ulp_mapper_parms *parms,
 
 	/* The_key is a byte array convert it to a search index */
 	cache_key = ulp_blob_data_get(&key, &tmplen);
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+	BNXT_TF_DBG(DEBUG, "The gen_tbl[%u] key\n", tbl_idx);
+	ulp_mapper_blob_dump(&key);
+#endif
+#endif
 	/* get the generic table  */
 	gen_tbl_list = &parms->mapper_data->gen_tbl_list[tbl_idx];
 
@@ -3495,6 +3595,11 @@ ulp_mapper_tbls_process(struct bnxt_ulp_mapper_parms *parms, uint32_t tid)
 	for (tbl_idx = 0; tbl_idx < num_tbls && cond_goto;) {
 		tbl = &tbls[tbl_idx];
 		cond_goto = tbl->execute_info.cond_true_goto;
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_MAPPER
+		ulp_mapper_table_dump(tbl, tbl_idx);
+#endif
+#endif
 		/* Process the conditional func code opcodes */
 		if (ulp_mapper_func_info_process(parms, tbl)) {
 			BNXT_TF_DBG(ERR, "Failed to process cond update\n");
diff --git a/drivers/net/bnxt/tf_ulp/ulp_matcher.c b/drivers/net/bnxt/tf_ulp/ulp_matcher.c
index 67fa61fc7c..e06d8f6287 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_matcher.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_matcher.c
@@ -6,6 +6,10 @@
 #include "ulp_matcher.h"
 #include "ulp_utils.h"
 
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#include "ulp_template_debug_proto.h"
+#endif
+
 /* Utility function to calculate the class matcher hash */
 static uint32_t
 ulp_matcher_class_hash_calculate(uint64_t hi_sig, uint64_t lo_sig)
@@ -95,6 +99,11 @@ ulp_matcher_pattern_match(struct ulp_rte_parser_params *params,
 
 error:
 	BNXT_TF_DBG(DEBUG, "Did not find any matching template\n");
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+	BNXT_TF_DBG(DEBUG, "class_hid:0x%x, Hdr:%" PRIX64 " Fld:%" PRIX64 "\n",
+		    class_hid, params->hdr_bitmap.bits,
+		    params->fld_bitmap.bits);
+#endif
 	*class_id = 0;
 	return BNXT_TF_RC_ERROR;
 }
@@ -142,6 +151,10 @@ ulp_matcher_action_match(struct ulp_rte_parser_params *params,
 
 error:
 	BNXT_TF_DBG(DEBUG, "Did not find any matching action template\n");
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+	BNXT_TF_DBG(DEBUG, "act_hid:0x%x, Hdr:%" PRIX64 "\n",
+		    act_hid, params->act_bitmap.bits);
+#endif
 	*act_id = 0;
 	return BNXT_TF_RC_ERROR;
 }
diff --git a/drivers/net/bnxt/tf_ulp/ulp_port_db.c b/drivers/net/bnxt/tf_ulp/ulp_port_db.c
index 4045473097..7d9865b3e3 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_port_db.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_port_db.c
@@ -7,9 +7,13 @@
 #include "bnxt.h"
 #include "bnxt_vnic.h"
 #include "bnxt_tf_common.h"
+#include "bnxt_tf_pmd_shim.h"
 #include "ulp_port_db.h"
 #include "tfp.h"
-#include "bnxt_tf_pmd_shim.h"
+
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#include "ulp_tf_debug.h"
+#endif
 
 static uint32_t
 ulp_port_db_allocate_ifindex(struct bnxt_ulp_port_db *port_db)
@@ -151,12 +155,12 @@ int32_t	ulp_port_db_dev_port_intf_update(struct bnxt_ulp_context *ulp_ctxt,
 
 	intf->type = bnxt_pmd_get_interface_type(port_id);
 	intf->drv_func_id = bnxt_pmd_get_fw_func_id(port_id,
-						    BNXT_ULP_INTF_TYPE_INVALID);
+						BNXT_ULP_INTF_TYPE_INVALID);
 
 	func = &port_db->ulp_func_id_tbl[intf->drv_func_id];
 	if (!func->func_valid) {
 		func->func_svif = bnxt_pmd_get_svif(port_id, true,
-						    BNXT_ULP_INTF_TYPE_INVALID);
+						BNXT_ULP_INTF_TYPE_INVALID);
 		func->func_spif = bnxt_pmd_get_phy_port_id(port_id);
 		func->func_parif =
 			bnxt_pmd_get_parif(port_id, BNXT_ULP_INTF_TYPE_INVALID);
@@ -202,6 +206,11 @@ int32_t	ulp_port_db_dev_port_intf_update(struct bnxt_ulp_context *ulp_ctxt,
 		port_data->port_vport = bnxt_pmd_get_vport(port_id);
 		port_data->port_valid = true;
 	}
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_PORT
+	ulp_port_db_dump(port_db, intf, port_id);
+#endif
+#endif
 	return 0;
 }
 
diff --git a/drivers/net/bnxt/tf_ulp/ulp_rte_parser.c b/drivers/net/bnxt/tf_ulp/ulp_rte_parser.c
index 5fd52b8f36..79b9957781 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_rte_parser.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_rte_parser.c
@@ -8,6 +8,7 @@
 #include "ulp_template_struct.h"
 #include "bnxt_ulp.h"
 #include "bnxt_tf_common.h"
+#include "bnxt_tf_pmd_shim.h"
 #include "ulp_rte_parser.h"
 #include "ulp_matcher.h"
 #include "ulp_utils.h"
@@ -855,7 +856,7 @@ ulp_rte_vlan_hdr_handler(const struct rte_flow_item *item,
 			       BNXT_ULP_HDR_BIT_II_VLAN);
 		inner_flag = 1;
 	} else {
-		BNXT_TF_DBG(ERR, "Error Parsing:Vlan hdr found without eth\n");
+		BNXT_TF_DBG(ERR, "Error Parsing:Vlan hdr found withtout eth\n");
 		return BNXT_TF_RC_ERROR;
 	}
 	/* Update the field protocol hdr bitmap */
@@ -1135,8 +1136,8 @@ ulp_rte_ipv6_hdr_handler(const struct rte_flow_item *item,
 	ulp_rte_prsr_fld_mask(params, &idx, size, &ver_spec, &ver_mask,
 			      ULP_PRSR_ACT_DEFAULT);
 	/*
-	 * The TC and flow label field are ignored since OVS is setting
-	 * it for match and it is not supported.
+	 * The TC and flow label field are ignored since OVS is
+	 * setting it for match and it is not supported.
 	 * This is a work around and
 	 * shall be addressed in the future.
 	 */
@@ -2138,7 +2139,7 @@ ulp_rte_vf_act_handler(const struct rte_flow_action *action_item,
 		return BNXT_TF_RC_PARSE_ERR;
 	}
 
-	bp = bnxt_get_bp(params->port_id);
+	bp = bnxt_pmd_get_bp(params->port_id);
 	if (bp == NULL) {
 		BNXT_TF_DBG(ERR, "Invalid bp\n");
 		return BNXT_TF_RC_ERROR;
diff --git a/drivers/net/bnxt/tf_ulp/ulp_tun.c b/drivers/net/bnxt/tf_ulp/ulp_tun.c
index cb8530d791..a1dd5b902c 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_tun.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_tun.c
@@ -3,6 +3,8 @@
  * All rights reserved.
  */
 
+#include <sys/queue.h>
+
 #include <rte_malloc.h>
 
 #include "ulp_tun.h"
@@ -29,6 +31,15 @@ ulp_install_outer_tun_flow(struct ulp_rte_parser_params *params,
 
 	ULP_BITMAP_SET(params->hdr_bitmap.bits, BNXT_ULP_HDR_BIT_F1);
 
+#ifdef	RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef	RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_PARSER
+	/* Dump the rte flow pattern */
+	ulp_parser_hdr_info_dump(params);
+	/* Dump the rte flow action */
+	ulp_parser_act_info_dump(params);
+#endif
+#endif
+
 	ret = ulp_matcher_pattern_match(params, &params->class_id);
 	if (ret != BNXT_TF_RC_SUCCESS)
 		goto err;
@@ -146,6 +157,15 @@ ulp_post_process_cache_inner_tun_flow(struct ulp_rte_parser_params *params,
 	struct ulp_per_port_flow_info *flow_info;
 	int ret;
 
+#ifdef	RTE_LIBRTE_BNXT_TRUFLOW_DEBUG
+#ifdef	RTE_LIBRTE_BNXT_TRUFLOW_DEBUG_PARSER
+	/* Dump the rte flow pattern */
+	ulp_parser_hdr_info_dump(params);
+	/* Dump the rte flow action */
+	ulp_parser_act_info_dump(params);
+#endif
+#endif
+
 	ret = ulp_matcher_pattern_match(params, &params->class_id);
 	if (ret != BNXT_TF_RC_SUCCESS)
 		return BNXT_TF_RC_ERROR;
diff --git a/drivers/net/bnxt/tf_ulp/ulp_utils.c b/drivers/net/bnxt/tf_ulp/ulp_utils.c
index bafb539c8d..1649e157f2 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_utils.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_utils.c
@@ -62,7 +62,7 @@ ulp_regfile_read(struct ulp_regfile *regfile,
  * data [in] The value is written into this variable.  It is going to be in the
  * same byte order as it was written.
  *
- * size [in] The size in bytes of the value being written into this
+ * size [in] The size in bytes of the value beingritten into this
  * variable.
  *
  * returns 0 on success
@@ -295,7 +295,7 @@ ulp_blob_push(struct ulp_blob *blob,
 				     datalen,
 				     data);
 	if (!rc) {
-		BNXT_TF_DBG(ERR, "Failed to write blob\n");
+		BNXT_TF_DBG(ERR, "Failed ro write blob\n");
 		return 0;
 	}
 	blob->write_idx += datalen;
@@ -355,7 +355,7 @@ ulp_blob_insert(struct ulp_blob *blob, uint32_t offset,
 				     datalen,
 				     data);
 	if (!rc) {
-		BNXT_TF_DBG(ERR, "Failed to write blob\n");
+		BNXT_TF_DBG(ERR, "Failed ro write blob\n");
 		return 0;
 	}
 	/* copy the previously stored data */
@@ -409,7 +409,7 @@ ulp_blob_push_64(struct ulp_blob *blob,
  *
  * data [in] 32-bit value to be added to the blob.
  *
- * datalen [in] The number of bits to be added to the blob.
+ * datalen [in] The number of bits to be added ot the blob.
  *
  * The offset of the data is updated after each push of data.
  * NULL returned on error, pointer pushed value otherwise.
-- 
2.17.1