[dpdk-dev] [PATCH v2 0/6] fix transmit datapath cumulative series

[dpdk-dev] [PATCH v2 0/6] fix transmit datapath cumulative series

[Viacheslav Ovsiienko]

This series provides the fixes for mlx5 Tx datapath, mostly
resolving the minor performance and setup issues:

  - various inline data settings
  - wrong assert condition on minimal data length
  - large burst SQ overflow due to inefficient CQ drain

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>

---
v1 was sent as separated patches:
  - http://patches.dpdk.org/patch/57347/
  - http://patches.dpdk.org/patch/57348/
  - http://patches.dpdk.org/patch/57410/
  - http://patches.dpdk.org/patch/57411/
  - http://patches.dpdk.org/patch/57412/ (reverted in v2)
  - http://patches.dpdk.org/patch/57413/
  - http://patches.dpdk.org/patch/57414/


Viacheslav Ovsiienko (6):
  net/mlx5: fix default minimal data inline
  net/mlx5: fix inline data len assert condition
  net/mlx5: fix completion queue drain loop
  net/mlx5: fix inline data settings
  net/mlx5: fix packet size inline settings
  net/mlx5: fix completion queue overflow for large bursts

 doc/guides/nics/mlx5.rst     |   2 +-
 drivers/net/mlx5/mlx5.c      |   5 +-
 drivers/net/mlx5/mlx5_prm.h  |   5 +-
 drivers/net/mlx5/mlx5_rxtx.c | 136 ++++++++++++++++++++++++++++---------------
 drivers/net/mlx5/mlx5_rxtx.h |   9 +--
 drivers/net/mlx5/mlx5_txq.c  |  39 ++++++-------
 6 files changed, 115 insertions(+), 81 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v2 1/6] net/mlx5: fix default minimal data inline

[Viacheslav Ovsiienko]

The patch [Fixes] sets the default value of required minimal
inline data to 0 bytes. On some configurations (depends
on switchdev/legacy settings and FW version/settings)
the ConnectX-4LX NIC requires minimal 18 bytes of
Tx descriptor inline data to operate correctly.

Wrongly set to 0 default value may prevent NIC from operating
with out-of-the-box settings, this patch reverts default
value for ConnectX-4LX back to 18 bytes (inline L2).

Fixes: 9f350504bb32 ("net/mlx5: fix ConnectX-4LX minimal inline data limit")

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 drivers/net/mlx5/mlx5.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 909c22e..a3eacdb 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1325,12 +1325,9 @@ struct mlx5_dev_spawn_data {
 	switch (spawn->pci_dev->id.device_id) {
 	case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
 	case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
-		config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
-		config->hw_vlan_insert = 0;
-		break;
 	case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX:
 	case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
-		config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
+		config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
 		config->hw_vlan_insert = 0;
 		break;
 	case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v2 1/6] net/mlx5: fix default minimal data inline

[Raslan Darawsheh]

Hi,

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Viacheslav Ovsiienko
> Sent: Monday, August 5, 2019 4:04 PM
> To: dev@dpdk.org
> Cc: Yongseok Koh <yskoh@mellanox.com>; Matan Azrad
> <matan@mellanox.com>
> Subject: [dpdk-dev] [PATCH v2 1/6] net/mlx5: fix default minimal data inline
> 
> The patch [Fixes] sets the default value of required minimal inline data to 0
> bytes. On some configurations (depends on switchdev/legacy settings and
> FW version/settings) the ConnectX-4LX NIC requires minimal 18 bytes of Tx
> descriptor inline data to operate correctly.
> 
> Wrongly set to 0 default value may prevent NIC from operating with out-of-
> the-box settings, this patch reverts default value for ConnectX-4LX back to 18
> bytes (inline L2).
> 
> Fixes: 9f350504bb32 ("net/mlx5: fix ConnectX-4LX minimal inline data limit")
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> Acked-by: Matan Azrad <matan@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5.c | 5 +----
>  1 file changed, 1 insertion(+), 4 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 909c22e..a3eacdb 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -1325,12 +1325,9 @@ struct mlx5_dev_spawn_data {
>  	switch (spawn->pci_dev->id.device_id) {
>  	case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
>  	case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
> -		config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
> -		config->hw_vlan_insert = 0;
> -		break;
>  	case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX:
>  	case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
> -		config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
> +		config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
>  		config->hw_vlan_insert = 0;
>  		break;
>  	case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
> --
> 1.8.3.1

Series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh

[dpdk-dev] [PATCH v2 2/6] net/mlx5: fix inline data len assert condition

[Viacheslav Ovsiienko]

The debug assert wrongly triggers on the inline data 18B,
this should be passed successfully.

Fixes: 18a1c20044c0 ("net/mlx5: implement Tx burst template")

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index a4fc388..1ec3793 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -2594,7 +2594,7 @@ enum mlx5_txcmp_code {
 				 sizeof(struct rte_vlan_hdr) +
 				 2 * RTE_ETHER_ADDR_LEN),
 		      "invalid Ethernet Segment data size");
-	assert(inlen > MLX5_ESEG_MIN_INLINE_SIZE);
+	assert(inlen >= MLX5_ESEG_MIN_INLINE_SIZE);
 	es->inline_hdr_sz = rte_cpu_to_be_16(inlen);
 	pdst = (uint8_t *)&es->inline_data;
 	if (MLX5_TXOFF_CONFIG(VLAN) && vlan) {
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 3/6] net/mlx5: fix completion queue drain loop

[Viacheslav Ovsiienko]

The completion loop speed optimizations for error-free
operations are done - no CQE field fetch on each loop
iteration. Also, code size is oprimized - the flush
buffers routine is invoked once.

Fixes: 318ea4cfa1b1 ("net/mlx5: fix Tx completion descriptors fetching loop")

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 98 +++++++++++++++++++++++++++++---------------
 drivers/net/mlx5/mlx5_rxtx.h |  8 ++--
 2 files changed, 68 insertions(+), 38 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 1ec3793..a890f41 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -654,9 +654,10 @@ enum mlx5_txcmp_code {
  *   Pointer to the error CQE.
  *
  * @return
- *   The last Tx buffer element to free.
+ *   Negative value if queue recovery failed,
+ *   the last Tx buffer element to free otherwise.
  */
-uint16_t
+int
 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *restrict txq,
 			 volatile struct mlx5_err_cqe *err_cqe)
 {
@@ -706,6 +707,7 @@ enum mlx5_txcmp_code {
 			return txq->elts_head;
 		}
 		/* Recovering failed - try again later on the same WQE. */
+		return -1;
 	} else {
 		txq->cq_ci++;
 	}
@@ -2010,6 +2012,45 @@ enum mlx5_txcmp_code {
 }
 
 /**
+ * Update completion queue consuming index via doorbell
+ * and flush the completed data buffers.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param valid CQE pointer
+ *   if not NULL update txq->wqe_pi and flush the buffers
+ * @param itail
+ *   if not negative - flush the buffers till this index.
+ * @param olx
+ *   Configured Tx offloads mask. It is fully defined at
+ *   compile time and may be used for optimization.
+ */
+static __rte_always_inline void
+mlx5_tx_comp_flush(struct mlx5_txq_data *restrict txq,
+		   volatile struct mlx5_cqe *last_cqe,
+		   int itail,
+		   unsigned int olx __rte_unused)
+{
+	uint16_t tail;
+
+	if (likely(last_cqe != NULL)) {
+		txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter);
+		tail = ((volatile struct mlx5_wqe_cseg *)
+			(txq->wqes + (txq->wqe_pi & txq->wqe_m)))->misc;
+	} else if (itail >= 0) {
+		tail = (uint16_t)itail;
+	} else {
+		return;
+	}
+	rte_compiler_barrier();
+	*txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
+	if (likely(tail != txq->elts_tail)) {
+		mlx5_tx_free_elts(txq, tail, olx);
+		assert(tail == txq->elts_tail);
+	}
+}
+
+/**
  * Manage TX completions. This routine checks the CQ for
  * arrived CQEs, deduces the last accomplished WQE in SQ,
  * updates SQ producing index and frees all completed mbufs.
@@ -2028,10 +2069,11 @@ enum mlx5_txcmp_code {
 			  unsigned int olx __rte_unused)
 {
 	unsigned int count = MLX5_TX_COMP_MAX_CQE;
-	bool update = false;
-	uint16_t tail = txq->elts_tail;
+	volatile struct mlx5_cqe *last_cqe = NULL;
 	int ret;
 
+	static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value");
+	static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value");
 	do {
 		volatile struct mlx5_cqe *cqe;
 
@@ -2043,32 +2085,30 @@ enum mlx5_txcmp_code {
 				assert(ret == MLX5_CQE_STATUS_HW_OWN);
 				break;
 			}
-			/* Some error occurred, try to restart. */
+			/*
+			 * Some error occurred, try to restart.
+			 * We have no barrier after WQE related Doorbell
+			 * written, make sure all writes are completed
+			 * here, before we might perform SQ reset.
+			 */
 			rte_wmb();
-			tail = mlx5_tx_error_cqe_handle
+			ret = mlx5_tx_error_cqe_handle
 				(txq, (volatile struct mlx5_err_cqe *)cqe);
-			if (likely(tail != txq->elts_tail)) {
-				mlx5_tx_free_elts(txq, tail, olx);
-				assert(tail == txq->elts_tail);
-			}
-			/* Allow flushing all CQEs from the queue. */
-			count = txq->cqe_s;
-		} else {
-			volatile struct mlx5_wqe_cseg *cseg;
-
-			/* Normal transmit completion. */
-			++txq->cq_ci;
-			rte_cio_rmb();
-			txq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter);
-			cseg = (volatile struct mlx5_wqe_cseg *)
-				(txq->wqes + (txq->wqe_pi & txq->wqe_m));
-			tail = cseg->misc;
+			/*
+			 * Flush buffers, update consuming index
+			 * if recovery succeeded. Otherwise
+			 * just try to recover later.
+			 */
+			last_cqe = NULL;
+			break;
 		}
+		/* Normal transmit completion. */
+		++txq->cq_ci;
+		last_cqe = cqe;
 #ifndef NDEBUG
 		if (txq->cq_pi)
 			--txq->cq_pi;
 #endif
-		update = true;
 	/*
 	 * We have to restrict the amount of processed CQEs
 	 * in one tx_burst routine call. The CQ may be large
@@ -2078,17 +2118,7 @@ enum mlx5_txcmp_code {
 	 * latency.
 	 */
 	} while (--count);
-	if (likely(tail != txq->elts_tail)) {
-		/* Free data buffers from elts. */
-		mlx5_tx_free_elts(txq, tail, olx);
-		assert(tail == txq->elts_tail);
-	}
-	if (likely(update)) {
-		/* Update the consumer index. */
-		rte_compiler_barrier();
-		*txq->cq_db =
-		rte_cpu_to_be_32(txq->cq_ci);
-	}
+	mlx5_tx_comp_flush(txq, last_cqe, ret, olx);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index c209d99..aaa02a2 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -400,7 +400,7 @@ struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
 void mlx5_set_ptype_table(void);
 void mlx5_set_cksum_table(void);
 void mlx5_set_swp_types_table(void);
-__rte_noinline uint16_t mlx5_tx_error_cqe_handle
+__rte_noinline int mlx5_tx_error_cqe_handle
 				(struct mlx5_txq_data *restrict txq,
 				 volatile struct mlx5_err_cqe *err_cqe);
 uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n);
@@ -499,9 +499,9 @@ int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova,
 
 /* CQE status. */
 enum mlx5_cqe_status {
-	MLX5_CQE_STATUS_SW_OWN,
-	MLX5_CQE_STATUS_HW_OWN,
-	MLX5_CQE_STATUS_ERR,
+	MLX5_CQE_STATUS_SW_OWN = -1,
+	MLX5_CQE_STATUS_HW_OWN = -2,
+	MLX5_CQE_STATUS_ERR = -3,
 };
 
 /**
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 4/6] net/mlx5: fix inline data settings

[Viacheslav Ovsiienko]

If the minimal inline data are required the data inline feature
must be engaged. There were the incorrect settings enabling the
entire small packet inline (in size up to 82B) which may result
in sending rate declining if there is no enough cores. The same
problem was raised if inline was enabled to support VLAN tag
insertion by software.

Fixes: 38b4b397a57d ("net/mlx5: add Tx configuration and setup")

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 drivers/net/mlx5/mlx5_txq.c | 39 ++++++++++++++++++---------------------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index fe3b4ec..81f3b40 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -784,13 +784,11 @@ struct mlx5_txq_ibv *
 	txq_ctrl->txq.vlan_en = config->hw_vlan_insert;
 	vlan_inline = (dev_txoff & DEV_TX_OFFLOAD_VLAN_INSERT) &&
 		      !config->hw_vlan_insert;
-	if (vlan_inline)
-		inlen_send = RTE_MAX(inlen_send, MLX5_ESEG_MIN_INLINE_SIZE);
 	/*
 	 * If there are few Tx queues it is prioritized
 	 * to save CPU cycles and disable data inlining at all.
 	 */
-	if ((inlen_send && priv->txqs_n >= txqs_inline) || vlan_inline) {
+	if (inlen_send && priv->txqs_n >= txqs_inline) {
 		/*
 		 * The data sent with ordinal MLX5_OPCODE_SEND
 		 * may be inlined in Ethernet Segment, align the
@@ -825,32 +823,31 @@ struct mlx5_txq_ibv *
 				     MLX5_WQE_CSEG_SIZE -
 				     MLX5_WQE_ESEG_SIZE -
 				     MLX5_WQE_DSEG_SIZE * 2);
-		txq_ctrl->txq.inlen_send = inlen_send;
-		txq_ctrl->txq.inlen_mode = inlen_mode;
-		txq_ctrl->txq.inlen_empw = 0;
-	} else {
+	} else if (inlen_mode) {
 		/*
 		 * If minimal inlining is requested we must
 		 * enable inlining in general, despite the
-		 * number of configured queues.
+		 * number of configured queues. Ignore the
+		 * txq_inline_max devarg, this is not
+		 * full-featured inline.
 		 */
 		inlen_send = inlen_mode;
-		if (inlen_mode) {
-			/*
-			 * Extend space for inline data to allow
-			 * optional alignment of data buffer
-			 * start address, it may improve PCIe
-			 * performance.
-			 */
-			inlen_send = RTE_MIN(inlen_send + MLX5_WQE_SIZE,
-					     MLX5_SEND_MAX_INLINE_LEN);
-		}
-		txq_ctrl->txq.inlen_send = inlen_send;
-		txq_ctrl->txq.inlen_mode = inlen_mode;
-		txq_ctrl->txq.inlen_empw = 0;
+		inlen_empw = 0;
+	} else if (vlan_inline) {
+		/*
+		 * Hardware does not report offload for
+		 * VLAN insertion, we must enable data inline
+		 * to implement feature by software.
+		 */
+		inlen_send = MLX5_ESEG_MIN_INLINE_SIZE;
+		inlen_empw = 0;
+	} else {
 		inlen_send = 0;
 		inlen_empw = 0;
 	}
+	txq_ctrl->txq.inlen_send = inlen_send;
+	txq_ctrl->txq.inlen_mode = inlen_mode;
+	txq_ctrl->txq.inlen_empw = 0;
 	if (inlen_send && inlen_empw && priv->txqs_n >= txqs_inline) {
 		/*
 		 * The data sent with MLX5_OPCODE_ENHANCED_MPSW
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 5/6] net/mlx5: fix packet size inline settings

[Viacheslav Ovsiienko]

This patch fixes the default settings for packet size to inline
with Enhanced Multi-Packet Write feature, allowing 256B packets
to be inlined with Out-Of-the-Box settings.

Fixes: 50724e1bba76 ("net/mlx5: update Tx definitions")

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 doc/guides/nics/mlx5.rst    | 2 +-
 drivers/net/mlx5/mlx5_prm.h | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 46538b8..5102bcd 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -449,7 +449,7 @@ Run-time configuration
   and CPU resources are scarce), data inline is not performed by the driver.
   Assigning ``txqs_min_inline`` with zero always enables the data inline.
 
-  The default ``txq_inline_mpw`` value is 188. The specified value may be adjusted
+  The default ``txq_inline_mpw`` value is 268. The specified value may be adjusted
   by the driver in order not to exceed the limit (930 bytes) and to provide better
   WQE space filling without gaps, the adjustment is reflected in the debug log.
   Due to multiple packets may be included to the same WQE with Enhanced Multi
diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h
index 4ee6d89..d62837e 100644
--- a/drivers/net/mlx5/mlx5_prm.h
+++ b/drivers/net/mlx5/mlx5_prm.h
@@ -72,9 +72,8 @@
  * boundary with accounting the title Control and Ethernet
  * segments.
  */
-#define MLX5_EMPW_DEF_INLINE_LEN (3u * MLX5_WQE_SIZE + \
-				  MLX5_DSEG_MIN_INLINE_SIZE - \
-				  MLX5_WQE_DSEG_SIZE)
+#define MLX5_EMPW_DEF_INLINE_LEN (4u * MLX5_WQE_SIZE + \
+				  MLX5_DSEG_MIN_INLINE_SIZE)
 /*
  * Maximal inline data length sent with enhanced MPW.
  * Is based on maximal WQE size.
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 6/6] net/mlx5: fix completion queue overflow for large bursts

[Viacheslav Ovsiienko]

There is the limit on completion descriptor fetch to improve
latency. If burst size is large there might be not enough
resources freed in completion processing. This fix reiterates
sending loop and allows multiple completion descriptor fetch
and processing.

Fixes: 18a1c20044c0 ("net/mlx5: implement Tx burst template")

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 36 +++++++++++++++++++++++-------------
 drivers/net/mlx5/mlx5_rxtx.h |  1 +
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index a890f41..39076df 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -2142,8 +2142,7 @@ enum mlx5_txcmp_code {
 	uint16_t head = txq->elts_head;
 	unsigned int part;
 
-	part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc->pkts_sent -
-		(MLX5_TXOFF_CONFIG(MULTI) ? loc->pkts_copy : 0);
+	part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc->pkts_sent - loc->pkts_copy;
 	head += part;
 	if ((uint16_t)(head - txq->elts_comp) >= MLX5_TX_COMP_THRESH ||
 	     (MLX5_TXOFF_CONFIG(INLINE) &&
@@ -4520,6 +4519,14 @@ enum mlx5_txcmp_code {
 
 	assert(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));
 	assert(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));
+	if (unlikely(!pkts_n))
+		return 0;
+	loc.pkts_sent = 0;
+	loc.pkts_copy = 0;
+	loc.wqe_last = NULL;
+
+send_loop:
+	loc.pkts_loop = loc.pkts_sent;
 	/*
 	 * Check if there are some CQEs, if any:
 	 * - process an encountered errors
@@ -4527,9 +4534,7 @@ enum mlx5_txcmp_code {
 	 * - free related mbufs
 	 * - doorbell the NIC about processed CQEs
 	 */
-	if (unlikely(!pkts_n))
-		return 0;
-	rte_prefetch0(*pkts);
+	rte_prefetch0(*(pkts + loc.pkts_sent));
 	mlx5_tx_handle_completion(txq, olx);
 	/*
 	 * Calculate the number of available resources - elts and WQEs.
@@ -4546,10 +4551,7 @@ enum mlx5_txcmp_code {
 	loc.wqe_free = txq->wqe_s -
 				(uint16_t)(txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!loc.elts_free || !loc.wqe_free))
-		return 0;
-	loc.pkts_sent = 0;
-	loc.pkts_copy = 0;
-	loc.wqe_last = NULL;
+		return loc.pkts_sent;
 	for (;;) {
 		/*
 		 * Fetch the packet from array. Usually this is
@@ -4715,8 +4717,8 @@ enum mlx5_txcmp_code {
 	 */
 	assert(MLX5_TXOFF_CONFIG(INLINE) || loc.pkts_sent >= loc.pkts_copy);
 	/* Take a shortcut if nothing is sent. */
-	if (unlikely(loc.pkts_sent == 0))
-		return 0;
+	if (unlikely(loc.pkts_sent == loc.pkts_loop))
+		return loc.pkts_sent;
 	/*
 	 * Ring QP doorbell immediately after WQE building completion
 	 * to improve latencies. The pure software related data treatment
@@ -4725,8 +4727,7 @@ enum mlx5_txcmp_code {
 	 */
 	mlx5_tx_dbrec_cond_wmb(txq, loc.wqe_last, 0);
 	/* Not all of the mbufs may be stored into elts yet. */
-	part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc.pkts_sent -
-		(MLX5_TXOFF_CONFIG(MULTI) ? loc.pkts_copy : 0);
+	part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc.pkts_sent - loc.pkts_copy;
 	if (!MLX5_TXOFF_CONFIG(INLINE) && part) {
 		/*
 		 * There are some single-segment mbufs not stored in elts.
@@ -4738,6 +4739,7 @@ enum mlx5_txcmp_code {
 		 * inlined mbufs.
 		 */
 		mlx5_tx_copy_elts(txq, pkts + loc.pkts_copy, part, olx);
+		loc.pkts_copy = loc.pkts_sent;
 	}
 #ifdef MLX5_PMD_SOFT_COUNTERS
 	/* Increment sent packets counter. */
@@ -4745,6 +4747,14 @@ enum mlx5_txcmp_code {
 #endif
 	assert(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));
 	assert(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));
+	if (pkts_n > loc.pkts_sent) {
+		/*
+		 * If burst size is large there might be no enough CQE
+		 * fetched from completion queue and no enough resources
+		 * freed to send all the packets.
+		 */
+		goto send_loop;
+	}
 	return loc.pkts_sent;
 }
 
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index aaa02a2..bad9e9c 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -239,6 +239,7 @@ struct mlx5_txq_local {
 	struct rte_mbuf *mbuf; /* first mbuf to process. */
 	uint16_t pkts_copy; /* packets copied to elts. */
 	uint16_t pkts_sent; /* packets sent. */
+	uint16_t pkts_loop; /* packets sent on loop entry. */
 	uint16_t elts_free; /* available elts remain. */
 	uint16_t wqe_free; /* available wqe remain. */
 	uint16_t mbuf_off; /* data offset in current mbuf. */
-- 
1.8.3.1