[PATCH 0/4] net/failsafe: support deferred queue start

[PATCH 0/4] net/failsafe: support deferred queue start

[Andrew Rybchenko]

Ian Dolzhansky (4):
  app/testpmd: add queue deferred start switch
  net/failsafe: add checks for deferred queue setup
  net/failsafe: add Rx queue start and stop functions
  net/failsafe: add Tx queue start and stop functions

 app/test-pmd/cmdline.c                 |  91 ++++++++++++++
 doc/guides/nics/features/failsafe.ini  |   1 +
 doc/guides/rel_notes/release_18_11.rst |  13 ++
 drivers/net/failsafe/failsafe_ether.c  |  88 +++++++++++++
 drivers/net/failsafe/failsafe_ops.c    | 167 ++++++++++++++++++++++++-
 5 files changed, 359 insertions(+), 1 deletion(-)

-- 
2.17.1

[PATCH 1/4] app/testpmd: add queue deferred start switch

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 app/test-pmd/cmdline.c                 | 91 ++++++++++++++++++++++++++
 doc/guides/rel_notes/release_18_11.rst |  6 ++
 2 files changed, 97 insertions(+)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 589121d69..f47ec99f1 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -883,6 +883,10 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"    Start/stop a rx/tx queue of port X. Only take effect"
 			" when port X is started\n\n"
 
+			"port (port_id) (rxq|txq) (queue_id) deferred_start (on|off)\n"
+			"    Switch on/off a deferred start of port X rx/tx queue. Only"
+			" take effect when port X is stopped.\n\n"
+
 			"port (port_id) (rxq|txq) (queue_id) setup\n"
 			"    Setup a rx/tx queue of port X.\n\n"
 
@@ -2441,6 +2445,92 @@ cmdline_parse_inst_t cmd_config_rxtx_queue = {
 	},
 };
 
+/* *** configure port rxq/txq deferred start on/off *** */
+struct cmd_config_deferred_start_rxtx_queue {
+	cmdline_fixed_string_t port;
+	portid_t port_id;
+	cmdline_fixed_string_t rxtxq;
+	uint16_t qid;
+	cmdline_fixed_string_t opname;
+	cmdline_fixed_string_t state;
+};
+
+static void
+cmd_config_deferred_start_rxtx_queue_parsed(void *parsed_result,
+			__attribute__((unused)) struct cmdline *cl,
+			__attribute__((unused)) void *data)
+{
+	struct cmd_config_deferred_start_rxtx_queue *res = parsed_result;
+	struct rte_port *port;
+	uint8_t isrx;
+	uint8_t ison;
+	uint8_t needreconfig = 0;
+
+	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
+		return;
+
+	if (port_is_started(res->port_id) != 0) {
+		printf("Please stop port %u first\n", res->port_id);
+		return;
+	}
+
+	port = &ports[res->port_id];
+
+	isrx = !strcmp(res->rxtxq, "rxq");
+
+	if (isrx && rx_queue_id_is_invalid(res->qid))
+		return;
+	else if (!isrx && tx_queue_id_is_invalid(res->qid))
+		return;
+
+	ison = !strcmp(res->state, "on");
+
+	if (isrx && port->rx_conf[res->qid].rx_deferred_start != ison) {
+		port->rx_conf[res->qid].rx_deferred_start = ison;
+		needreconfig = 1;
+	} else if (!isrx && port->tx_conf[res->qid].tx_deferred_start != ison) {
+		port->tx_conf[res->qid].tx_deferred_start = ison;
+		needreconfig = 1;
+	}
+
+	if (needreconfig)
+		cmd_reconfig_device_queue(res->port_id, 0, 1);
+}
+
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						port, "port");
+cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_port_id =
+	TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						port_id, UINT16);
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_rxtxq =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						rxtxq, "rxq#txq");
+cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_qid =
+	TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						qid, UINT16);
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_opname =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						opname, "deferred_start");
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_state =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						state, "on#off");
+
+cmdline_parse_inst_t cmd_config_deferred_start_rxtx_queue = {
+	.f = cmd_config_deferred_start_rxtx_queue_parsed,
+	.data = NULL,
+	.help_str = "port <port_id> rxq|txq <queue_id> deferred_start on|off",
+	.tokens = {
+		(void *)&cmd_config_deferred_start_rxtx_queue_port,
+		(void *)&cmd_config_deferred_start_rxtx_queue_port_id,
+		(void *)&cmd_config_deferred_start_rxtx_queue_rxtxq,
+		(void *)&cmd_config_deferred_start_rxtx_queue_qid,
+		(void *)&cmd_config_deferred_start_rxtx_queue_opname,
+		(void *)&cmd_config_deferred_start_rxtx_queue_state,
+		NULL,
+	},
+};
+
 /* *** configure port rxq/txq setup *** */
 struct cmd_setup_rxtx_queue {
 	cmdline_fixed_string_t port;
@@ -17711,6 +17801,7 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_config_rss,
 	(cmdline_parse_inst_t *)&cmd_config_rxtx_ring_size,
 	(cmdline_parse_inst_t *)&cmd_config_rxtx_queue,
+	(cmdline_parse_inst_t *)&cmd_config_deferred_start_rxtx_queue,
 	(cmdline_parse_inst_t *)&cmd_setup_rxtx_queue,
 	(cmdline_parse_inst_t *)&cmd_config_rss_reta,
 	(cmdline_parse_inst_t *)&cmd_showport_reta,
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 24204e67b..1f17befd8 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -54,6 +54,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added ability to switch queue deferred start flag on testpmd app.**
+
+  Added a console command to testpmd app, giving ability to switch
+  ``rx_deferred_start`` or ``tx_deferred_start`` flag of the specified queue of
+  the specified port. The port must be stopped before the command call in order
+  to reconfigure queues.
 
 API Changes
 -----------
-- 
2.17.1

Re: [PATCH 1/4] app/testpmd: add queue deferred start switch

[Ferruh Yigit]

On 8/29/2018 8:16 AM, Andrew Rybchenko wrote:
> From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
> 
> Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
> ---
>  app/test-pmd/cmdline.c                 | 91 ++++++++++++++++++++++++++
>  doc/guides/rel_notes/release_18_11.rst |  6 ++
>  2 files changed, 97 insertions(+)
> 
> diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> index 589121d69..f47ec99f1 100644
> --- a/app/test-pmd/cmdline.c
> +++ b/app/test-pmd/cmdline.c
> @@ -883,6 +883,10 @@ static void cmd_help_long_parsed(void *parsed_result,
>  			"    Start/stop a rx/tx queue of port X. Only take effect"
>  			" when port X is started\n\n"
>  
> +			"port (port_id) (rxq|txq) (queue_id) deferred_start (on|off)\n"
> +			"    Switch on/off a deferred start of port X rx/tx queue. Only"
> +			" take effect when port X is stopped.\n\n"

Overall looks good to me.

But testpmd doc needs to be updated to add new command.
doc/guides/testpmd_app_ug/testpmd_funcs.rst

Also is there a way to see the current deferred_start status of a port, either
with a specific command or as part of other command like "show port info"?
If not does it make sense to have a way to see it?

Re: [PATCH 1/4] app/testpmd: add queue deferred start switch

[Andrew Rybchenko]

On 9/3/18 7:56 PM, Ferruh Yigit wrote:
> On 8/29/2018 8:16 AM, Andrew Rybchenko wrote:
>> From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
>>
>> Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
>> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
>> ---
>>   app/test-pmd/cmdline.c                 | 91 ++++++++++++++++++++++++++
>>   doc/guides/rel_notes/release_18_11.rst |  6 ++
>>   2 files changed, 97 insertions(+)
>>
>> diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
>> index 589121d69..f47ec99f1 100644
>> --- a/app/test-pmd/cmdline.c
>> +++ b/app/test-pmd/cmdline.c
>> @@ -883,6 +883,10 @@ static void cmd_help_long_parsed(void *parsed_result,
>>   			"    Start/stop a rx/tx queue of port X. Only take effect"
>>   			" when port X is started\n\n"
>>   
>> +			"port (port_id) (rxq|txq) (queue_id) deferred_start (on|off)\n"
>> +			"    Switch on/off a deferred start of port X rx/tx queue. Only"
>> +			" take effect when port X is stopped.\n\n"
> Overall looks good to me.
>
> But testpmd doc needs to be updated to add new command.
> doc/guides/testpmd_app_ug/testpmd_funcs.rst

Will fix in v2

> Also is there a way to see the current deferred_start status of a port, either
> with a specific command or as part of other command like "show port info"?
> If not does it make sense to have a way to see it?

The following command does it:
show rxq|txq info <port_id> <queue_id>

Thanks,
Andrew.

[PATCH 2/4] net/failsafe: add checks for deferred queue setup

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Fixes: a46f8d584eb8 ("net/failsafe: add fail-safe PMD")
Cc: stable@dpdk.org

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 drivers/net/failsafe/failsafe_ops.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index 24e91c931..f7cce0d8f 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -340,6 +340,11 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	if (rx_conf->rx_deferred_start) {
+		ERROR("Rx queue deferred start is not supported");
+		return -EINVAL;
+	}
+
 	fs_lock(dev, 0);
 	rxq = dev->data->rx_queues[rx_queue_id];
 	if (rxq != NULL) {
@@ -497,6 +502,11 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	if (tx_conf->tx_deferred_start) {
+		ERROR("Tx queue deferred start is not supported");
+		return -EINVAL;
+	}
+
 	fs_lock(dev, 0);
 	txq = dev->data->tx_queues[tx_queue_id];
 	if (txq != NULL) {
-- 
2.17.1

[PATCH 3/4] net/failsafe: add Rx queue start and stop functions

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Support Rx queue deferred start.

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 doc/guides/nics/features/failsafe.ini  |  1 +
 doc/guides/rel_notes/release_18_11.rst |  7 ++
 drivers/net/failsafe/failsafe_ether.c  | 44 ++++++++++++
 drivers/net/failsafe/failsafe_ops.c    | 96 ++++++++++++++++++++++++--
 4 files changed, 143 insertions(+), 5 deletions(-)

diff --git a/doc/guides/nics/features/failsafe.ini b/doc/guides/nics/features/failsafe.ini
index 39ee57965..712c0b7f7 100644
--- a/doc/guides/nics/features/failsafe.ini
+++ b/doc/guides/nics/features/failsafe.ini
@@ -7,6 +7,7 @@
 Link status          = Y
 Link status event    = Y
 Rx interrupt         = Y
+Queue start/stop     = P
 MTU update           = Y
 Jumbo frame          = Y
 Promiscuous mode     = Y
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 1f17befd8..882ef8ac6 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -54,6 +54,13 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Updated failsafe driver.**
+
+  Updated the failsafe driver including the following changes:
+
+  * Support for Rx queues start and stop.
+  * Support for Rx queues deferred start.
+
 * **Added ability to switch queue deferred start flag on testpmd app.**
 
   Added a console command to testpmd app, giving ability to switch
diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c
index 5b5cb3b49..305deed63 100644
--- a/drivers/net/failsafe/failsafe_ether.c
+++ b/drivers/net/failsafe/failsafe_ether.c
@@ -366,6 +366,47 @@ failsafe_dev_remove(struct rte_eth_dev *dev)
 		}
 }
 
+static int
+failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
+{
+	struct rxq *rxq;
+	int ret;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		rxq = dev->data->rx_queues[i];
+
+		if (rxq->info.conf.rx_deferred_start &&
+		    dev->data->rx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STARTED) {
+			/*
+			 * The subdevice Rx queue does not launch on device
+			 * start if deferred start flag is set. It needs to be
+			 * started manually in case an appropriate failsafe Rx
+			 * queue has been started earlier.
+			 */
+			ret = dev->dev_ops->rx_queue_start(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Rx queue %d", i);
+				return ret;
+			}
+		} else if (dev->data->rx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STOPPED) {
+			/*
+			 * The subdevice Rx queue needs to be stopped manually
+			 * in case an appropriate failsafe Rx queue has been
+			 * stopped earlier.
+			 */
+			ret = dev->dev_ops->rx_queue_stop(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Rx queue %d", i);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
 int
 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 {
@@ -422,6 +463,9 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 	if (PRIV(dev)->state < DEV_STARTED)
 		return 0;
 	ret = dev->dev_ops->dev_start(dev);
+	if (ret)
+		goto err_remove;
+	ret = failsafe_eth_dev_rx_queues_sync(dev);
 	if (ret)
 		goto err_remove;
 	return 0;
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index f7cce0d8f..412d522cf 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -170,6 +170,20 @@ fs_dev_configure(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+fs_set_queues_state_start(struct rte_eth_dev *dev)
+{
+	struct rxq *rxq;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		rxq = dev->data->rx_queues[i];
+		if (!rxq->info.conf.rx_deferred_start)
+			dev->data->rx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STARTED;
+	}
+}
+
 static int
 fs_dev_start(struct rte_eth_dev *dev)
 {
@@ -204,13 +218,24 @@ fs_dev_start(struct rte_eth_dev *dev)
 		}
 		sdev->state = DEV_STARTED;
 	}
-	if (PRIV(dev)->state < DEV_STARTED)
+	if (PRIV(dev)->state < DEV_STARTED) {
 		PRIV(dev)->state = DEV_STARTED;
+		fs_set_queues_state_start(dev);
+	}
 	fs_switch_dev(dev, NULL);
 	fs_unlock(dev, 0);
 	return 0;
 }
 
+static void
+fs_set_queues_state_stop(struct rte_eth_dev *dev)
+{
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+}
+
 static void
 fs_dev_stop(struct rte_eth_dev *dev)
 {
@@ -225,6 +250,7 @@ fs_dev_stop(struct rte_eth_dev *dev)
 		sdev->state = DEV_STARTED - 1;
 	}
 	failsafe_rx_intr_uninstall(dev);
+	fs_set_queues_state_stop(dev);
 	fs_unlock(dev, 0);
 }
 
@@ -340,12 +366,17 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	fs_lock(dev, 0);
 	if (rx_conf->rx_deferred_start) {
-		ERROR("Rx queue deferred start is not supported");
-		return -EINVAL;
+		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
+			if (SUBOPS(sdev, rx_queue_start) == NULL) {
+				ERROR("Rx queue deferred start is not "
+					"supported for subdevice %d", i);
+				fs_unlock(dev, 0);
+				return -EINVAL;
+			}
+		}
 	}
-
-	fs_lock(dev, 0);
 	rxq = dev->data->rx_queues[rx_queue_id];
 	if (rxq != NULL) {
 		fs_rx_queue_release(rxq);
@@ -393,6 +424,59 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
 	return ret;
 }
 
+static int
+fs_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+	int err = 0;
+	bool failure = true;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_rx_queue_stop(port_id, rx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Rx queue stop failed for subdevice %d", i);
+			err = ret;
+		} else {
+			failure = false;
+		}
+	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	fs_unlock(dev, 0);
+	/* Return 0 in case of at least one successful queue stop */
+	return (failure) ? err : 0;
+}
+
+static int
+fs_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_rx_queue_start(port_id, rx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Rx queue start failed for subdevice %d", i);
+			fs_rx_queue_stop(dev, rx_queue_id);
+			fs_unlock(dev, 0);
+			return ret;
+		}
+	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	fs_unlock(dev, 0);
+	return 0;
+}
+
 static int
 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 {
@@ -1037,6 +1121,8 @@ const struct eth_dev_ops failsafe_ops = {
 	.vlan_filter_set = fs_vlan_filter_set,
 	.rx_queue_setup = fs_rx_queue_setup,
 	.tx_queue_setup = fs_tx_queue_setup,
+	.rx_queue_start = fs_rx_queue_start,
+	.rx_queue_stop = fs_rx_queue_stop,
 	.rx_queue_release = fs_rx_queue_release,
 	.tx_queue_release = fs_tx_queue_release,
 	.rx_queue_intr_enable = fs_rx_intr_enable,
-- 
2.17.1

[PATCH 4/4] net/failsafe: add Tx queue start and stop functions

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Support Tx queue deferred start.

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 doc/guides/nics/features/failsafe.ini  |  2 +-
 doc/guides/rel_notes/release_18_11.rst |  4 +-
 drivers/net/failsafe/failsafe_ether.c  | 44 +++++++++++++++
 drivers/net/failsafe/failsafe_ops.c    | 77 ++++++++++++++++++++++++--
 4 files changed, 120 insertions(+), 7 deletions(-)

diff --git a/doc/guides/nics/features/failsafe.ini b/doc/guides/nics/features/failsafe.ini
index 712c0b7f7..74eae4a62 100644
--- a/doc/guides/nics/features/failsafe.ini
+++ b/doc/guides/nics/features/failsafe.ini
@@ -7,7 +7,7 @@
 Link status          = Y
 Link status event    = Y
 Rx interrupt         = Y
-Queue start/stop     = P
+Queue start/stop     = Y
 MTU update           = Y
 Jumbo frame          = Y
 Promiscuous mode     = Y
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 882ef8ac6..ad08a204f 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -58,8 +58,8 @@ New Features
 
   Updated the failsafe driver including the following changes:
 
-  * Support for Rx queues start and stop.
-  * Support for Rx queues deferred start.
+  * Support for Rx and Tx queues start and stop.
+  * Support for Rx and Tx queues deferred start.
 
 * **Added ability to switch queue deferred start flag on testpmd app.**
 
diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c
index 305deed63..191f95f14 100644
--- a/drivers/net/failsafe/failsafe_ether.c
+++ b/drivers/net/failsafe/failsafe_ether.c
@@ -407,6 +407,47 @@ failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static int
+failsafe_eth_dev_tx_queues_sync(struct rte_eth_dev *dev)
+{
+	struct txq *txq;
+	int ret;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		txq = dev->data->tx_queues[i];
+
+		if (txq->info.conf.tx_deferred_start &&
+		    dev->data->tx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STARTED) {
+			/*
+			 * The subdevice Tx queue does not launch on device
+			 * start if deferred start flag is set. It needs to be
+			 * started manually in case an appropriate failsafe Tx
+			 * queue has been started earlier.
+			 */
+			ret = dev->dev_ops->tx_queue_start(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Tx queue %d", i);
+				return ret;
+			}
+		} else if (dev->data->tx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STOPPED) {
+			/*
+			 * The subdevice Tx queue needs to be stopped manually
+			 * in case an appropriate failsafe Tx queue has been
+			 * stopped earlier.
+			 */
+			ret = dev->dev_ops->tx_queue_stop(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Tx queue %d", i);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
 int
 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 {
@@ -466,6 +507,9 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 	if (ret)
 		goto err_remove;
 	ret = failsafe_eth_dev_rx_queues_sync(dev);
+	if (ret)
+		goto err_remove;
+	ret = failsafe_eth_dev_tx_queues_sync(dev);
 	if (ret)
 		goto err_remove;
 	return 0;
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index 412d522cf..4d30eb22d 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -174,6 +174,7 @@ static void
 fs_set_queues_state_start(struct rte_eth_dev *dev)
 {
 	struct rxq *rxq;
+	struct txq *txq;
 	uint16_t i;
 
 	for (i = 0; i < dev->data->nb_rx_queues; i++) {
@@ -182,6 +183,12 @@ fs_set_queues_state_start(struct rte_eth_dev *dev)
 			dev->data->rx_queue_state[i] =
 						RTE_ETH_QUEUE_STATE_STARTED;
 	}
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		txq = dev->data->tx_queues[i];
+		if (!txq->info.conf.tx_deferred_start)
+			dev->data->tx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STARTED;
+	}
 }
 
 static int
@@ -234,6 +241,8 @@ fs_set_queues_state_stop(struct rte_eth_dev *dev)
 
 	for (i = 0; i < dev->data->nb_rx_queues; i++)
 		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
 }
 
 static void
@@ -586,12 +595,17 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	fs_lock(dev, 0);
 	if (tx_conf->tx_deferred_start) {
-		ERROR("Tx queue deferred start is not supported");
-		return -EINVAL;
+		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
+			if (SUBOPS(sdev, tx_queue_start) == NULL) {
+				ERROR("Tx queue deferred start is not "
+					"supported for subdevice %d", i);
+				fs_unlock(dev, 0);
+				return -EINVAL;
+			}
+		}
 	}
-
-	fs_lock(dev, 0);
 	txq = dev->data->tx_queues[tx_queue_id];
 	if (txq != NULL) {
 		fs_tx_queue_release(txq);
@@ -631,6 +645,59 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
 	return ret;
 }
 
+static int
+fs_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+	int err = 0;
+	bool failure = true;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_tx_queue_stop(port_id, tx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Tx queue stop failed for subdevice %d", i);
+			err = ret;
+		} else {
+			failure = false;
+		}
+	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	fs_unlock(dev, 0);
+	/* Return 0 in case of at least one successful queue stop */
+	return (failure) ? err : 0;
+}
+
+static int
+fs_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_tx_queue_start(port_id, tx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Tx queue start failed for subdevice %d", i);
+			fs_tx_queue_stop(dev, tx_queue_id);
+			fs_unlock(dev, 0);
+			return ret;
+		}
+	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	fs_unlock(dev, 0);
+	return 0;
+}
+
 static void
 fs_dev_free_queues(struct rte_eth_dev *dev)
 {
@@ -1122,7 +1189,9 @@ const struct eth_dev_ops failsafe_ops = {
 	.rx_queue_setup = fs_rx_queue_setup,
 	.tx_queue_setup = fs_tx_queue_setup,
 	.rx_queue_start = fs_rx_queue_start,
+	.tx_queue_start = fs_tx_queue_start,
 	.rx_queue_stop = fs_rx_queue_stop,
+	.tx_queue_stop = fs_tx_queue_stop,
 	.rx_queue_release = fs_rx_queue_release,
 	.tx_queue_release = fs_tx_queue_release,
 	.rx_queue_intr_enable = fs_rx_intr_enable,
-- 
2.17.1

Re: [PATCH 0/4] net/failsafe: support deferred queue start

[Gaëtan Rivet]

Hello Ian, Andrew,

Clean patchset.
Only nitpick would be on eth_dev_ops ordering, but this is secondary.

For the series:
Acked-by: Gaetan Rivet <gaetan.rivet@6wind.com>

On Wed, Aug 29, 2018 at 08:16:02AM +0100, Andrew Rybchenko wrote:
> Ian Dolzhansky (4):
>   app/testpmd: add queue deferred start switch
>   net/failsafe: add checks for deferred queue setup
>   net/failsafe: add Rx queue start and stop functions
>   net/failsafe: add Tx queue start and stop functions
> 
>  app/test-pmd/cmdline.c                 |  91 ++++++++++++++
>  doc/guides/nics/features/failsafe.ini  |   1 +
>  doc/guides/rel_notes/release_18_11.rst |  13 ++
>  drivers/net/failsafe/failsafe_ether.c  |  88 +++++++++++++
>  drivers/net/failsafe/failsafe_ops.c    | 167 ++++++++++++++++++++++++-
>  5 files changed, 359 insertions(+), 1 deletion(-)
> 
> -- 
> 2.17.1
> 

-- 
Gaëtan Rivet
6WIND

[PATCH v2 0/4] net/failsafe: support deferred queue start

[Andrew Rybchenko]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="y", Size: 1233 bytes --]

Since the topic is raised in multicast address list patchset, I'd like
to highlight it here as well. Current version uses failsafe ops directly
on sync to synchronize queues state which iterates over all sub-devices.
For already in sync sub-devices it does not go to driver since ethdev
functions checks current state and do nothing if it is already OK.
In theory it is possible to limit it to inactive devices and use
ethdev API instead of direct ops, but it requires a bit more lines of
code.

v2:
    - fix ops ordering
    - update testpmd documentation
    - add Gaëtan's acks

Ian Dolzhansky (4):
  app/testpmd: add queue deferred start switch
  net/failsafe: add checks for deferred queue setup
  net/failsafe: add Rx queue start and stop functions
  net/failsafe: add Tx queue start and stop functions

 app/test-pmd/cmdline.c                      |  91 +++++++++++
 doc/guides/nics/features/failsafe.ini       |   1 +
 doc/guides/rel_notes/release_18_11.rst      |  15 ++
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |   7 +
 drivers/net/failsafe/failsafe_ether.c       |  88 +++++++++++
 drivers/net/failsafe/failsafe_ops.c         | 167 +++++++++++++++++++-
 6 files changed, 368 insertions(+), 1 deletion(-)

-- 
2.17.1

[PATCH v2 1/4] app/testpmd: add queue deferred start switch

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Acked-by: Gaetan Rivet <gaetan.rivet@6wind.com>
---
 app/test-pmd/cmdline.c                      | 91 +++++++++++++++++++++
 doc/guides/rel_notes/release_18_11.rst      |  8 ++
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  7 ++
 3 files changed, 106 insertions(+)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 0cbd340c1..0c5399dc4 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -883,6 +883,10 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"    Start/stop a rx/tx queue of port X. Only take effect"
 			" when port X is started\n\n"
 
+			"port (port_id) (rxq|txq) (queue_id) deferred_start (on|off)\n"
+			"    Switch on/off a deferred start of port X rx/tx queue. Only"
+			" take effect when port X is stopped.\n\n"
+
 			"port (port_id) (rxq|txq) (queue_id) setup\n"
 			"    Setup a rx/tx queue of port X.\n\n"
 
@@ -2439,6 +2443,92 @@ cmdline_parse_inst_t cmd_config_rxtx_queue = {
 	},
 };
 
+/* *** configure port rxq/txq deferred start on/off *** */
+struct cmd_config_deferred_start_rxtx_queue {
+	cmdline_fixed_string_t port;
+	portid_t port_id;
+	cmdline_fixed_string_t rxtxq;
+	uint16_t qid;
+	cmdline_fixed_string_t opname;
+	cmdline_fixed_string_t state;
+};
+
+static void
+cmd_config_deferred_start_rxtx_queue_parsed(void *parsed_result,
+			__attribute__((unused)) struct cmdline *cl,
+			__attribute__((unused)) void *data)
+{
+	struct cmd_config_deferred_start_rxtx_queue *res = parsed_result;
+	struct rte_port *port;
+	uint8_t isrx;
+	uint8_t ison;
+	uint8_t needreconfig = 0;
+
+	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
+		return;
+
+	if (port_is_started(res->port_id) != 0) {
+		printf("Please stop port %u first\n", res->port_id);
+		return;
+	}
+
+	port = &ports[res->port_id];
+
+	isrx = !strcmp(res->rxtxq, "rxq");
+
+	if (isrx && rx_queue_id_is_invalid(res->qid))
+		return;
+	else if (!isrx && tx_queue_id_is_invalid(res->qid))
+		return;
+
+	ison = !strcmp(res->state, "on");
+
+	if (isrx && port->rx_conf[res->qid].rx_deferred_start != ison) {
+		port->rx_conf[res->qid].rx_deferred_start = ison;
+		needreconfig = 1;
+	} else if (!isrx && port->tx_conf[res->qid].tx_deferred_start != ison) {
+		port->tx_conf[res->qid].tx_deferred_start = ison;
+		needreconfig = 1;
+	}
+
+	if (needreconfig)
+		cmd_reconfig_device_queue(res->port_id, 0, 1);
+}
+
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						port, "port");
+cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_port_id =
+	TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						port_id, UINT16);
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_rxtxq =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						rxtxq, "rxq#txq");
+cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_qid =
+	TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						qid, UINT16);
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_opname =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						opname, "deferred_start");
+cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_state =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue,
+						state, "on#off");
+
+cmdline_parse_inst_t cmd_config_deferred_start_rxtx_queue = {
+	.f = cmd_config_deferred_start_rxtx_queue_parsed,
+	.data = NULL,
+	.help_str = "port <port_id> rxq|txq <queue_id> deferred_start on|off",
+	.tokens = {
+		(void *)&cmd_config_deferred_start_rxtx_queue_port,
+		(void *)&cmd_config_deferred_start_rxtx_queue_port_id,
+		(void *)&cmd_config_deferred_start_rxtx_queue_rxtxq,
+		(void *)&cmd_config_deferred_start_rxtx_queue_qid,
+		(void *)&cmd_config_deferred_start_rxtx_queue_opname,
+		(void *)&cmd_config_deferred_start_rxtx_queue_state,
+		NULL,
+	},
+};
+
 /* *** configure port rxq/txq setup *** */
 struct cmd_setup_rxtx_queue {
 	cmdline_fixed_string_t port;
@@ -17709,6 +17799,7 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_config_rss,
 	(cmdline_parse_inst_t *)&cmd_config_rxtx_ring_size,
 	(cmdline_parse_inst_t *)&cmd_config_rxtx_queue,
+	(cmdline_parse_inst_t *)&cmd_config_deferred_start_rxtx_queue,
 	(cmdline_parse_inst_t *)&cmd_setup_rxtx_queue,
 	(cmdline_parse_inst_t *)&cmd_config_rss_reta,
 	(cmdline_parse_inst_t *)&cmd_showport_reta,
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 219b0b1cf..e9f3a415c 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -72,6 +72,14 @@ New Features
   choose the latest vector path that the platform supported. For example, users
   can use AVX2 vector path on BDW/HSW to get better performance.
 
+* **Added ability to switch queue deferred start flag on testpmd app.**
+
+  Added a console command to testpmd app, giving ability to switch
+  ``rx_deferred_start`` or ``tx_deferred_start`` flag of the specified queue of
+  the specified port. The port must be stopped before the command call in order
+  to reconfigure queues.
+
+
 API Changes
 -----------
 
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index dde205a2b..3a73000a6 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -1765,6 +1765,13 @@ Start/stop a rx/tx queue on a specific port::
 
    testpmd> port (port_id) (rxq|txq) (queue_id) (start|stop)
 
+port config - queue deferred start
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Switch on/off deferred start of a specific port queue::
+
+   testpmd> port (port_id) (rxq|txq) (queue_id) deferred_start (on|off)
+
 port setup queue
 ~~~~~~~~~~~~~~~~~~~~~
 
-- 
2.17.1

[PATCH v2 2/4] net/failsafe: add checks for deferred queue setup

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Fixes: a46f8d584eb8 ("net/failsafe: add fail-safe PMD")
Cc: stable@dpdk.org

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Acked-by: Gaetan Rivet <gaetan.rivet@6wind.com>
---
 drivers/net/failsafe/failsafe_ops.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index d989a17bf..9608d09cb 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -338,6 +338,11 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	if (rx_conf->rx_deferred_start) {
+		ERROR("Rx queue deferred start is not supported");
+		return -EINVAL;
+	}
+
 	fs_lock(dev, 0);
 	rxq = dev->data->rx_queues[rx_queue_id];
 	if (rxq != NULL) {
@@ -495,6 +500,11 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	if (tx_conf->tx_deferred_start) {
+		ERROR("Tx queue deferred start is not supported");
+		return -EINVAL;
+	}
+
 	fs_lock(dev, 0);
 	txq = dev->data->tx_queues[tx_queue_id];
 	if (txq != NULL) {
-- 
2.17.1

[PATCH v2 3/4] net/failsafe: add Rx queue start and stop functions

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Support Rx queue deferred start.

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Acked-by: Gaetan Rivet <gaetan.rivet@6wind.com>
---
 doc/guides/nics/features/failsafe.ini  |  1 +
 doc/guides/rel_notes/release_18_11.rst |  7 ++
 drivers/net/failsafe/failsafe_ether.c  | 44 ++++++++++++
 drivers/net/failsafe/failsafe_ops.c    | 96 ++++++++++++++++++++++++--
 4 files changed, 143 insertions(+), 5 deletions(-)

diff --git a/doc/guides/nics/features/failsafe.ini b/doc/guides/nics/features/failsafe.ini
index 39ee57965..712c0b7f7 100644
--- a/doc/guides/nics/features/failsafe.ini
+++ b/doc/guides/nics/features/failsafe.ini
@@ -7,6 +7,7 @@
 Link status          = Y
 Link status event    = Y
 Rx interrupt         = Y
+Queue start/stop     = P
 MTU update           = Y
 Jumbo frame          = Y
 Promiscuous mode     = Y
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index e9f3a415c..d6af403b1 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -72,6 +72,13 @@ New Features
   choose the latest vector path that the platform supported. For example, users
   can use AVX2 vector path on BDW/HSW to get better performance.
 
+* **Updated failsafe driver.**
+
+  Updated the failsafe driver including the following changes:
+
+  * Support for Rx queues start and stop.
+  * Support for Rx queues deferred start.
+
 * **Added ability to switch queue deferred start flag on testpmd app.**
 
   Added a console command to testpmd app, giving ability to switch
diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c
index 5b5cb3b49..305deed63 100644
--- a/drivers/net/failsafe/failsafe_ether.c
+++ b/drivers/net/failsafe/failsafe_ether.c
@@ -366,6 +366,47 @@ failsafe_dev_remove(struct rte_eth_dev *dev)
 		}
 }
 
+static int
+failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
+{
+	struct rxq *rxq;
+	int ret;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		rxq = dev->data->rx_queues[i];
+
+		if (rxq->info.conf.rx_deferred_start &&
+		    dev->data->rx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STARTED) {
+			/*
+			 * The subdevice Rx queue does not launch on device
+			 * start if deferred start flag is set. It needs to be
+			 * started manually in case an appropriate failsafe Rx
+			 * queue has been started earlier.
+			 */
+			ret = dev->dev_ops->rx_queue_start(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Rx queue %d", i);
+				return ret;
+			}
+		} else if (dev->data->rx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STOPPED) {
+			/*
+			 * The subdevice Rx queue needs to be stopped manually
+			 * in case an appropriate failsafe Rx queue has been
+			 * stopped earlier.
+			 */
+			ret = dev->dev_ops->rx_queue_stop(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Rx queue %d", i);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
 int
 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 {
@@ -422,6 +463,9 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 	if (PRIV(dev)->state < DEV_STARTED)
 		return 0;
 	ret = dev->dev_ops->dev_start(dev);
+	if (ret)
+		goto err_remove;
+	ret = failsafe_eth_dev_rx_queues_sync(dev);
 	if (ret)
 		goto err_remove;
 	return 0;
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index 9608d09cb..402c5b62c 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -168,6 +168,20 @@ fs_dev_configure(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+fs_set_queues_state_start(struct rte_eth_dev *dev)
+{
+	struct rxq *rxq;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		rxq = dev->data->rx_queues[i];
+		if (!rxq->info.conf.rx_deferred_start)
+			dev->data->rx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STARTED;
+	}
+}
+
 static int
 fs_dev_start(struct rte_eth_dev *dev)
 {
@@ -202,13 +216,24 @@ fs_dev_start(struct rte_eth_dev *dev)
 		}
 		sdev->state = DEV_STARTED;
 	}
-	if (PRIV(dev)->state < DEV_STARTED)
+	if (PRIV(dev)->state < DEV_STARTED) {
 		PRIV(dev)->state = DEV_STARTED;
+		fs_set_queues_state_start(dev);
+	}
 	fs_switch_dev(dev, NULL);
 	fs_unlock(dev, 0);
 	return 0;
 }
 
+static void
+fs_set_queues_state_stop(struct rte_eth_dev *dev)
+{
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+}
+
 static void
 fs_dev_stop(struct rte_eth_dev *dev)
 {
@@ -223,6 +248,7 @@ fs_dev_stop(struct rte_eth_dev *dev)
 		sdev->state = DEV_STARTED - 1;
 	}
 	failsafe_rx_intr_uninstall(dev);
+	fs_set_queues_state_stop(dev);
 	fs_unlock(dev, 0);
 }
 
@@ -292,6 +318,59 @@ fs_dev_close(struct rte_eth_dev *dev)
 	fs_unlock(dev, 0);
 }
 
+static int
+fs_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+	int err = 0;
+	bool failure = true;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_rx_queue_stop(port_id, rx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Rx queue stop failed for subdevice %d", i);
+			err = ret;
+		} else {
+			failure = false;
+		}
+	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	fs_unlock(dev, 0);
+	/* Return 0 in case of at least one successful queue stop */
+	return (failure) ? err : 0;
+}
+
+static int
+fs_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_rx_queue_start(port_id, rx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Rx queue start failed for subdevice %d", i);
+			fs_rx_queue_stop(dev, rx_queue_id);
+			fs_unlock(dev, 0);
+			return ret;
+		}
+	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	fs_unlock(dev, 0);
+	return 0;
+}
+
 static void
 fs_rx_queue_release(void *queue)
 {
@@ -338,12 +417,17 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	fs_lock(dev, 0);
 	if (rx_conf->rx_deferred_start) {
-		ERROR("Rx queue deferred start is not supported");
-		return -EINVAL;
+		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
+			if (SUBOPS(sdev, rx_queue_start) == NULL) {
+				ERROR("Rx queue deferred start is not "
+					"supported for subdevice %d", i);
+				fs_unlock(dev, 0);
+				return -EINVAL;
+			}
+		}
 	}
-
-	fs_lock(dev, 0);
 	rxq = dev->data->rx_queues[rx_queue_id];
 	if (rxq != NULL) {
 		fs_rx_queue_release(rxq);
@@ -1033,6 +1117,8 @@ const struct eth_dev_ops failsafe_ops = {
 	.dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
 	.mtu_set = fs_mtu_set,
 	.vlan_filter_set = fs_vlan_filter_set,
+	.rx_queue_start = fs_rx_queue_start,
+	.rx_queue_stop = fs_rx_queue_stop,
 	.rx_queue_setup = fs_rx_queue_setup,
 	.tx_queue_setup = fs_tx_queue_setup,
 	.rx_queue_release = fs_rx_queue_release,
-- 
2.17.1

[PATCH v2 4/4] net/failsafe: add Tx queue start and stop functions

[Andrew Rybchenko]

From: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>

Support Tx queue deferred start.

Signed-off-by: Ian Dolzhansky <Ian.Dolzhansky@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Acked-by: Gaetan Rivet <gaetan.rivet@6wind.com>
---
 doc/guides/nics/features/failsafe.ini  |  2 +-
 doc/guides/rel_notes/release_18_11.rst |  4 +-
 drivers/net/failsafe/failsafe_ether.c  | 44 +++++++++++++++
 drivers/net/failsafe/failsafe_ops.c    | 77 ++++++++++++++++++++++++--
 4 files changed, 120 insertions(+), 7 deletions(-)

diff --git a/doc/guides/nics/features/failsafe.ini b/doc/guides/nics/features/failsafe.ini
index 712c0b7f7..74eae4a62 100644
--- a/doc/guides/nics/features/failsafe.ini
+++ b/doc/guides/nics/features/failsafe.ini
@@ -7,7 +7,7 @@
 Link status          = Y
 Link status event    = Y
 Rx interrupt         = Y
-Queue start/stop     = P
+Queue start/stop     = Y
 MTU update           = Y
 Jumbo frame          = Y
 Promiscuous mode     = Y
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index d6af403b1..d8faf9ed7 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -76,8 +76,8 @@ New Features
 
   Updated the failsafe driver including the following changes:
 
-  * Support for Rx queues start and stop.
-  * Support for Rx queues deferred start.
+  * Support for Rx and Tx queues start and stop.
+  * Support for Rx and Tx queues deferred start.
 
 * **Added ability to switch queue deferred start flag on testpmd app.**
 
diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c
index 305deed63..191f95f14 100644
--- a/drivers/net/failsafe/failsafe_ether.c
+++ b/drivers/net/failsafe/failsafe_ether.c
@@ -407,6 +407,47 @@ failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static int
+failsafe_eth_dev_tx_queues_sync(struct rte_eth_dev *dev)
+{
+	struct txq *txq;
+	int ret;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		txq = dev->data->tx_queues[i];
+
+		if (txq->info.conf.tx_deferred_start &&
+		    dev->data->tx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STARTED) {
+			/*
+			 * The subdevice Tx queue does not launch on device
+			 * start if deferred start flag is set. It needs to be
+			 * started manually in case an appropriate failsafe Tx
+			 * queue has been started earlier.
+			 */
+			ret = dev->dev_ops->tx_queue_start(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Tx queue %d", i);
+				return ret;
+			}
+		} else if (dev->data->tx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STOPPED) {
+			/*
+			 * The subdevice Tx queue needs to be stopped manually
+			 * in case an appropriate failsafe Tx queue has been
+			 * stopped earlier.
+			 */
+			ret = dev->dev_ops->tx_queue_stop(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Tx queue %d", i);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
 int
 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 {
@@ -466,6 +507,9 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 	if (ret)
 		goto err_remove;
 	ret = failsafe_eth_dev_rx_queues_sync(dev);
+	if (ret)
+		goto err_remove;
+	ret = failsafe_eth_dev_tx_queues_sync(dev);
 	if (ret)
 		goto err_remove;
 	return 0;
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index 402c5b62c..9237e6331 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -172,6 +172,7 @@ static void
 fs_set_queues_state_start(struct rte_eth_dev *dev)
 {
 	struct rxq *rxq;
+	struct txq *txq;
 	uint16_t i;
 
 	for (i = 0; i < dev->data->nb_rx_queues; i++) {
@@ -180,6 +181,12 @@ fs_set_queues_state_start(struct rte_eth_dev *dev)
 			dev->data->rx_queue_state[i] =
 						RTE_ETH_QUEUE_STATE_STARTED;
 	}
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		txq = dev->data->tx_queues[i];
+		if (!txq->info.conf.tx_deferred_start)
+			dev->data->tx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STARTED;
+	}
 }
 
 static int
@@ -232,6 +239,8 @@ fs_set_queues_state_stop(struct rte_eth_dev *dev)
 
 	for (i = 0; i < dev->data->nb_rx_queues; i++)
 		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
 }
 
 static void
@@ -371,6 +380,59 @@ fs_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	return 0;
 }
 
+static int
+fs_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+	int err = 0;
+	bool failure = true;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_tx_queue_stop(port_id, tx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Tx queue stop failed for subdevice %d", i);
+			err = ret;
+		} else {
+			failure = false;
+		}
+	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	fs_unlock(dev, 0);
+	/* Return 0 in case of at least one successful queue stop */
+	return (failure) ? err : 0;
+}
+
+static int
+fs_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_tx_queue_start(port_id, tx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Tx queue start failed for subdevice %d", i);
+			fs_tx_queue_stop(dev, tx_queue_id);
+			fs_unlock(dev, 0);
+			return ret;
+		}
+	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	fs_unlock(dev, 0);
+	return 0;
+}
+
 static void
 fs_rx_queue_release(void *queue)
 {
@@ -584,12 +646,17 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
 	uint8_t i;
 	int ret;
 
+	fs_lock(dev, 0);
 	if (tx_conf->tx_deferred_start) {
-		ERROR("Tx queue deferred start is not supported");
-		return -EINVAL;
+		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
+			if (SUBOPS(sdev, tx_queue_start) == NULL) {
+				ERROR("Tx queue deferred start is not "
+					"supported for subdevice %d", i);
+				fs_unlock(dev, 0);
+				return -EINVAL;
+			}
+		}
 	}
-
-	fs_lock(dev, 0);
 	txq = dev->data->tx_queues[tx_queue_id];
 	if (txq != NULL) {
 		fs_tx_queue_release(txq);
@@ -1119,6 +1186,8 @@ const struct eth_dev_ops failsafe_ops = {
 	.vlan_filter_set = fs_vlan_filter_set,
 	.rx_queue_start = fs_rx_queue_start,
 	.rx_queue_stop = fs_rx_queue_stop,
+	.tx_queue_start = fs_tx_queue_start,
+	.tx_queue_stop = fs_tx_queue_stop,
 	.rx_queue_setup = fs_rx_queue_setup,
 	.tx_queue_setup = fs_tx_queue_setup,
 	.rx_queue_release = fs_rx_queue_release,
-- 
2.17.1

Re: [PATCH v2 0/4] net/failsafe: support deferred queue start

[Ferruh Yigit]

On 9/20/2018 2:55 PM, Andrew Rybchenko wrote:
> Since the topic is raised in multicast address list patchset, I'd like
> to highlight it here as well. Current version uses failsafe ops directly
> on sync to synchronize queues state which iterates over all sub-devices.
> For already in sync sub-devices it does not go to driver since ethdev
> functions checks current state and do nothing if it is already OK.
> In theory it is possible to limit it to inactive devices and use
> ethdev API instead of direct ops, but it requires a bit more lines of
> code.
> 
> v2:
>     - fix ops ordering
>     - update testpmd documentation
>     - add Gaëtan's acks
> 
> Ian Dolzhansky (4):
>   app/testpmd: add queue deferred start switch
>   net/failsafe: add checks for deferred queue setup
>   net/failsafe: add Rx queue start and stop functions
>   net/failsafe: add Tx queue start and stop functions

Series applied to dpdk-next-net/master, thanks.