[PATCH v2 12/13] net/failsafe: support device removal

[Gaetan Rivet <gaetan.rivet@6wind.com>]

Signed-off-by: Gaetan Rivet <gaetan.rivet@6wind.com>
Acked-by: Olga Shern <olgas@mellanox.com>
---
 doc/guides/nics/fail_safe.rst           | 14 +++++++
 drivers/net/failsafe/failsafe_args.c    | 22 +++++++++++
 drivers/net/failsafe/failsafe_eal.c     |  2 +
 drivers/net/failsafe/failsafe_ether.c   | 67 ++++++++++++++++++++++++++++++++-
 drivers/net/failsafe/failsafe_ops.c     | 21 +++++++++++
 drivers/net/failsafe/failsafe_private.h |  7 ++++
 6 files changed, 132 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/fail_safe.rst b/doc/guides/nics/fail_safe.rst
index bb8a221..8811ed3 100644
--- a/doc/guides/nics/fail_safe.rst
+++ b/doc/guides/nics/fail_safe.rst
@@ -51,6 +51,12 @@ The Fail-safe PMD only supports a limited set of features. If you plan to use a
 device underneath the Fail-safe PMD with a specific feature, this feature must
 be supported by the Fail-safe PMD to avoid throwing any error.
 
+A notable exception is the device removal feature. The fail-safe PMD being a
+virtual device, it cannot currently be removed in the sense of a specific bus
+hotplug, like for PCI for example. It will however enable this feature for its
+sub-device automatically, detecting those that are capable and register the
+relevant callback for such event.
+
 Check the feature matrix for the complete set of supported features.
 
 Compilation options
@@ -166,3 +172,11 @@ emit and receive packets. It will store any applied configuration, and try to
 apply it upon the probing of its missing sub-device. After this configuration
 pass, the new sub-device will be synchronized with other sub-devices, i.e. be
 started if the fail-safe PMD has been started by the user before.
+
+Plug-out feature
+----------------
+
+A sub-device supporting the device removal event can be removed from its bus at
+any time. The fail-safe PMD will register a callback for such event and react
+accordingly. It will try to safely stop, close and uninit the sub-device having
+emitted this event, allowing it to free its eventual resources.
diff --git a/drivers/net/failsafe/failsafe_args.c b/drivers/net/failsafe/failsafe_args.c
index 839831f..62033c4 100644
--- a/drivers/net/failsafe/failsafe_args.c
+++ b/drivers/net/failsafe/failsafe_args.c
@@ -462,6 +462,26 @@ failsafe_args_count_subdevice(struct rte_eth_dev *dev,
 				dev, params);
 }
 
+static int
+parse_sub_device(struct sub_device *sdev)
+{
+	struct rte_devargs *da;
+	char params[DEVARGS_MAXLEN] = "";
+
+	da = &sdev->devargs;
+	if (da->type == RTE_DEVTYPE_VIRTUAL)
+		snprintf(params, sizeof(params) - 1,
+			"%s,%s", da->virt.drv_name, da->args);
+	else
+		snprintf(params, sizeof(params) - 1,
+			PCI_PRI_FMT ",%s",
+			da->pci.addr.domain, da->pci.addr.bus,
+			da->pci.addr.devid, da->pci.addr.function,
+			da->args);
+
+	return parse_device(sdev, params);
+}
+
 int
 failsafe_args_parse_subs(struct rte_eth_dev *dev)
 {
@@ -474,6 +494,8 @@ failsafe_args_parse_subs(struct rte_eth_dev *dev)
 			continue;
 		if (sdev->cmdline)
 			ret = execute_cmd(sdev, sdev->cmdline);
+		else
+			ret = parse_sub_device(sdev);
 		if (ret == 0)
 			sdev->state = DEV_PARSED;
 	}
diff --git a/drivers/net/failsafe/failsafe_eal.c b/drivers/net/failsafe/failsafe_eal.c
index 9817fc9..8bb8d45 100644
--- a/drivers/net/failsafe/failsafe_eal.c
+++ b/drivers/net/failsafe/failsafe_eal.c
@@ -140,6 +140,7 @@ dev_init(struct rte_eth_dev *dev)
 			}
 			ETH(sdev)->state = RTE_ETH_DEV_DEFERRED;
 			SUB_ID(sdev) = i;
+			sdev->fs_dev = dev;
 			sdev->state = DEV_PROBED;
 		}
 	}
@@ -191,6 +192,7 @@ pci_probe(struct rte_eth_dev *dev)
 			}
 			ETH(sdev)->state = RTE_ETH_DEV_DEFERRED;
 			SUB_ID(sdev) = i;
+			sdev->fs_dev = dev;
 			sdev->state = DEV_PROBED;
 		}
 	}
diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c
index 8c73b4c..f12b8d7 100644
--- a/drivers/net/failsafe/failsafe_ether.c
+++ b/drivers/net/failsafe/failsafe_ether.c
@@ -33,6 +33,7 @@
 
 #include <unistd.h>
 
+#include <rte_alarm.h>
 #include <rte_flow.h>
 #include <rte_flow_driver.h>
 
@@ -256,6 +257,43 @@ eth_dev_conf_apply(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static void
+fs_dev_remove(void *arg)
+{
+	struct sub_device *sdev = arg;
+	struct rte_devargs *da;
+	struct rte_pci_device *pdev;
+
+	switch (sdev->state) {
+	case DEV_STARTED:
+		rte_eth_dev_stop(PORT_ID(sdev));
+		sdev->state = DEV_ACTIVE;
+		/* fallthrough */
+	case DEV_ACTIVE:
+		rte_eth_dev_close(PORT_ID(sdev));
+		sdev->state = DEV_PROBED;
+		/* fallthrough */
+	case DEV_PROBED:
+		da = &sdev->devargs;
+		if (da->type == RTE_DEVTYPE_WHITELISTED_PCI) {
+			pdev = &sdev->pci_device;
+			rte_eal_pci_detach_all_drivers(pdev);
+		} else if (da->type == RTE_DEVTYPE_VIRTUAL) {
+			rte_eal_vdev_uninit(da->virt.drv_name);
+		}
+		sdev->eth_dev->state = RTE_ETH_DEV_UNUSED;
+		sdev->state = DEV_PARSED;
+		/* fallthrough */
+	case DEV_SCANNED:
+	case DEV_PARSED:
+	case DEV_UNDEFINED:
+		sdev->state = DEV_UNDEFINED;
+		/* the end */
+		break;
+	}
+	failsafe_plugin_alarm_install(sdev->fs_dev);
+}
+
 int
 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 {
@@ -291,7 +329,7 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 			if (ret) {
 				ERROR("Could not apply configuration to sub_device %d",
 				      i);
-				/* TODO: disable device */
+				fs_dev_remove(sdev);
 				return ret;
 			}
 		}
@@ -309,3 +347,30 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 		return ret;
 	return 0;
 }
+
+void
+failsafe_eth_rmv_event_callback(uint8_t port_id __rte_unused,
+				enum rte_eth_event_type event,
+				void *cb_arg)
+{
+	struct sub_device *sdev = cb_arg;
+	enum dev_state state;
+
+	if (event != RTE_ETH_EVENT_INTR_RMV) {
+		ERROR("Incorrect event");
+		return;
+	}
+	/* Switch as soon as possible tx_dev. */
+	state = sdev->state;
+	sdev->state = DEV_UNDEFINED;
+	fs_switch_dev(sdev->fs_dev);
+	sdev->state = state;
+	/*
+	 * Async removal, the sub-PMD will try to unregister
+	 * the callback at the source of the current thread context.
+	 */
+	if (rte_eal_alarm_set(FAILSAFE_PLUGOUT_ASYNC_RESCHED_US,
+			      fs_dev_remove,
+			      cb_arg))
+		ERROR("Could not set up deferred sub_device removal");
+}
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index 2a4d102..8d0e7a2 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -198,8 +198,19 @@ fs_dev_configure(struct rte_eth_dev *dev)
 		}
 	}
 	FOREACH_SUBDEV(sdev, i, dev) {
+		int rmv_interrupt = 0;
+
 		if (sdev->state != DEV_PROBED)
 			continue;
+
+		rmv_interrupt = ETH(sdev)->data->dev_flags &
+				RTE_ETH_DEV_INTR_RMV;
+		if (rmv_interrupt) {
+			DEBUG("Enabling RMV interrupts for sub_device %d", i);
+			dev->data->dev_conf.intr_conf.rmv = 1;
+		} else {
+			DEBUG("sub_device %d does not support RMV event", i);
+		}
 		DEBUG("Configuring sub-device %d", i);
 		ret = rte_eth_dev_configure(PORT_ID(sdev),
 					dev->data->nb_rx_queues,
@@ -209,6 +220,16 @@ fs_dev_configure(struct rte_eth_dev *dev)
 			ERROR("Could not configure sub_device %d", i);
 			return ret;
 		}
+		if (rmv_interrupt) {
+			ret = rte_eth_dev_callback_register(PORT_ID(sdev),
+					RTE_ETH_EVENT_INTR_RMV,
+					failsafe_eth_rmv_event_callback,
+					sdev);
+			if (ret)
+				WARN("Failed to register RMV callback for sub_device %d",
+				     SUB_ID(sdev));
+		}
+		dev->data->dev_conf.intr_conf.rmv = 0;
 		sdev->state = DEV_ACTIVE;
 	}
 	if (PRIV(dev)->state < DEV_ACTIVE)
diff --git a/drivers/net/failsafe/failsafe_private.h b/drivers/net/failsafe/failsafe_private.h
index faf0e71..5efd084 100644
--- a/drivers/net/failsafe/failsafe_private.h
+++ b/drivers/net/failsafe/failsafe_private.h
@@ -53,6 +53,7 @@
 	""
 
 #define FAILSAFE_PLUGIN_DEFAULT_TIMEOUT_MS 2000
+#define FAILSAFE_PLUGOUT_ASYNC_RESCHED_US 100000
 
 #define FAILSAFE_MAX_ETHPORTS (RTE_MAX_ETHPORTS - 1)
 #define FAILSAFE_MAX_ETHADDR 128
@@ -108,6 +109,9 @@ struct sub_device {
 	enum dev_state state;
 	/* Some device are defined as a command line */
 	char *cmdline;
+
+	/* fail-safe device backreference */
+	struct rte_eth_dev *fs_dev;
 };
 
 struct fs_priv {
@@ -175,6 +179,9 @@ int failsafe_eal_uninit(struct rte_eth_dev *dev);
 /* ETH_DEV */
 
 int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev);
+void failsafe_eth_rmv_event_callback(uint8_t port_id,
+				     enum rte_eth_event_type type,
+				     void *arg);
 
 /* GLOBALS */
 
-- 
2.1.4