[dpdk-dev] [PATCH v4 12/25] raw/ioat: add vdev probe for DSA/idxd devices

[Bruce Richardson <bruce.richardson@intel.com>]

From: Kevin Laatz <kevin.laatz@intel.com>

The Intel DSA devices can be exposed to userspace via kernel driver, so can
be used without having to bind them to vfio/uio. Therefore we add support
for using those kernel-configured devices as vdevs, taking as parameter the
individual HW work queue to be used by the vdev.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
 doc/guides/rawdevs/ioat.rst  |  68 +++++++++++++++++--
 drivers/raw/ioat/idxd_vdev.c | 123 +++++++++++++++++++++++++++++++++++
 drivers/raw/ioat/meson.build |   6 +-
 3 files changed, 192 insertions(+), 5 deletions(-)
 create mode 100644 drivers/raw/ioat/idxd_vdev.c

diff --git a/doc/guides/rawdevs/ioat.rst b/doc/guides/rawdevs/ioat.rst
index b898f98d5f..5b8d27980e 100644
--- a/doc/guides/rawdevs/ioat.rst
+++ b/doc/guides/rawdevs/ioat.rst
@@ -37,9 +37,62 @@ No additional compilation steps are necessary.
 Device Setup
 -------------
 
+Depending on support provided by the PMD, HW devices can either use the kernel configured driver
+or be bound to a user-space IO driver for use.
+For example, Intel\ |reg| DSA devices can use the IDXD kernel driver or DPDK-supported drivers,
+such as ``vfio-pci``.
+
+Intel\ |reg| DSA devices using idxd kernel driver
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To use a Intel\ |reg| DSA device bound to the IDXD kernel driver, the device must first be configured.
+The `accel-config <https://github.com/intel/idxd-config>`_ utility library can be used for configuration.
+
+.. note::
+        The device configuration can also be done by directly interacting with the sysfs nodes.
+
+There are some mandatory configuration steps before being able to use a device with an application.
+The internal engines, which do the copies or other operations,
+and the work-queues, which are used by applications to assign work to the device,
+need to be assigned to groups, and the various other configuration options,
+such as priority or queue depth, need to be set for each queue.
+
+To assign an engine to a group::
+
+        $ accel-config config-engine dsa0/engine0.0 --group-id=0
+        $ accel-config config-engine dsa0/engine0.1 --group-id=1
+
+To assign work queues to groups for passing descriptors to the engines a similar accel-config command can be used.
+However, the work queues also need to be configured depending on the use-case.
+Some configuration options include:
+
+* mode (Dedicated/Shared): Indicates whether a WQ may accept jobs from multiple queues simultaneously.
+* priority: WQ priority between 1 and 15. Larger value means higher priority.
+* wq-size: the size of the WQ. Sum of all WQ sizes must be less that the total-size defined by the device.
+* type: WQ type (kernel/mdev/user). Determines how the device is presented.
+* name: identifier given to the WQ.
+
+Example configuration for a work queue::
+
+        $ accel-config config-wq dsa0/wq0.0 --group-id=0 \
+           --mode=dedicated --priority=10 --wq-size=8 \
+           --type=user --name=app1
+
+Once the devices have been configured, they need to be enabled::
+
+        $ accel-config enable-device dsa0
+        $ accel-config enable-wq dsa0/wq0.0
+
+Check the device configuration::
+
+        $ accel-config list
+
+Devices using VFIO/UIO drivers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 The HW devices to be used will need to be bound to a user-space IO driver for use.
 The ``dpdk-devbind.py`` script can be used to view the state of the devices
-and to bind them to a suitable DPDK-supported kernel driver, such as ``vfio-pci``.
+and to bind them to a suitable DPDK-supported driver, such as ``vfio-pci``.
 For example::
 
 	$ dpdk-devbind.py -b vfio-pci 00:04.0 00:04.1
@@ -47,9 +100,16 @@ For example::
 Device Probing and Initialization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Once bound to a suitable kernel device driver, the HW devices will be found
-as part of the PCI scan done at application initialization time. No vdev
-parameters need to be passed to create or initialize the device.
+For devices bound to a suitable DPDK-supported VFIO/UIO driver, the HW devices will
+be found as part of the device scan done at application initialization time without
+the need to pass parameters to the application.
+
+If the device is bound to the IDXD kernel driver (and previously configured with sysfs),
+then a specific work queue needs to be passed to the application via a vdev parameter.
+This vdev parameter take the driver name and work queue name as parameters.
+For example, to use work queue 0 on Intel\ |reg| DSA instance 0::
+
+        $ dpdk-test --no-pci --vdev=rawdev_idxd,wq=0.0
 
 Once probed successfully, the device will appear as a ``rawdev``, that is a
 "raw device type" inside DPDK, and can be accessed using APIs from the
diff --git a/drivers/raw/ioat/idxd_vdev.c b/drivers/raw/ioat/idxd_vdev.c
new file mode 100644
index 0000000000..0509fc0842
--- /dev/null
+++ b/drivers/raw/ioat/idxd_vdev.c
@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include <rte_bus_vdev.h>
+#include <rte_kvargs.h>
+#include <rte_string_fns.h>
+#include <rte_rawdev_pmd.h>
+
+#include "ioat_private.h"
+
+/** Name of the device driver */
+#define IDXD_PMD_RAWDEV_NAME rawdev_idxd
+/* takes a work queue(WQ) as parameter */
+#define IDXD_ARG_WQ		"wq"
+
+static const char * const valid_args[] = {
+	IDXD_ARG_WQ,
+	NULL
+};
+
+struct idxd_vdev_args {
+	uint8_t device_id;
+	uint8_t wq_id;
+};
+
+static int
+idxd_rawdev_parse_wq(const char *key __rte_unused, const char *value,
+			  void *extra_args)
+{
+	struct idxd_vdev_args *args = (struct idxd_vdev_args *)extra_args;
+	int dev, wq, bytes = -1;
+	int read = sscanf(value, "%d.%d%n", &dev, &wq, &bytes);
+
+	if (read != 2 || bytes != (int)strlen(value)) {
+		IOAT_PMD_ERR("Error parsing work-queue id. Must be in <dev_id>.<queue_id> format");
+		return -EINVAL;
+	}
+
+	if (dev >= UINT8_MAX || wq >= UINT8_MAX) {
+		IOAT_PMD_ERR("Device or work queue id out of range");
+		return -EINVAL;
+	}
+
+	args->device_id = dev;
+	args->wq_id = wq;
+
+	return 0;
+}
+
+static int
+idxd_vdev_parse_params(struct rte_kvargs *kvlist, struct idxd_vdev_args *args)
+{
+	if (rte_kvargs_count(kvlist, IDXD_ARG_WQ) == 1) {
+		if (rte_kvargs_process(kvlist, IDXD_ARG_WQ,
+				&idxd_rawdev_parse_wq, args) < 0) {
+			IOAT_PMD_ERR("Error parsing %s", IDXD_ARG_WQ);
+			goto free;
+		}
+	} else {
+		IOAT_PMD_ERR("%s is a mandatory arg", IDXD_ARG_WQ);
+		return -EINVAL;
+	}
+
+	return 0;
+
+free:
+	if (kvlist)
+		rte_kvargs_free(kvlist);
+	return -EINVAL;
+}
+
+static int
+idxd_rawdev_probe_vdev(struct rte_vdev_device *vdev)
+{
+	struct rte_kvargs *kvlist;
+	struct idxd_vdev_args vdev_args;
+	const char *name;
+	int ret = 0;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+
+	IOAT_PMD_INFO("Initializing pmd_idxd for %s", name);
+
+	kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_args);
+	if (kvlist == NULL) {
+		IOAT_PMD_ERR("Invalid kvargs key");
+		return -EINVAL;
+	}
+
+	ret = idxd_vdev_parse_params(kvlist, &vdev_args);
+	if (ret) {
+		IOAT_PMD_ERR("Failed to parse kvargs");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+idxd_rawdev_remove_vdev(struct rte_vdev_device *vdev)
+{
+	const char *name;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+
+	IOAT_PMD_INFO("Remove DSA vdev %p", name);
+
+	return 0;
+}
+
+struct rte_vdev_driver idxd_rawdev_drv_vdev = {
+	.probe = idxd_rawdev_probe_vdev,
+	.remove = idxd_rawdev_remove_vdev,
+};
+
+RTE_PMD_REGISTER_VDEV(IDXD_PMD_RAWDEV_NAME, idxd_rawdev_drv_vdev);
+RTE_PMD_REGISTER_PARAM_STRING(IDXD_PMD_RAWDEV_NAME,
+			      "wq=<string>");
diff --git a/drivers/raw/ioat/meson.build b/drivers/raw/ioat/meson.build
index 3529635e9c..b343b7367b 100644
--- a/drivers/raw/ioat/meson.build
+++ b/drivers/raw/ioat/meson.build
@@ -5,9 +5,13 @@ build = dpdk_conf.has('RTE_ARCH_X86')
 reason = 'only supported on x86'
 sources = files(
 	'idxd_pci.c',
+	'idxd_vdev.c',
 	'ioat_rawdev.c',
 	'ioat_rawdev_test.c')
-deps += ['rawdev', 'bus_pci', 'mbuf']
+deps += ['bus_pci',
+	'bus_vdev',
+	'mbuf',
+	'rawdev']
 
 install_headers('rte_ioat_rawdev.h',
 		'rte_ioat_rawdev_fns.h')
-- 
2.25.1